Merge pull request #7418 from wainersm/release_0.7.0-fix

workflows: fix malformed CC payload s390x yaml

.github/workflows/cc-payload-after-push-amd64.yaml

@@ -0,0 +1,170 @@
+name: CI | Publish CC runtime payload for amd64
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-asset:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        measured_rootfs:
+          - no
+        asset:
+          - cc-cloud-hypervisor
+          - cc-qemu
+          - cc-virtiofsd
+          - cc-sev-kernel
+          - cc-sev-ovmf
+          - cc-x86_64-ovmf
+          - cc-snp-qemu
+          - cc-sev-rootfs-initrd
+          - cc-tdx-qemu
+          - cc-tdx-td-shim
+          - cc-tdx-tdvf
+        include:
+          - measured_rootfs: yes
+            asset: cc-kernel
+          - measured_rootfs: yes
+            asset: cc-tdx-kernel
+          - measured_rootfs: yes
+            asset: cc-rootfs-image
+          - measured_rootfs: yes
+            asset: cc-tdx-rootfs-image
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: yes
+          MEASURED_ROOTFS: ${{ matrix.measured_rootfs }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+      - name: store-artifact root_hash_tdx.txt 
+        uses: actions/upload-artifact@v3
+        with:
+          name: root_hash_tdx.txt
+          path: tools/osbuilder/root_hash_tdx.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+      - name: store-artifact root_hash_vanilla.txt 
+        uses: actions/upload-artifact@v3
+        with:
+          name: root_hash_vanilla.txt
+          path: tools/osbuilder/root_hash_vanilla.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+  build-asset-cc-shim-v2:
+    runs-on: ubuntu-latest
+    needs: build-asset
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+
+      - name: Get root_hash_tdx.txt
+        uses: actions/download-artifact@v3
+        with:
+          name: root_hash_tdx.txt
+          path: tools/osbuilder/
+
+      - name: Get root_hash_vanilla.txt
+        uses: actions/download-artifact@v3
+        with:
+          name: root_hash_vanilla.txt
+          path: tools/osbuilder/
+
+      - name: Build cc-shim-v2
+        run: |
+          make cc-shim-v2-tarball
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          PUSH_TO_REGISTRY: yes
+          MEASURED_ROOTFS: yes
+
+      - name: store-artifact cc-shim-v2
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-cc-shim-v2.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: [build-asset, build-asset-cc-shim-v2]
+    steps:
+      - uses: actions/checkout@v3
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  kata-payload:
+    needs: create-kata-tarball
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to Confidential Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz "quay.io/confidential-containers/runtime-payload-ci" \
+          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/cc-payload-after-push-s390x.yaml

@@ -0,0 +1,171 @@
+name: CI | Publish CC runtime payload for s390x
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-asset:
+    runs-on: s390x
+    strategy:
+      matrix:
+        measured_rootfs:
+          - no
+        asset:
+          - cc-qemu
+          - cc-rootfs-initrd
+          - cc-se-image
+          - cc-virtiofsd
+        include:
+          - measured_rootfs: yes
+            asset: cc-kernel
+          - measured_rootfs: yes
+            asset: cc-rootfs-image
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Place a host key document
+        run: |
+          mkdir -p "host-key-document"
+          cp "${CI_HKD_PATH}" "host-key-document"
+        env:
+          CI_HKD_PATH: ${{ secrets.CI_HKD_PATH }}
+
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+          sudo chown -R $(id -u):$(id -g) "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: yes
+          MEASURED_ROOTFS: ${{ matrix.measured_rootfs }}
+          HKD_PATH: "host-key-document"
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+      - name: store-artifact root_hash_vanilla.txt 
+        uses: actions/upload-artifact@v3
+        with:
+          name: root_hash_vanilla.txt-s390x
+          path: tools/osbuilder/root_hash_vanilla.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+  build-asset-cc-shim-v2:
+    runs-on: s390x
+    needs: build-asset
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+
+      - name: Get root_hash_vanilla.txt
+        uses: actions/download-artifact@v3
+        with:
+          name: root_hash_vanilla.txt-s390x
+          path: tools/osbuilder/
+
+      - name: Build cc-shim-v2
+        run: |
+          make cc-shim-v2-tarball
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          PUSH_TO_REGISTRY: yes
+          MEASURED_ROOTFS: yes
+
+      - name: store-artifact cc-shim-v2
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-build/kata-static-cc-shim-v2.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: s390x
+    needs: [build-asset, build-asset-cc-shim-v2]
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  kata-payload:
+    needs: create-kata-tarball
+    runs-on: s390x
+    steps:
+      - name: Login to Confidential Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz "quay.io/confidential-containers/runtime-payload-ci" \
+          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/cc-payload-after-push.yaml

@@ -0,0 +1,47 @@
+name: CI | Publish Kata Containers payload for Confidential Containers
+on:
+  push:
+    branches:
+      - CCv0
+  workflow_dispatch:
+
+jobs:
+  build-assets-amd64:
+    uses: ./.github/workflows/cc-payload-after-push-amd64.yaml
+    with:
+      target-arch: amd64
+    secrets: inherit
+
+  build-assets-s390x:
+    uses: ./.github/workflows/cc-payload-after-push-s390x.yaml
+    with:
+      target-arch: s390x
+    secrets: inherit
+
+  publish:
+    runs-on: ubuntu-latest
+    needs: [build-assets-amd64, build-assets-s390x]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Login to Confidential Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Push commit multi-arch manifest
+        run: |
+          docker manifest create quay.io/confidential-containers/runtime-payload-ci:kata-containers-${GITHUB_SHA} \
+          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-${GITHUB_SHA}-amd64 \
+          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-${GITHUB_SHA}-s390x
+          docker manifest push quay.io/confidential-containers/runtime-payload-ci:kata-containers-${GITHUB_SHA}
+
+      - name: Push latest multi-arch manifest
+        run: |
+          docker manifest create quay.io/confidential-containers/runtime-payload-ci:kata-containers-latest \
+          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-amd64 \
+          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-s390x
+          docker manifest push quay.io/confidential-containers/runtime-payload-ci:kata-containers-latest

.github/workflows/cc-payload-amd64.yaml

@@ -0,0 +1,154 @@
+name: Publish Kata Containers payload for Confidential Containers (amd64)
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-asset:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        measured_rootfs:
+          - no
+        asset:
+          - cc-cloud-hypervisor
+          - cc-qemu
+          - cc-virtiofsd
+          - cc-sev-kernel
+          - cc-sev-ovmf
+          - cc-x86_64-ovmf
+          - cc-snp-qemu
+          - cc-sev-rootfs-initrd
+          - cc-tdx-qemu
+          - cc-tdx-td-shim
+          - cc-tdx-tdvf
+        include:
+          - measured_rootfs: yes
+            asset: cc-kernel
+          - measured_rootfs: yes
+            asset: cc-tdx-kernel
+          - measured_rootfs: yes
+            asset: cc-rootfs-image
+          - measured_rootfs: yes
+            asset: cc-tdx-rootfs-image
+    steps:
+      - uses: actions/checkout@v3
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          MEASURED_ROOTFS: ${{ matrix.measured_rootfs }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+      - name: store-artifact root_hash_tdx.txt 
+        uses: actions/upload-artifact@v3
+        with:
+          name: root_hash_tdx.txt
+          path: tools/osbuilder/root_hash_tdx.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+      - name: store-artifact root_hash_vanilla.txt 
+        uses: actions/upload-artifact@v3
+        with:
+          name: root_hash_vanilla.txt
+          path: tools/osbuilder/root_hash_vanilla.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+  build-asset-cc-shim-v2:
+    runs-on: ubuntu-latest
+    needs: build-asset
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Get root_hash_tdx.txt
+        uses: actions/download-artifact@v3
+        with:
+          name: root_hash_tdx.txt
+          path: tools/osbuilder/
+
+      - name: Get root_hash_vanilla.txt
+        uses: actions/download-artifact@v3
+        with:
+          name: root_hash_vanilla.txt
+          path: tools/osbuilder/
+
+      - name: Build cc-shim-v2
+        run: |
+          make cc-shim-v2-tarball
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          MEASURED_ROOTFS: yes
+
+      - name: store-artifact cc-shim-v2
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-cc-shim-v2.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: [build-asset, build-asset-cc-shim-v2]
+    steps:
+      - uses: actions/checkout@v3
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  kata-payload:
+    needs: create-kata-tarball
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz \
+          "quay.io/confidential-containers/runtime-payload" \
+          "kata-containers-${{ inputs.target-arch }}"
+

.github/workflows/cc-payload-s390x.yaml

@@ -0,0 +1,142 @@
+name: Publish Kata Containers payload for Confidential Containers (s390x)
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-asset:
+    runs-on: s390x
+    strategy:
+      matrix:
+        measured_rootfs:
+          - no
+        asset:
+          - cc-qemu
+          - cc-virtiofsd
+        include:
+          - measured_rootfs: yes
+            asset: cc-kernel
+          - measured_rootfs: yes
+            asset: cc-rootfs-image
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          MEASURED_ROOTFS: ${{ matrix.measured_rootfs }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+      - name: store-artifact root_hash_vanilla.txt 
+        uses: actions/upload-artifact@v3
+        with:
+          name: root_hash_vanilla.txt-s390x
+          path: tools/osbuilder/root_hash_vanilla.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+  build-asset-cc-shim-v2:
+    runs-on: s390x
+    needs: build-asset
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+
+      - name: Get root_hash_vanilla.txt
+        uses: actions/download-artifact@v3
+        with:
+          name: root_hash_vanilla.txt-s390x
+          path: tools/osbuilder/
+
+      - name: Build cc-shim-v2
+        run: |
+          make cc-shim-v2-tarball
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          MEASURED_ROOTFS: yes
+
+      - name: store-artifact cc-shim-v2
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-build/kata-static-cc-shim-v2.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: s390x
+    needs: [build-asset, build-asset-cc-shim-v2]
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  kata-payload:
+    needs: create-kata-tarball
+    runs-on: s390x
+    steps:
+      - name: Login to quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz \
+          "quay.io/confidential-containers/runtime-payload" \
+          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/cc-payload.yaml

@@ -0,0 +1,46 @@
+name: Publish Kata Containers payload for Confidential Containers
+on:
+  push:
+    tags:
+      - 'CC\-[0-9]+.[0-9]+.[0-9]+'
+
+jobs:
+  build-assets-amd64:
+    uses: ./.github/workflows/cc-payload-amd64.yaml
+    with:
+      target-arch: amd64
+    secrets: inherit
+
+  build-assets-s390x:
+    uses: ./.github/workflows/cc-payload-s390x.yaml
+    with:
+      target-arch: s390x
+    secrets: inherit
+
+  publish:
+    runs-on: ubuntu-latest
+    needs: [build-assets-amd64, build-assets-s390x]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Login to Confidential Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Push commit multi-arch manifest
+        run: |
+          docker manifest create quay.io/confidential-containers/runtime-payload:kata-containers-${GITHUB_SHA} \
+          --amend quay.io/confidential-containers/runtime-payload:kata-containers-${GITHUB_SHA}-amd64 \
+          --amend quay.io/confidential-containers/runtime-payload:kata-containers-${GITHUB_SHA}-s390x
+          docker manifest push quay.io/confidential-containers/runtime-payload:kata-containers-${GITHUB_SHA}
+
+      - name: Push latest multi-arch manifest
+        run: |
+          docker manifest create quay.io/confidential-containers/runtime-payload:kata-containers-latest \
+          --amend quay.io/confidential-containers/runtime-payload:kata-containers-amd64 \
+          --amend quay.io/confidential-containers/runtime-payload:kata-containers-s390x
+          docker manifest push quay.io/confidential-containers/runtime-payload:kata-containers-latest

.github/workflows/commit-message-check.yaml

@@ -47,7 +47,7 @@ jobs:
       uses: tim-actions/commit-message-checker-with-regex@v0.3.1
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
-        pattern: '^.{0,75}(\n.*)*$'
+        pattern: '^.{0,75}(\n.*)*$|^Merge pull request (?:kata-containers)?#[\d]+ from.*'
         error: 'Subject too long (max 75)'
         post_error: ${{ env.error_msg }}
 
@@ -98,6 +98,6 @@ jobs:
       uses: tim-actions/commit-message-checker-with-regex@v0.3.1
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
-        pattern: '^[\s\t]*[^:\s\t]+[\s\t]*:'
+        pattern: '^[\s\t]*[^:\s\t]+[\s\t]*:|^Merge pull request (?:kata-containers)?#[\d]+ from.*'
         error: 'Failed to find subsystem in subject'
         post_error: ${{ env.error_msg }}

.github/workflows/deploy-ccv0-demo.yaml

@@ -0,0 +1,124 @@
+on:
+  issue_comment:
+    types: [created, edited]
+
+name: deploy-ccv0-demo
+
+jobs:
+  check-comment-and-membership:
+    runs-on: ubuntu-latest
+    if: |
+      github.event.issue.pull_request
+      && github.event_name == 'issue_comment'
+      && github.event.action == 'created'
+      && startsWith(github.event.comment.body, '/deploy-ccv0-demo')
+    steps:
+      - name: Check membership
+        uses: kata-containers/is-organization-member@1.0.1
+        id: is_organization_member
+        with:
+          organization: kata-containers
+          username: ${{ github.event.comment.user.login }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Fail if not member
+        run: |
+          result=${{ steps.is_organization_member.outputs.result }}
+          if [ $result == false ]; then
+              user=${{ github.event.comment.user.login }}
+              echo Either ${user} is not part of the kata-containers organization
+              echo or ${user} has its Organization Visibility set to Private at
+              echo https://github.com/orgs/kata-containers/people?query=${user}
+              echo 
+              echo Ensure you change your Organization Visibility to Public and
+              echo trigger the test again.
+              exit 1
+          fi
+
+  build-asset:
+    runs-on: ubuntu-latest
+    needs: check-comment-and-membership
+    strategy:
+      matrix:
+        asset:
+          - cloud-hypervisor
+          - firecracker
+          - kernel
+          - qemu
+          - rootfs-image
+          - rootfs-initrd
+          - shim-v2
+    steps:
+      - uses: actions/checkout@v2
+      - name: Prepare confidential container rootfs
+        if: ${{ matrix.asset == 'rootfs-initrd' }}
+        run: |
+          pushd include_rootfs/etc
+          curl -LO https://raw.githubusercontent.com/confidential-containers/documentation/main/demos/ssh-demo/aa-offline_fs_kbc-keys.json
+          mkdir kata-containers
+          envsubst < docs/how-to/data/confidential-agent-config.toml.in > kata-containers/agent.toml
+          popd
+        env:
+          AA_KBC_PARAMS: offline_fs_kbc::null
+
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          AA_KBC: offline_fs_kbc
+          INCLUDE_ROOTFS: include_rootfs
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: build-asset
+    steps:
+      - uses: actions/checkout@v2
+      - name: get-artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: kata-artifacts
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: kata-static-tarball
+          path: kata-static.tar.xz
+
+  kata-deploy:
+    needs: create-kata-tarball
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v2
+        with:
+          name: kata-static-tarball
+      - name: build-and-push-kata-deploy-ci
+        id: build-and-push-kata-deploy-ci
+        run: |
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          pushd $GITHUB_WORKSPACE
+          git checkout $tag
+          pkg_sha=$(git rev-parse HEAD)
+          popd
+          mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
+          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t quay.io/confidential-containers/runtime-payload:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
+          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
+          docker push quay.io/confidential-containers/runtime-payload:$pkg_sha
+          mkdir -p packaging/kata-deploy
+          ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
+          echo "::set-output name=PKG_SHA::${pkg_sha}"

.github/workflows/static-checks.yaml

@@ -41,8 +41,8 @@ jobs:
         cd "${{ github.workspace }}/src/github.com/${{ github.repository }}"
         kernel_dir="tools/packaging/kernel/"
         kernel_version_file="${kernel_dir}kata_config_version"
-        modified_files=$(git diff --name-only origin/main..HEAD)
-        if git diff --name-only origin/main..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then
+        modified_files=$(git diff --name-only origin/CCv0..HEAD)
+        if git diff --name-only origin/CCv0..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then
           echo "Kernel directory has changed, checking if $kernel_version_file has been updated"
           if echo "$modified_files" | grep -v "README.md" | grep "${kernel_dir}" >>"/dev/null"; then
             echo "$modified_files" | grep "$kernel_version_file" >>/dev/null || ( echo "Please bump version in $kernel_version_file" && exit 1)

ci/install_libseccomp.sh

@@ -72,7 +72,8 @@ build_and_install_gperf() {
     curl -sLO "${gperf_tarball_url}"
     tar -xf "${gperf_tarball}"
     pushd "gperf-${gperf_version}"
-    # Unset $CC for configure, we will always use native for gperf
+    # gperf is a build time dependency of libseccomp and not to be used in the target.
+    # Unset $CC since that might point to a cross compiler.
     CC= ./configure --prefix="${gperf_install_dir}"
     make
     make install

ci/lib.sh

@@ -64,3 +64,86 @@ run_get_pr_changed_file_details()
 	source "$tests_repo_dir/.ci/lib.sh"
 	get_pr_changed_file_details
 }
+
+# Check if the 1st argument version is greater than and equal to 2nd one
+# Version format: [0-9]+ separated by period (e.g. 2.4.6, 1.11.3 and etc.)
+#
+# Parameters:
+#	$1	- a version to be tested
+#	$2	- a target version
+#
+# Return:
+# 	0 if $1 is greater than and equal to $2
+#	1 otherwise
+version_greater_than_equal() {
+	local current_version=$1
+	local target_version=$2
+	smaller_version=$(echo -e "$current_version\n$target_version" | sort -V | head -1)
+	if [ "${smaller_version}" = "${target_version}" ]; then
+		return 0
+	else
+		return 1
+	fi
+}
+
+# Build a IBM zSystem secure execution (SE) image
+#
+# Parameters:
+#	$1	- kernel_parameters
+#	$2	- a source directory where kernel and initrd are located
+#	$3	- a destination directory where a SE image is built
+#
+# Return:
+# 	0 if the image is successfully built
+#	1 otherwise
+build_secure_image() {
+	kernel_params="${1:-}"
+	install_src_dir="${2:-}"
+	install_dest_dir="${3:-}"
+
+	if [ ! -f "${install_src_dir}/vmlinuz.container" ] ||
+		[ ! -f "${install_src_dir}/kata-containers-initrd.img" ]; then
+		cat << EOF >&2
+Either kernel or initrd does not exist or is mistakenly named
+A file name for kernel must be vmlinuz.container (raw binary)
+A file name for initrd must be kata-containers-initrd.img
+EOF
+		return 1
+	fi
+
+	cmdline="${kernel_params} panic=1 scsi_mod.scan=none swiotlb=262144"
+	parmfile="$(mktemp --suffix=-cmdline)"
+	echo "${cmdline}" > "${parmfile}"
+	chmod 600 "${parmfile}"
+
+	[ -n "${HKD_PATH:-}" ] || (echo >&2 "No host key document specified." && return 1)
+	cert_list=($(ls -1 $HKD_PATH))
+	declare hkd_options
+	eval "for cert in ${cert_list[*]}; do
+		hkd_options+=\"--host-key-document=\\\"\$HKD_PATH/\$cert\\\" \"
+	done"
+
+	command -v genprotimg > /dev/null 2>&1 || { apt update; apt install -y s390-tools; }
+	extra_arguments=""
+	genprotimg_version=$(genprotimg --version | grep -Po '(?<=version )[^-]+')
+	if ! version_greater_than_equal "${genprotimg_version}" "2.17.0"; then
+		extra_arguments="--x-pcf '0xe0'"
+	fi
+
+	eval genprotimg \
+		"${extra_arguments}" \
+		"${hkd_options}" \
+		--output="${install_dest_dir}/kata-containers-secure.img" \
+		--image="${install_src_dir}/vmlinuz.container" \
+		--ramdisk="${install_src_dir}/kata-containers-initrd.img" \
+		--parmfile="${parmfile}" \
+		--no-verify # no verification for CI testing purposes
+
+	build_result=$?
+	rm -f "${parmfile}"
+	if [ $build_result -eq 0 ]; then
+		return 0
+	else
+		return 1
+	fi
+}

docs/how-to/README.md

@@ -45,4 +45,8 @@
 - [How to run Kata Containers with `nydus`](how-to-use-virtio-fs-nydus-with-kata.md)
 - [How to run Kata Containers with AMD SEV-SNP](how-to-run-kata-containers-with-SNP-VMs.md)
 - [How to use EROFS to build rootfs in Kata Containers](how-to-use-erofs-build-rootfs.md)
-- [How to run Kata Containers with kinds of Block Volumes](how-to-run-kata-containers-with-kinds-of-Block-Volumes.md)
+- [How to run Kata Containers with kinds of Block Volumes](how-to-run-kata-containers-with-kinds-of-Block-Volumes.md)
+
+## Confidential Containers
+- [How to use build and test the Confidential Containers `CCv0` proof of concept](how-to-build-and-test-ccv0.md)
+- [How to generate a Kata Containers payload for the Confidential Containers Operator](how-to-generate-a-kata-containers-payload-for-the-confidential-containers-operator.md)

docs/how-to/ccv0.sh

@@ -0,0 +1,635 @@
+#!/bin/bash -e
+#
+# Copyright (c) 2021, 2023 IBM Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# Disclaimer: This script is work in progress for supporting the CCv0 prototype
+# It shouldn't be considered supported by the Kata Containers community, or anyone else
+
+# Based on https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md,
+# but with elements of the tests/.ci scripts used
+
+readonly script_name="$(basename "${BASH_SOURCE[0]}")"
+
+# By default in Golang >= 1.16 GO111MODULE is set to "on", but not all modules support it, so overwrite to "auto"
+export GO111MODULE="auto"
+
+# Setup kata containers environments if not set - we default to use containerd
+export CRI_CONTAINERD=${CRI_CONTAINERD:-"yes"}
+export CRI_RUNTIME=${CRI_RUNTIME:-"containerd"}
+export CRIO=${CRIO:-"no"}
+export KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}"
+export KUBERNETES=${KUBERNETES:-"no"}
+export AGENT_INIT="${AGENT_INIT:-${TEST_INITRD:-no}}"
+export AA_KBC="${AA_KBC:-offline_fs_kbc}"
+export KATA_BUILD_CC=${KATA_BUILD_CC:-"yes"}
+export TEE_TYPE=${TEE_TYPE:-}
+export PREFIX="${PREFIX:-/opt/confidential-containers}"
+export RUNTIME_CONFIG_PATH="${RUNTIME_CONFIG_PATH:-${PREFIX}/share/defaults/kata-containers/configuration.toml}"
+
+# Allow the user to overwrite the default repo and branch names if they want to build from a fork
+export katacontainers_repo="${katacontainers_repo:-github.com/kata-containers/kata-containers}"
+export katacontainers_branch="${katacontainers_branch:-CCv0}" 
+export kata_default_branch=${katacontainers_branch}
+export tests_repo="${tests_repo:-github.com/kata-containers/tests}"
+export tests_branch="${tests_branch:-CCv0}"
+export target_branch=${tests_branch} # kata-containers/ci/lib.sh uses target branch var to check out tests repo
+
+# if .bash_profile exists then use it, otherwise fall back to .profile
+export PROFILE="${HOME}/.profile"
+if [ -r "${HOME}/.bash_profile" ]; then
+    export PROFILE="${HOME}/.bash_profile"
+fi
+# Stop PS1: unbound variable error happening
+export PS1=${PS1:-}
+
+# Create a bunch of common, derived values up front so we don't need to create them in all the different functions
+. ${PROFILE}
+if [ -z ${GOPATH} ]; then
+    export GOPATH=${HOME}/go
+fi
+export tests_repo_dir="${GOPATH}/src/${tests_repo}"
+export katacontainers_repo_dir="${GOPATH}/src/${katacontainers_repo}"
+export ROOTFS_DIR="${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder/rootfs"
+export PULL_IMAGE="${PULL_IMAGE:-quay.io/kata-containers/confidential-containers:signed}" # Doesn't need authentication
+export CONTAINER_ID="${CONTAINER_ID:-0123456789}"
+source /etc/os-release || source /usr/lib/os-release
+grep -Eq "\<fedora\>" /etc/os-release 2> /dev/null && export USE_PODMAN=true
+
+
+# If we've already checked out the test repo then source the confidential scripts
+if [ "${KUBERNETES}" == "yes" ]; then
+    export BATS_TEST_DIRNAME="${tests_repo_dir}/integration/kubernetes/confidential"
+    [ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/lib.sh"
+else
+    export BATS_TEST_DIRNAME="${tests_repo_dir}/integration/containerd/confidential"
+    [ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/lib.sh"
+fi
+
+[ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
+
+usage() {
+    exit_code="$1"
+    cat <<EOF
+Overview:
+    Build and test kata containers from source
+    Optionally set kata-containers and tests repo and branch as exported variables before running
+    e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && ~/${script_name} build_and_install_all
+Usage:
+    ${script_name} [options] <command>
+Commands:
+- agent_create_container:           Run CreateContainer command against the agent with agent-ctl
+- agent_pull_image:                 Run PullImage command against the agent with agent-ctl
+- all:                              Build and install everything, test kata with containerd and capture the logs
+- build_and_add_agent_to_rootfs:    Builds the kata-agent and adds it to the rootfs
+- build_and_install_all:            Build and install everything
+- build_and_install_rootfs:         Builds and installs the rootfs image
+- build_kata_runtime:               Build and install the kata runtime
+- build_cloud_hypervisor            Checkout, patch, build and install Cloud Hypervisor
+- build_qemu:                       Checkout, patch, build and install QEMU
+- configure:                        Configure Kata to use rootfs and enable debug
+- connect_to_ssh_demo_pod:          Ssh into the ssh demo pod, showing that the decryption succeeded
+- copy_signature_files_to_guest     Copies signature verification files to guest
+- create_rootfs:                    Create a local rootfs
+- crictl_create_cc_container        Use crictl to create a new busybox container in the kata cc pod
+- crictl_create_cc_pod              Use crictl to create a new kata cc pod
+- crictl_delete_cc                  Use crictl to delete the kata cc pod sandbox and container in it
+- help:                             Display this help
+- init_kubernetes:                  initialize a Kubernetes cluster on this system
+- initialize:                       Install dependencies and check out kata-containers source
+- install_guest_kernel:             Setup, build and install the guest kernel
+- kubernetes_create_cc_pod:         Create a Kata CC runtime busybox-based pod in Kubernetes
+- kubernetes_create_ssh_demo_pod:   Create a Kata CC runtime pod based on the ssh demo
+- kubernetes_delete_cc_pod:         Delete the Kata CC runtime busybox-based pod in Kubernetes
+- kubernetes_delete_ssh_demo_pod:   Delete the Kata CC runtime pod based on the ssh demo
+- open_kata_shell:                  Open a shell into the kata runtime
+- rebuild_and_install_kata:         Rebuild the kata runtime and agent and build and install the image
+- shim_pull_image:                  Run PullImage command against the shim with ctr
+- test_capture_logs:                Test using kata with containerd and capture the logs in the user's home directory
+- test:                             Test using kata with containerd
+
+Options:
+    -d: Enable debug
+    -h: Display this help
+EOF
+    # if script sourced don't exit as this will exit the main shell, just return instead
+    [[ $_ != $0 ]] && return "$exit_code" || exit "$exit_code"
+}
+
+build_and_install_all() {
+    initialize
+    build_and_install_kata_runtime
+    configure
+    create_a_local_rootfs
+    build_and_install_rootfs
+    install_guest_kernel_image
+    case "$KATA_HYPERVISOR" in
+        "qemu") 
+            build_qemu
+            ;;
+        "cloud-hypervisor") 
+            build_cloud_hypervisor
+            ;;
+        *)
+            echo "Invalid option: $KATA_HYPERVISOR is not supported." >&2
+            ;;
+    esac
+
+    check_kata_runtime
+    if [ "${KUBERNETES}" == "yes" ]; then
+        init_kubernetes
+    fi
+}
+
+rebuild_and_install_kata() {
+    checkout_tests_repo
+    checkout_kata_containers_repo
+    build_and_install_kata_runtime
+    build_and_add_agent_to_rootfs
+    build_and_install_rootfs
+    check_kata_runtime
+}
+
+# Based on the jenkins_job_build.sh script in kata-containers/tests/.ci - checks out source code and installs dependencies
+initialize() {
+    # We need git to checkout and bootstrap the ci scripts and some other packages used in testing
+    sudo apt-get update && sudo apt-get install -y curl git qemu-utils
+    
+    grep -qxF "export GOPATH=\${HOME}/go" "${PROFILE}" || echo "export GOPATH=\${HOME}/go" >> "${PROFILE}"
+    grep -qxF "export GOROOT=/usr/local/go" "${PROFILE}" || echo "export GOROOT=/usr/local/go" >> "${PROFILE}"
+    grep -qxF "export PATH=\${GOPATH}/bin:/usr/local/go/bin:\${PATH}" "${PROFILE}" || echo "export PATH=\${GOPATH}/bin:/usr/local/go/bin:\${PATH}" >> "${PROFILE}"
+    
+    # Load the new go and PATH parameters from the profile
+    . ${PROFILE}
+    mkdir -p "${GOPATH}"
+
+    checkout_tests_repo
+
+    pushd "${tests_repo_dir}"
+    local ci_dir_name=".ci"
+    sudo -E PATH=$PATH -s "${ci_dir_name}/install_go.sh" -p -f
+    sudo -E PATH=$PATH -s "${ci_dir_name}/install_rust.sh"
+    # Need to change ownership of rustup so later process can create temp files there
+    sudo chown -R ${USER}:${USER} "${HOME}/.rustup"
+
+    checkout_kata_containers_repo
+
+    # Run setup, but don't install kata as we will build it ourselves in locations matching the developer guide
+    export INSTALL_KATA="no"
+    sudo -E PATH=$PATH -s ${ci_dir_name}/setup.sh
+    # Reload the profile to pick up installed dependencies
+    . ${PROFILE}
+    popd
+}
+
+checkout_tests_repo() {
+    echo "Creating repo: ${tests_repo} and branch ${tests_branch} into ${tests_repo_dir}..."
+    # Due to git https://github.blog/2022-04-12-git-security-vulnerability-announced/ the tests repo needs
+    # to be owned by root as it is re-checked out in rootfs.sh
+    mkdir -p $(dirname "${tests_repo_dir}")
+    [ -d "${tests_repo_dir}" ] || sudo -E git clone "https://${tests_repo}.git" "${tests_repo_dir}"
+    sudo -E chown -R root:root "${tests_repo_dir}"
+    pushd "${tests_repo_dir}"
+    sudo -E git fetch
+    if [ -n "${tests_branch}" ]; then
+        sudo -E git checkout ${tests_branch}
+    fi
+    sudo -E git reset --hard origin/${tests_branch}
+    popd
+
+    source "${BATS_TEST_DIRNAME}/lib.sh"
+    source "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
+}
+
+# Note: clone_katacontainers_repo using go, so that needs to be installed first
+checkout_kata_containers_repo() {
+    source "${tests_repo_dir}/.ci/lib.sh"
+    echo "Creating repo: ${katacontainers_repo} and branch ${kata_default_branch} into ${katacontainers_repo_dir}..."
+    clone_katacontainers_repo
+    sudo -E chown -R ${USER}:${USER} "${katacontainers_repo_dir}"
+}
+
+build_and_install_kata_runtime() {
+    export DEFAULT_HYPERVISOR=${KATA_HYPERVISOR}
+    ${tests_repo_dir}/.ci/install_runtime.sh
+}
+
+configure() {
+    # configure kata to use rootfs, not initrd
+    sudo sed -i 's/^\(initrd =.*\)/# \1/g' ${RUNTIME_CONFIG_PATH}
+
+    enable_full_debug
+    enable_agent_console
+
+    # Switch image offload to true in kata config
+    switch_image_service_offload "on"
+
+    configure_cc_containerd
+    # From crictl v1.24.1 the default timoout leads to the pod creation failing, so update it
+    sudo crictl config --set timeout=10
+
+    # Verity checks aren't working locally, as we aren't re-genning the hash maybe? so remove it from the kernel parameters
+    remove_kernel_param "cc_rootfs_verity.scheme"
+}
+
+build_and_add_agent_to_rootfs() {
+    build_a_custom_kata_agent
+    add_custom_agent_to_rootfs
+}
+
+build_a_custom_kata_agent() {
+    # Install libseccomp for static linking
+    sudo -E PATH=$PATH GOPATH=$GOPATH ${katacontainers_repo_dir}/ci/install_libseccomp.sh /tmp/kata-libseccomp /tmp/kata-gperf
+    export LIBSECCOMP_LINK_TYPE=static
+    export LIBSECCOMP_LIB_PATH=/tmp/kata-libseccomp/lib
+
+    . "$HOME/.cargo/env"
+    pushd ${katacontainers_repo_dir}/src/agent
+    sudo -E PATH=$PATH make
+
+    ARCH=$(uname -m)
+    [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
+    [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
+
+    # Run a make install into the rootfs directory in order to create the kata-agent.service file which is required when we add to the rootfs
+    sudo -E PATH=$PATH make install DESTDIR="${ROOTFS_DIR}"
+    popd
+}
+
+create_a_local_rootfs() {
+    sudo rm -rf "${ROOTFS_DIR}"
+    pushd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder
+    export distro="ubuntu"
+    [[ -z "${USE_PODMAN:-}" ]] && use_docker="${use_docker:-1}"
+    sudo -E OS_VERSION="${OS_VERSION:-}" GOPATH=$GOPATH EXTRA_PKGS="vim iputils-ping net-tools" DEBUG="${DEBUG:-}" USE_DOCKER="${use_docker:-}" SKOPEO=${SKOPEO:-} AA_KBC=${AA_KBC:-} UMOCI=yes SECCOMP=yes ./rootfs.sh -r ${ROOTFS_DIR} ${distro}
+
+     # Install_rust.sh during rootfs.sh switches us to the main branch of the tests repo, so switch back now
+    pushd "${tests_repo_dir}"
+    sudo -E git checkout ${tests_branch}
+    popd
+    # During the ./rootfs.sh call the kata agent is built as root, so we need to update the permissions, so we can rebuild it
+    sudo chown -R ${USER}:${USER} "${katacontainers_repo_dir}/src/agent/"
+
+    popd
+}
+
+add_custom_agent_to_rootfs() {
+    pushd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder
+
+    ARCH=$(uname -m)
+    [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
+    [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
+
+    sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/usr/bin ${katacontainers_repo_dir}/src/agent/target/${ARCH}-unknown-linux-${LIBC}/release/kata-agent
+    sudo install -o root -g root -m 0440 ../../../src/agent/kata-agent.service ${ROOTFS_DIR}/usr/lib/systemd/system/
+    sudo install -o root -g root -m 0440 ../../../src/agent/kata-containers.target ${ROOTFS_DIR}/usr/lib/systemd/system/
+    popd
+}
+
+build_and_install_rootfs() {
+    build_rootfs_image
+    install_rootfs_image
+}
+
+build_rootfs_image() {
+    pushd ${katacontainers_repo_dir}/tools/osbuilder/image-builder
+    # Logic from install_kata_image.sh - if we aren't using podman (ie on a fedora like), then use docker
+    [[ -z "${USE_PODMAN:-}" ]] && use_docker="${use_docker:-1}"
+    sudo -E USE_DOCKER="${use_docker:-}" ./image_builder.sh ${ROOTFS_DIR}
+    popd
+}
+
+install_rootfs_image() {
+    pushd ${katacontainers_repo_dir}/tools/osbuilder/image-builder
+    local commit=$(git log --format=%h -1 HEAD)
+    local date=$(date +%Y-%m-%d-%T.%N%z)
+    local image="kata-containers-${date}-${commit}"
+    sudo install -o root -g root -m 0640 -D kata-containers.img "${PREFIX}/share/kata-containers/${image}"
+    (cd ${PREFIX}/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
+    echo "Built Rootfs from ${ROOTFS_DIR} to ${PREFIX}/share/kata-containers/${image}"
+    ls -al ${PREFIX}/share/kata-containers
+    popd
+}
+
+install_guest_kernel_image() {
+   ${tests_repo_dir}/.ci/install_kata_kernel.sh
+}
+
+build_qemu() {
+    ${tests_repo_dir}/.ci/install_virtiofsd.sh
+    ${tests_repo_dir}/.ci/install_qemu.sh
+}
+
+build_cloud_hypervisor() {
+    ${tests_repo_dir}/.ci/install_virtiofsd.sh
+    ${tests_repo_dir}/.ci/install_cloud_hypervisor.sh
+}
+
+check_kata_runtime() {
+    sudo kata-runtime check
+}
+
+k8s_pod_file="${HOME}/busybox-cc.yaml"
+init_kubernetes() {
+    # Check that kubeadm was installed and install it otherwise
+    if ! [ -x "$(command -v kubeadm)" ]; then
+        pushd "${tests_repo_dir}/.ci"
+        sudo -E PATH=$PATH -s install_kubernetes.sh
+        if [ "${CRI_CONTAINERD}" == "yes" ]; then
+            sudo -E PATH=$PATH -s "configure_containerd_for_kubernetes.sh"
+        fi
+        popd
+    fi
+
+    # If kubernetes init has previously run we need to clean it by removing the image and resetting k8s
+    local cid=$(sudo docker ps -a -q -f name=^/kata-registry$)
+    if [ -n "${cid}" ]; then
+        sudo docker stop ${cid} && sudo docker rm ${cid}
+    fi
+    local k8s_nodes=$(kubectl get nodes -o name 2>/dev/null || true)
+    if [ -n "${k8s_nodes}" ]; then
+        sudo kubeadm reset -f
+    fi
+
+    export CI="true" && sudo -E PATH=$PATH -s ${tests_repo_dir}/integration/kubernetes/init.sh
+    sudo chown ${USER}:$(id -g -n ${USER}) "$HOME/.kube/config"
+    cat << EOF > ${k8s_pod_file}
+apiVersion: v1
+kind: Pod
+metadata:
+  name: busybox-cc
+spec:
+  runtimeClassName: kata
+  containers:
+  - name: nginx
+    image: quay.io/kata-containers/confidential-containers:signed
+    imagePullPolicy: Always  
+EOF
+}
+
+call_kubernetes_create_cc_pod() {
+    kubernetes_create_cc_pod ${k8s_pod_file}
+}
+
+call_kubernetes_delete_cc_pod() {
+    pod_name=$(kubectl get pods -o jsonpath='{.items..metadata.name}')
+    kubernetes_delete_cc_pod $pod_name
+}
+
+call_kubernetes_create_ssh_demo_pod() {
+    setup_decryption_files_in_guest
+    kubernetes_create_ssh_demo_pod
+}
+
+call_connect_to_ssh_demo_pod() {
+    connect_to_ssh_demo_pod
+}
+
+call_kubernetes_delete_ssh_demo_pod() {
+    pod=$(kubectl get pods -o jsonpath='{.items..metadata.name}')
+    kubernetes_delete_ssh_demo_pod $pod
+}
+
+crictl_sandbox_name=kata-cc-busybox-sandbox
+call_crictl_create_cc_pod() {
+    # Update iptables to allow forwarding to the cni0 bridge avoiding issues caused by the docker0 bridge
+    sudo iptables -P FORWARD ACCEPT
+    
+    # get_pod_config in tests_common exports `pod_config` that points to the prepared pod config yaml 
+    get_pod_config
+
+    crictl_delete_cc_pod_if_exists "${crictl_sandbox_name}"
+    crictl_create_cc_pod "${pod_config}"
+    sudo crictl pods
+}
+
+call_crictl_create_cc_container() {
+    # Create container configuration yaml based on our test copy of busybox
+    # get_pod_config in tests_common exports `pod_config` that points to the prepared pod config yaml 
+    get_pod_config
+
+    local container_config="${FIXTURES_DIR}/${CONTAINER_CONFIG_FILE:-container-config.yaml}"
+    local pod_name=${crictl_sandbox_name}
+    crictl_create_cc_container ${pod_name} ${pod_config} ${container_config}
+    sudo crictl ps -a
+}
+
+crictl_delete_cc() {
+    crictl_delete_cc_pod ${crictl_sandbox_name}
+}
+
+test_kata_runtime() {
+    echo "Running ctr with the kata runtime..."
+    local test_image="quay.io/kata-containers/confidential-containers:signed"
+    if [ -z $(sudo ctr images ls -q name=="${test_image}") ]; then
+        sudo ctr image pull "${test_image}"
+    fi
+    sudo ctr run --runtime "io.containerd.kata.v2" --rm -t "${test_image}" test-kata uname -a
+}
+
+run_kata_and_capture_logs() {
+    echo "Clearing systemd journal..."
+    sudo systemctl stop systemd-journald
+    sudo rm -f /var/log/journal/*/* /run/log/journal/*/*
+    sudo systemctl start systemd-journald
+    test_kata_runtime
+    echo "Collecting logs..."
+    sudo journalctl -q -o cat -a -t kata-runtime > ${HOME}/kata-runtime.log
+    sudo journalctl -q -o cat -a -t kata > ${HOME}/shimv2.log
+    echo "Logs output to ${HOME}/kata-runtime.log and ${HOME}/shimv2.log"
+}
+
+get_ids() {
+    guest_cid=$(sudo ss -H --vsock | awk '{print $6}' | cut -d: -f1)
+    sandbox_id=$(ps -ef | grep containerd-shim-kata-v2 | egrep -o "id [^,][^,].* " | awk '{print $2}')
+}
+
+open_kata_shell() {
+    get_ids
+    sudo -E "PATH=$PATH" kata-runtime exec ${sandbox_id}
+}
+
+build_bundle_dir_if_necessary() {
+    bundle_dir="/tmp/bundle"
+    if [ ! -d "${bundle_dir}" ]; then
+        rootfs_dir="$bundle_dir/rootfs"
+        image="quay.io/kata-containers/confidential-containers:signed"
+        mkdir -p "$rootfs_dir" && (cd "$bundle_dir" && runc spec)
+        sudo docker export $(sudo docker create "$image") | tar -C "$rootfs_dir" -xvf -
+    fi
+    # There were errors in create container agent-ctl command due to /bin/ seemingly not being on the path, so hardcode it
+    sudo sed -i -e 's%^\(\t*\)"sh"$%\1"/bin/sh"%g' "${bundle_dir}/config.json"
+}
+
+build_agent_ctl() {
+    cd ${GOPATH}/src/${katacontainers_repo}/src/tools/agent-ctl/
+    if [ -e "${HOME}/.cargo/registry" ]; then
+        sudo chown -R ${USER}:${USER} "${HOME}/.cargo/registry"
+    fi
+    sudo -E PATH=$PATH -s  make
+    ARCH=$(uname -m)
+    [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
+    [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
+    cd "./target/${ARCH}-unknown-linux-${LIBC}/release/"
+}
+
+run_agent_ctl_command() {
+    get_ids
+    build_bundle_dir_if_necessary
+    command=$1
+    # If kata-agent-ctl pre-built in this directory, use it directly, otherwise build it first and switch to release
+    if [ ! -x kata-agent-ctl ]; then
+        build_agent_ctl
+    fi 
+     ./kata-agent-ctl -l debug connect --bundle-dir "${bundle_dir}" --server-address "vsock://${guest_cid}:1024" -c "${command}"
+}
+
+agent_pull_image() {
+    run_agent_ctl_command "PullImage image=${PULL_IMAGE} cid=${CONTAINER_ID} source_creds=${SOURCE_CREDS}"
+}
+
+agent_create_container() {
+    run_agent_ctl_command "CreateContainer cid=${CONTAINER_ID}"
+}
+
+shim_pull_image() {
+    get_ids
+    local ctr_shim_command="sudo ctr --namespace k8s.io shim --id ${sandbox_id} pull-image ${PULL_IMAGE} ${CONTAINER_ID}"
+    echo "Issuing command '${ctr_shim_command}'"
+    ${ctr_shim_command}
+}
+
+call_copy_signature_files_to_guest() {
+    # TODO #5173 - remove this once the kernel_params aren't ignored by the agent config
+    export DEBUG_CONSOLE="true"
+    
+    if [ "${SKOPEO:-}" = "yes" ]; then
+        add_kernel_params "agent.container_policy_file=/etc/containers/quay_verification/quay_policy.json"
+        setup_skopeo_signature_files_in_guest
+    else
+        # TODO #4888 - set config to specifically enable signature verification to be on in ImageClient
+        setup_offline_fs_kbc_signature_files_in_guest
+    fi
+}
+
+main() {
+    while getopts "dh" opt; do
+        case "$opt" in
+            d) 
+                export DEBUG="-d"
+                set -x
+                ;;
+            h) 
+                usage 0
+                ;;
+            \?)
+                echo "Invalid option: -$OPTARG" >&2
+                usage 1
+                ;;
+        esac
+    done
+
+    shift $((OPTIND - 1))
+
+    subcmd="${1:-}"
+
+    [ -z "${subcmd}" ] && usage 1
+
+    case "${subcmd}" in
+        all)
+            build_and_install_all
+            run_kata_and_capture_logs
+            ;;
+        build_and_install_all)
+            build_and_install_all
+            ;;
+        rebuild_and_install_kata)
+            rebuild_and_install_kata
+            ;;
+        initialize)
+            initialize
+            ;;
+        build_kata_runtime)
+            build_and_install_kata_runtime
+            ;;
+        configure)
+            configure
+            ;;
+        create_rootfs)
+            create_a_local_rootfs
+            ;;
+        build_and_add_agent_to_rootfs)
+            build_and_add_agent_to_rootfs
+            ;;
+        build_and_install_rootfs)
+            build_and_install_rootfs
+            ;;
+        install_guest_kernel)
+            install_guest_kernel_image
+            ;;
+        build_cloud_hypervisor)
+            build_cloud_hypervisor
+            ;;
+        build_qemu)
+            build_qemu
+            ;;
+        init_kubernetes)
+            init_kubernetes
+            ;;
+        crictl_create_cc_pod)
+            call_crictl_create_cc_pod
+            ;;
+        crictl_create_cc_container)
+            call_crictl_create_cc_container
+            ;;
+        crictl_delete_cc)
+            crictl_delete_cc
+            ;;
+        kubernetes_create_cc_pod)
+            call_kubernetes_create_cc_pod
+            ;;
+        kubernetes_delete_cc_pod)
+            call_kubernetes_delete_cc_pod
+            ;;
+        kubernetes_create_ssh_demo_pod)
+            call_kubernetes_create_ssh_demo_pod
+            ;;
+        connect_to_ssh_demo_pod)
+            call_connect_to_ssh_demo_pod
+            ;;
+        kubernetes_delete_ssh_demo_pod)
+            call_kubernetes_delete_ssh_demo_pod
+            ;;
+        test)
+            test_kata_runtime
+            ;;
+        test_capture_logs)
+            run_kata_and_capture_logs
+            ;;
+        open_kata_console)
+            open_kata_console
+            ;;
+        open_kata_shell)
+            open_kata_shell
+            ;;
+        agent_pull_image)
+            agent_pull_image
+            ;;
+        shim_pull_image)
+            shim_pull_image
+            ;;
+        agent_create_container)
+            agent_create_container
+            ;;
+        copy_signature_files_to_guest)
+            call_copy_signature_files_to_guest
+            ;;
+        *)
+            usage 1
+            ;;
+    esac
+}
+
+main $@

docs/how-to/data/confidential-agent-config.toml.in

@@ -0,0 +1,45 @@
+# Copyright (c) 2021 IBM Corp.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+aa_kbc_params = "$AA_KBC_PARAMS"
+https_proxy = "$HTTPS_PROXY"
+[endpoints]
+allowed = [
+"AddARPNeighborsRequest",
+"AddSwapRequest",
+"CloseStdinRequest",
+"CopyFileRequest",
+"CreateContainerRequest",
+"CreateSandboxRequest",
+"DestroySandboxRequest",
+#"ExecProcessRequest",
+"GetMetricsRequest",
+"GetOOMEventRequest",
+"GuestDetailsRequest",
+"ListInterfacesRequest",
+"ListRoutesRequest",
+"MemHotplugByProbeRequest",
+"OnlineCPUMemRequest",
+"PauseContainerRequest",
+"PullImageRequest",
+"ReadStreamRequest",
+"RemoveContainerRequest",
+#"ReseedRandomDevRequest",
+"ResizeVolumeRequest",
+"ResumeContainerRequest",
+"SetGuestDateTimeRequest",
+"SignalProcessRequest",
+"StartContainerRequest",
+"StartTracingRequest",
+"StatsContainerRequest",
+"StopTracingRequest",
+"TtyWinResizeRequest",
+"UpdateContainerRequest",
+"UpdateInterfaceRequest",
+"UpdateRoutesRequest",
+"VolumeStatsRequest",
+"WaitProcessRequest",
+"WriteStreamRequest"
+]

docs/how-to/how-to-build-and-test-ccv0.md

@@ -0,0 +1,475 @@
+# How to build, run and test Kata CCv0
+
+## Introduction and Background
+
+In order to try and make building (locally) and demoing the Kata Containers `CCv0` code base as simple as possible I've
+shared a script [`ccv0.sh`](./ccv0.sh). This script was originally my attempt to automate the steps of the 
+[Developer Guide](https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md) so that I could do
+different sections of them repeatedly and reliably as I was playing around with make changes to different parts of the 
+Kata code base. I then tried to weave in some of the [`tests/.ci`](https://github.com/kata-containers/tests/tree/main/.ci) 
+scripts in order to have less duplicated code.
+As we're progress on the confidential containers journey I hope to add more features to demonstrate the functionality
+we have working.
+
+*Disclaimer: This script has mostly just been used and tested by me ([@stevenhorsman](https://github.com/stevenhorsman)),*
+*so there might be issues with it. I'm happy to try and help solve these if possible, but this shouldn't be considered a*
+*fully supported process by the Kata Containers community.*
+
+### Basic script set-up and optional environment variables
+
+In order to build, configure and demo the CCv0 functionality, these are the set-up steps I take:
+- Provision a new VM
+    - *I choose a Ubuntu 20.04 8GB VM for this as I had one available. There are some dependences on apt-get installed*
+    *packages, so these will need re-working to be compatible with other platforms.*
+- Copy the script over to your VM *(I put it in the home directory)* and ensure it has execute permission by running 
+```bash
+$ chmod u+x ccv0.sh
+```
+- Optionally set up some environment variables
+    - By default the script checks out the `CCv0` branches of the `kata-containers/kata-containers` and 
+      `kata-containers/tests` repositories, but it is designed to be used to test of personal forks and branches as well. 
+      If you want to build and run these you can export the `katacontainers_repo`, `katacontainers_branch`, `tests_repo`
+      and `tests_branch` variables e.g.
+      ```bash
+      $ export katacontainers_repo=github.com/stevenhorsman/kata-containers
+      $ export katacontainers_branch=stevenh/agent-pull-image-endpoint
+      $ export tests_repo=github.com/stevenhorsman/tests
+      $ export tests_branch=stevenh/add-ccv0-changes-to-build
+      ```
+      before running the script.
+    - By default the build and configuration are using `QEMU` as the hypervisor. In order to use `Cloud Hypervisor` instead
+      set:
+      ```
+      $ export KATA_HYPERVISOR="cloud-hypervisor"
+      ```
+      before running the build.
+
+- At this point you can provision a Kata confidential containers pod and container with either
+  [`crictl`](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image),
+  or [Kubernetes](#using-kubernetes-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image)
+  and then test and use it. 
+
+### Using crictl for end-to-end provisioning of a Kata confidential containers pod with an unencrypted image
+
+- Run the full build process with Kubernetes turned off, so its configuration doesn't interfere with `crictl` using:
+  ```bash
+  $ export KUBERNETES="no"
+  $ export KATA_HYPERVISOR="qemu"
+  $ ~/ccv0.sh -d build_and_install_all
+  ```
+    > **Note**: Much of this script has to be run as `sudo`, so you are likely to get prompted for your password.
+    - *I run this script sourced just so that the required installed components are accessible on the `PATH` to the rest*
+      *of the process without having to reload the session.*
+    - The steps that `build_and_install_all` takes is:
+        - Checkout the git repos for the `tests` and `kata-containers` repos as specified by the environment variables
+        (default to `CCv0` branches if they are not supplied)
+        - Use the `tests/.ci` scripts to install the build dependencies
+        - Build and install the Kata runtime
+        - Configure Kata to use containerd and for debug and confidential containers features to be enabled (including
+          enabling console access to the Kata guest shell, which should only be done in development)
+        - Create, build and install a rootfs for the Kata hypervisor to use. For 'CCv0' this is currently based on Ubuntu
+        20.04.
+        - Build the Kata guest kernel
+        - Install the hypervisor (in order to select which hypervisor will be used, the `KATA_HYPERVISOR` environment
+        variable can be used to select between `qemu` or `cloud-hypervisor`)
+    > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
+    during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
+    this, login into Docker Hub and pull the images used with:
+  >  ```bash
+  >  $ sudo docker login
+  >  $ sudo docker pull ubuntu
+  >  ```
+  >  then re-run the command.
+    - The first time this runs it may take a while, but subsequent runs will be quicker as more things are already
+      installed and they can be further cut down by not running all the above steps 
+      [see "Additional script usage" below](#additional-script-usage)
+   
+- Create a new Kata sandbox pod using `crictl` with:
+  ```bash
+  $ ~/ccv0.sh crictl_create_cc_pod
+  ```
+    - This creates a pod configuration file, creates the pod from this using 
+    `sudo crictl runp -r kata ~/pod-config.yaml` and runs `sudo crictl pods` to show the pod
+- Create a new Kata confidential container with:
+  ```bash
+  $ ~/ccv0.sh crictl_create_cc_container
+  ```
+    - This creates a container (based on `busybox:1.33.1`) in the Kata cc sandbox and prints a list of containers.
+      This will have been created based on an image pulled in the Kata pod sandbox/guest, not on the host machine.
+
+As this point you should have a `crictl` pod and container that is using the Kata confidential containers runtime.
+You can [validate that the container image was pulled on the guest](#validate-that-the-container-image-was-pulled-on-the-guest)
+or [using the Kata pod sandbox for testing with `agent-ctl` or `ctr shim`](#using-a-kata-pod-sandbox-for-testing-with-agent-ctl-or-ctr-shim)
+
+#### Clean up the `crictl` pod sandbox and container
+- When the testing is complete you can delete the container and pod by running:
+  ```bash
+  $ ~/ccv0.sh crictl_delete_cc
+  ```
+### Using Kubernetes for end-to-end provisioning of a Kata confidential containers pod with an unencrypted image
+
+- Run the full build process with the Kubernetes environment variable set to `"yes"`, so the Kubernetes cluster is
+  configured and created using the VM
+  as a single node cluster: 
+  ```bash
+  $ export KUBERNETES="yes"
+  $ ~/ccv0.sh build_and_install_all
+  ```
+    > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
+    during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
+    this, login into Docker Hub and pull the images used with:
+  >  ```bash
+  >  $ sudo docker login
+  >  $ sudo docker pull registry:2
+  >  $ sudo docker pull ubuntu:20.04
+  >  ```
+  >  then re-run the command.
+- Check that your Kubernetes cluster has been correctly set-up by running : 
+  ```bash
+  $ kubectl get nodes
+  ```
+  and checking that you see a single node e.g.
+  ```text
+  NAME                             STATUS   ROLES                  AGE   VERSION
+  stevenh-ccv0-k8s1.fyre.ibm.com   Ready    control-plane,master   43s   v1.22.0
+  ```
+- Create a Kata confidential containers pod by running:
+  ```bash
+  $ ~/ccv0.sh kubernetes_create_cc_pod
+  ```
+- Wait a few seconds for pod to start then check that the pod's status is `Running` with 
+  ```bash
+  $ kubectl get pods
+  ```
+  which should show something like:
+  ```text
+  NAME         READY   STATUS    RESTARTS   AGE
+  busybox-cc   1/1     Running   0          54s
+  ```
+
+- As this point you should have a Kubernetes pod and container running, that is using the Kata 
+confidential containers runtime.
+You can [validate that the container image was pulled on the guest](#validate-that-the-container-image-was-pulled-on-the-guest)
+or [using the Kata pod sandbox for testing with `agent-ctl` or `ctr shim`](#using-a-kata-pod-sandbox-for-testing-with-agent-ctl-or-ctr-shim)
+
+#### Clean up the Kubernetes pod sandbox and container
+- When the testing is complete you can delete the container and pod by running:
+  ```bash
+  $ ~/ccv0.sh kubernetes_delete_cc_pod
+  ```
+
+### Validate that the container image was pulled on the guest
+
+There are a couple of ways we can check that the container pull image action was offloaded to the guest, by checking
+the guest's file system for the unpacked bundle and checking the host's directories to ensure it wasn't also pulled
+there.
+- To check the guest's file system:
+    - Open a shell into the Kata guest with:
+      ```bash
+      $ ~/ccv0.sh open_kata_shell
+      ```
+    - List the files in the directory that the container image bundle should have been unpacked to with:
+      ```bash
+      $ ls -ltr /run/kata-containers/confidential-containers_signed/
+      ```
+    - This should give something like
+        ```
+        total 72
+        -rw-r--r--  1 root root  2977 Jan 20 10:03 config.json
+        drwxr-xr-x 12 root root   240 Jan 20 10:03 rootfs
+        ```
+        which shows how the image has been pulled and then unbundled on the guest.
+    - Leave the Kata guest shell by running:
+      ```bash
+      $ exit
+      ```
+- To verify that the image wasn't pulled on the host system we can look at the shared sandbox on the host and we
+  should only see a single bundle for the pause container as the `busybox` based container image should have been
+  pulled on the guest:
+    - Find all the `rootfs` directories under in the pod's shared directory with:
+      ```bash
+      $ pod_id=$(ps -ef | grep containerd-shim-kata-v2 | egrep -o "id [^,][^,].* " | awk '{print $2}')
+      $ sudo find /run/kata-containers/shared/sandboxes/${pod_id}/shared -name rootfs
+      ```
+      which should only show a single `rootfs` directory if the container image was pulled on the guest, not the host
+    - Looking that `rootfs` directory with
+      ```bash
+      $ sudo ls -ltr $(sudo find /run/kata-containers/shared/sandboxes/${pod_id}/shared -name rootfs)
+      ```
+      shows something similar to
+      ```
+      total 668
+      -rwxr-xr-x 1 root root 682696 Aug 25 13:58 pause
+      drwxr-xr-x 2 root root      6 Jan 20 02:01 proc
+      drwxr-xr-x 2 root root      6 Jan 20 02:01 dev
+      drwxr-xr-x 2 root root      6 Jan 20 02:01 sys
+      drwxr-xr-x 2 root root     25 Jan 20 02:01 etc
+      ```
+      which is clearly the pause container indicating that the `busybox` based container image is not exposed to the host.
+
+### Using a Kata pod sandbox for testing with `agent-ctl` or `ctr shim`
+
+Once you have a kata pod sandbox created as described above, either using 
+[`crictl`](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image), or [Kubernetes](#using-kubernetes-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image)
+, you can use this to test specific components of the Kata confidential
+containers architecture. This can be useful for development and debugging to isolate and test features
+that aren't broadly supported end-to-end. Here are some examples:
+
+- In the first terminal run the pull image on guest command against the Kata agent, via the shim (`containerd-shim-kata-v2`).
+This can be achieved using the [containerd](https://github.com/containerd/containerd) CLI tool, `ctr`, which can be used to
+interact with the shim directly. The command takes the form
+`ctr --namespace k8s.io shim --id <sandbox-id> pull-image <image> <new-container-id>` and can been run directly, or through
+the `ccv0.sh` script to automatically fill in the variables:
+  - Optionally, set up some environment variables to set the image and credentials used:
+    - By default the shim pull image test in `ccv0.sh` will use the `busybox:1.33.1` based test image
+     `quay.io/kata-containers/confidential-containers:signed` which requires no authentication. To use a different
+      image, set the `PULL_IMAGE` environment variable e.g. 
+      ```bash
+      $ export PULL_IMAGE="docker.io/library/busybox:latest"
+      ```
+      Currently the containerd shim pull image
+      code doesn't support using a container registry that requires authentication, so if this is required, see the 
+      below steps to run the pull image command against the agent directly.
+  - Run the pull image agent endpoint with:
+    ```bash
+    $ ~/ccv0.sh shim_pull_image
+    ```
+    which we print the `ctr shim` command for reference
+- Alternatively you can issue the command directly to the `kata-agent` pull image endpoint, which also supports
+  credentials in order to pull from an authenticated registry:
+    - Optionally set up some environment variables to set the image and credentials used:
+        - Set the `PULL_IMAGE` environment variable e.g. `export PULL_IMAGE="docker.io/library/busybox:latest"`
+          if a specific container image is required.
+        - If the container registry for the image requires authentication then this can be set with an environment
+          variable `SOURCE_CREDS`. For example to use Docker Hub (`docker.io`) as an authenticated user first run 
+          `export SOURCE_CREDS="<dockerhub username>:<dockerhub api key>"`
+            > **Note**: the credentials support on the agent request is a tactical solution for the short-term
+              proof of concept to allow more images to be pulled and tested. Once we have support for getting
+              keys into the Kata guest image using the attestation-agent and/or KBS I'd expect container registry
+              credentials to be looked up using that mechanism.
+    - Run the pull image agent endpoint with
+      ```bash
+      $ ~/ccv0.sh agent_pull_image
+      ```
+      and you should see output which includes `Command PullImage (1 of 1) returned (Ok(()), false)` to indicate
+      that the `PullImage` request was successful e.g.
+      ```
+      Finished release [optimized] target(s) in 0.21s
+      {"msg":"announce","level":"INFO","ts":"2021-09-15T08:40:14.189360410-07:00","subsystem":"rpc","name":"kata-agent-ctl","pid":"830920","version":"0.1.0","source":"kata-agent-ctl","config":"Config { server_address: \"vsock://1970354082:1024\", bundle_dir: \"/tmp/bundle\", timeout_nano: 0, interactive: false, ignore_errors: false }"}
+      {"msg":"client setup complete","level":"INFO","ts":"2021-09-15T08:40:14.193639057-07:00","pid":"830920","source":"kata-agent-ctl","name":"kata-agent-ctl","subsystem":"rpc","version":"0.1.0","server-address":"vsock://1970354082:1024"}
+      {"msg":"Run command PullImage (1 of 1)","level":"INFO","ts":"2021-09-15T08:40:14.196643765-07:00","pid":"830920","source":"kata-agent-ctl","subsystem":"rpc","name":"kata-agent-ctl","version":"0.1.0"}
+      {"msg":"response received","level":"INFO","ts":"2021-09-15T08:40:43.828200633-07:00","source":"kata-agent-ctl","name":"kata-agent-ctl","subsystem":"rpc","version":"0.1.0","pid":"830920","response":""}
+      {"msg":"Command PullImage (1 of 1) returned (Ok(()), false)","level":"INFO","ts":"2021-09-15T08:40:43.828261708-07:00","subsystem":"rpc","pid":"830920","source":"kata-agent-ctl","version":"0.1.0","name":"kata-agent-ctl"}
+      ```
+      > **Note**: The first time that `~/ccv0.sh agent_pull_image` is run, the `agent-ctl` tool will be built
+      which may take a few minutes.
+- To validate that the image pull was successful, you can open a shell into the Kata guest with:
+  ```bash
+  $ ~/ccv0.sh open_kata_shell
+  ```
+- Check the `/run/kata-containers/` directory to verify that the container image bundle has been created in a directory
+  named either `01234556789` (for the container id), or the container image name, e.g.
+  ```bash
+  $ ls -ltr /run/kata-containers/confidential-containers_signed/
+  ```
+  which should show something like
+  ```
+  total 72
+  drwxr-xr-x 10 root root   200 Jan  1  1970 rootfs
+  -rw-r--r--  1 root root  2977 Jan 20 16:45 config.json
+  ```
+- Leave the Kata shell by running:
+  ```bash
+  $ exit
+  ```
+
+## Verifying signed images
+
+For this sample demo, we use local attestation to pass through the required
+configuration to do container image signature verification. Due to this, the ability to verify images is limited
+to a pre-created selection of test images in our test
+repository [`quay.io/kata-containers/confidential-containers`](https://quay.io/repository/kata-containers/confidential-containers?tab=tags).
+For pulling images not in this test repository (called an *unprotected* registry below), we fall back to the behaviour
+of not enforcing signatures. More documentation on how to customise this to match your own containers through local,
+or remote attestation will be available in future.
+
+In our test repository there are three tagged images:
+
+| Test Image | Base Image used | Signature status | GPG key status |
+| --- | --- | --- | --- |
+| `quay.io/kata-containers/confidential-containers:signed` | `busybox:1.33.1` | [signature](https://github.com/kata-containers/tests/tree/CCv0/integration/confidential/fixtures/quay_verification/x86_64/signatures.tar) embedded in kata rootfs |  [public key](https://github.com/kata-containers/tests/tree/CCv0/integration/confidential/fixtures/quay_verification/x86_64/public.gpg) embedded in kata rootfs |
+| `quay.io/kata-containers/confidential-containers:unsigned` | `busybox:1.33.1` | not signed | not signed |
+| `quay.io/kata-containers/confidential-containers:other_signed` | `nginx:1.21.3` | [signature](https://github.com/kata-containers/tests/tree/CCv0/integration/confidential/fixtures/quay_verification/x86_64/signatures.tar) embedded in kata rootfs | GPG key not kept |
+
+Using a standard unsigned `busybox` image that can be pulled from another, *unprotected*, `quay.io` repository we can
+test a few scenarios.
+
+In this sample, with local attestation, we pass in the the public GPG key and signature files, and the [`offline_fs_kbc`
+configuration](https://github.com/confidential-containers/attestation-agent/blob/main/src/kbc_modules/offline_fs_kbc/README.md)
+into the guest image which specifies that any container image from `quay.io/kata-containers`
+must be signed with the embedded GPG key and the agent configuration needs updating to enable this. 
+With this policy set a few tests of image verification can be done to test different scenarios by attempting
+to create containers from these images using `crictl`:
+
+- If you don't already have the Kata Containers CC code built and configured for `crictl`, then follow the 
+[instructions above](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image)
+up to the `~/ccv0.sh crictl_create_cc_pod` command.
+
+- In order to enable the guest image, you will need to setup the required configuration, policy and signature files
+needed by running 
+`~/ccv0.sh copy_signature_files_to_guest` and then run `~/ccv0.sh crictl_create_cc_pod` which will delete and recreate 
+your pod - adding in the new files.
+ 
+- To test the fallback behaviour works using an unsigned image from an *unprotected* registry we can pull the `busybox`
+image by running:
+  ```bash
+  $ export CONTAINER_CONFIG_FILE=container-config_unsigned-unprotected.yaml
+  $ ~/ccv0.sh crictl_create_cc_container
+  ```
+  - This finishes showing the running container e.g.
+  ```text
+  CONTAINER           IMAGE                               CREATED                  STATE               NAME                        ATTEMPT             POD ID
+  98c70fefe997a       quay.io/prometheus/busybox:latest   Less than a second ago   Running             prometheus-busybox-signed   0                   70119e0539238
+  ```
+- To test that an unsigned image from our *protected* test container registry is rejected we can run:
+  ```bash
+  $ export CONTAINER_CONFIG_FILE=container-config_unsigned-protected.yaml
+  $ ~/ccv0.sh crictl_create_cc_container
+  ```
+  - This correctly results in an error message from `crictl`:
+  `PullImage from image service failed" err="rpc error: code = Internal desc = Security validate failed: Validate image failed: The signatures do not satisfied! Reject reason: [Match reference failed.]" image="quay.io/kata-containers/confidential-containers:unsigned"`
+- To test that the signed image our *protected* test container registry is accepted we can run:
+  ```bash
+  $ export CONTAINER_CONFIG_FILE=container-config.yaml
+  $ ~/ccv0.sh crictl_create_cc_container
+  ```
+  - This finishes by showing a new `kata-cc-busybox-signed` running container e.g.
+  ```text
+  CONTAINER           IMAGE                                                    CREATED                  STATE               NAME                        ATTEMPT             POD ID
+  b4d85c2132ed9       quay.io/kata-containers/confidential-containers:signed   Less than a second ago   Running             kata-cc-busybox-signed      0                   70119e0539238
+  ...
+  ```
+- Finally to check the image with a valid signature, but invalid GPG key (the real trusted piece of information we really
+want to protect with the attestation agent in future) fails we can run: 
+  ```bash
+  $ export CONTAINER_CONFIG_FILE=container-config_signed-protected-other.yaml
+  $ ~/ccv0.sh crictl_create_cc_container
+  ```
+  - Again this results in an error message from `crictl`:
+  `"PullImage from image service failed" err="rpc error: code = Internal desc = Security validate failed: Validate image failed: The signatures do not satisfied! Reject reason: [signature verify failed! There is no pubkey can verify the signature!]" image="quay.io/kata-containers/confidential-containers:other_signed"`
+
+### Using Kubernetes to create a Kata confidential containers pod from the encrypted ssh demo sample image
+
+The [ssh-demo](https://github.com/confidential-containers/documentation/tree/main/demos/ssh-demo) explains how to
+demonstrate creating a Kata confidential containers pod from an encrypted image with the runtime created by the
+[confidential-containers operator](https://github.com/confidential-containers/documentation/blob/main/demos/operator-demo).
+To be fully confidential, this should be run on a Trusted Execution Environment, but it can be tested on generic
+hardware as well.
+
+If you wish to build the Kata confidential containers runtime to do this yourself, then you can using the following
+steps:
+
+- Run the full build process with the Kubernetes environment variable set to `"yes"`, so the Kubernetes cluster is
+  configured and created using the VM as a single node cluster and with `AA_KBC` set to `offline_fs_kbc`. 
+  ```bash
+  $ export KUBERNETES="yes"
+  $ export AA_KBC=offline_fs_kbc
+  $ ~/ccv0.sh build_and_install_all
+  ```
+    - The `AA_KBC=offline_fs_kbc` mode will ensure that, when creating the rootfs of the Kata guest, the 
+      [attestation-agent](https://github.com/confidential-containers/attestation-agent) will be added along with the
+      [sample offline KBC](https://github.com/confidential-containers/documentation/blob/main/demos/ssh-demo/aa-offline_fs_kbc-keys.json)
+      and an agent configuration file
+    > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
+    during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
+    this, login into Docker Hub and pull the images used with:
+  >  ```bash
+  >  $ sudo docker login
+  >  $ sudo docker pull registry:2
+  >  $ sudo docker pull ubuntu:20.04
+  >  ```
+  >  then re-run the command.
+- Check that your Kubernetes cluster has been correctly set-up by running : 
+  ```bash
+  $ kubectl get nodes
+  ```
+  and checking that you see a single node e.g.
+  ```text
+  NAME                             STATUS   ROLES                  AGE   VERSION
+  stevenh-ccv0-k8s1.fyre.ibm.com   Ready    control-plane,master   43s   v1.22.0
+  ```
+- Create a sample Kata confidential containers ssh pod by running:
+  ```bash
+  $ ~/ccv0.sh kubernetes_create_ssh_demo_pod
+  ```
+- As this point you should have a Kubernetes pod running the Kata confidential containers runtime that has pulled
+the [sample image](https://hub.docker.com/r/katadocker/ccv0-ssh) which was encrypted by the key file that we included
+in the rootfs.
+During the pod deployment the image was pulled and then decrypted using the key file, on the Kata guest image, without
+it ever being available to the host.
+
+- To validate that the container is working you, can connect to the image via SSH by running:
+  ```bash
+  $ ~/ccv0.sh connect_to_ssh_demo_pod
+  ```
+  - During this connection the host key fingerprint is shown and should match:
+    `ED25519 key fingerprint is SHA256:wK7uOpqpYQczcgV00fGCh+X97sJL3f6G1Ku4rvlwtR0.`
+  - After you are finished connecting then run:
+    ```bash
+    $ exit
+    ```
+  
+- To delete the sample SSH demo pod run:
+  ```bash
+  $ ~/ccv0.sh kubernetes_delete_ssh_demo_pod
+  ```
+
+## Additional script usage
+
+As well as being able to use the script as above to build all of `kata-containers` from scratch it can be used to just
+re-build bits of it by running the script with different parameters. For example after the first build you will often
+not need to re-install the dependencies, the hypervisor or the Guest kernel, but just test code changes made to the
+runtime and agent. This can be done by running `~/ccv0.sh rebuild_and_install_kata`. (*Note this does a hard checkout*
+*from git, so if your changes are only made locally it is better to do the individual steps e.g.* 
+`~/ccv0.sh build_kata_runtime && ~/ccv0.sh build_and_add_agent_to_rootfs && ~/ccv0.sh build_and_install_rootfs`).
+There are commands for a lot of steps in building, setting up and testing and the full list can be seen by running
+`~/ccv0.sh help`:
+```
+$ ~/ccv0.sh help
+Overview:
+    Build and test kata containers from source
+    Optionally set kata-containers and tests repo and branch as exported variables before running
+    e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && ~/ccv0.sh build_and_install_all
+Usage:
+    ccv0.sh [options] <command>
+Commands:
+- help:                         Display this help
+- all:                          Build and install everything, test kata with containerd and capture the logs
+- build_and_install_all:        Build and install everything
+- initialize:                   Install dependencies and check out kata-containers source
+- rebuild_and_install_kata:     Rebuild the kata runtime and agent and build and install the image
+- build_kata_runtime:           Build and install the kata runtime
+- configure:                    Configure Kata to use rootfs and enable debug
+- create_rootfs:                Create a local rootfs
+- build_and_add_agent_to_rootfs:Builds the kata-agent and adds it to the rootfs
+- build_and_install_rootfs:     Builds and installs the rootfs image
+- install_guest_kernel:         Setup, build and install the guest kernel
+- build_cloud_hypervisor        Checkout, patch, build and install Cloud Hypervisor
+- build_qemu:                   Checkout, patch, build and install QEMU
+- init_kubernetes:              initialize a Kubernetes cluster on this system
+- crictl_create_cc_pod          Use crictl to create a new kata cc pod
+- crictl_create_cc_container    Use crictl to create a new busybox container in the kata cc pod
+- crictl_delete_cc              Use crictl to delete the kata cc pod sandbox and container in it
+- kubernetes_create_cc_pod:     Create a Kata CC runtime busybox-based pod in Kubernetes
+- kubernetes_delete_cc_pod:     Delete the Kata CC runtime busybox-based pod in Kubernetes
+- open_kata_shell:              Open a shell into the kata runtime
+- agent_pull_image:             Run PullImage command against the agent with agent-ctl
+- shim_pull_image:              Run PullImage command against the shim with ctr
+- agent_create_container:       Run CreateContainer command against the agent with agent-ctl
+- test:                         Test using kata with containerd
+- test_capture_logs:            Test using kata with containerd and capture the logs in the user's home directory
+
+Options:
+    -d: Enable debug
+    -h: Display this help
+```

docs/how-to/how-to-generate-a-kata-containers-payload-for-the-confidential-containers-operator.md

@@ -0,0 +1,44 @@
+# Generating a Kata Containers payload for the Confidential Containers Operator
+
+[Confidential Containers
+Operator](https://github.com/confidential-containers/operator) consumes a Kata
+Containers payload, generated from the `CCv0` branch, and here one can find all
+the necessary info on how to build such a payload.
+
+## Requirements
+
+* `make` installed in the machine
+* Docker installed in the machine
+* `sudo` access to the machine
+
+## Process
+
+* Clone [Kata Containers](https://github.com/kata-containers/kata-containers)
+  ```sh
+  git clone --branch CCv0 https://github.com/kata-containers/kata-containers
+  ```
+  * In case you've already cloned the repo, make sure to switch to the `CCv0` branch
+    ```sh
+    git checkout CCv0
+    ```
+  * Ensure your tree is clean and in sync with upstream `CCv0`
+    ```sh
+    git clean -xfd
+    git reset --hard <upstream>/CCv0
+    ```
+* Make sure you're authenticated to `quay.io`
+    ```sh
+    sudo docker login quay.io
+    ```
+* From the top repo directory, run:
+  ```sh
+  sudo make cc-payload
+  ```
+* Make sure the image was upload to the [Confidential Containers
+  runtime-payload
+registry](https://quay.io/repository/confidential-containers/runtime-payload?tab=tags)
+
+## Notes
+
+Make sure to run it on a machine that's not the one you're hacking on, prepare a
+cup of tea, and get back to it an hour later (at least).

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -94,6 +94,16 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.enable_guest_swap` | `boolean` | enable swap in the guest |
 | `io.katacontainers.config.hypervisor.use_legacy_serial` | `boolean` | uses legacy serial device for guest's console (QEMU) |
 
+## Confidential Computing Options
+| Key | Value Type | Comments |
+|-------| ----- | ----- |
+| `io.katacontainers.config.pre_attestation.enabled"` | `bool` |
+determines if SEV/-ES attestation is enabled |
+| `io.katacontainers.config.pre_attestation.uri"` | `string` |
+specify the location of the attestation server |
+| `io.katacontainers.config.sev.policy"` | `uint32` |
+specify the SEV guest policy |
+
 ## Container Options
 | Key | Value Type | Comments |
 |-------| ----- | ----- |

docs/how-to/how-to-setup-swap-devices-in-guest-kernel.md

@@ -27,6 +27,8 @@ $ image="quay.io/prometheus/busybox:latest"
 $ cat << EOF > "${pod_yaml}"
 metadata:
   name: busybox-sandbox1
+  uid: $(uuidgen)
+  namespace: default
 EOF
 $ cat << EOF > "${container_yaml}"
 metadata:

docs/how-to/how-to-use-virtio-fs-nydus-with-kata.md

@@ -32,6 +32,7 @@ The `nydus-sandbox.yaml` looks like below:
 metadata:
   attempt: 1
   name: nydus-sandbox
+  uid: nydus-uid
   namespace: default
 log_directory: /tmp
 linux:

docs/how-to/how-to-use-virtio-mem-with-kata.md

@@ -42,6 +42,8 @@ $ image="quay.io/prometheus/busybox:latest"
 $ cat << EOF > "${pod_yaml}"
 metadata:
   name: busybox-sandbox1
+  uid: $(uuidgen)
+  namespace: default
 EOF
 $ cat << EOF > "${container_yaml}"
 metadata:

src/agent/Cargo.toml

@@ -23,11 +23,12 @@ regex = "1.5.6"
 serial_test = "0.5.1"
 kata-sys-util = { path = "../libs/kata-sys-util" }
 kata-types = { path = "../libs/kata-types" }
+url = "2.2.2"
 
 # Async helpers
 async-trait = "0.1.42"
 async-recursion = "0.3.2"
-futures = "0.3.17"
+futures = "0.3.28"
 
 # Async runtime
 tokio = { version = "1.28.1", features = ["full"] }
@@ -66,12 +67,22 @@ serde = { version = "1.0.129", features = ["derive"] }
 toml = "0.5.8"
 clap = { version = "3.0.1", features = ["derive"] }
 
+# "vendored" feature for openssl is required by musl build
+openssl = { version = "0.10.38", features = ["vendored"] }
+
+# Image pull/decrypt
+image-rs = { git = "https://github.com/confidential-containers/guest-components", tag = "v0.7.0", default-features = false, features = ["kata-cc-native-tls"] }
+
+[patch.crates-io]
+oci-distribution = { git = "https://github.com/krustlet/oci-distribution.git", rev = "f44124c" }
+
 [dev-dependencies]
 tempfile = "3.1.0"
 test-utils = { path = "../libs/test-utils" }
 which = "4.3.0"
 
 [workspace]
+resolver = "2"
 members = [
     "rustjail",
 ]

src/agent/rustjail/src/cgroups/fs/mod.rs

@@ -541,8 +541,11 @@ fn linux_device_to_cgroup_device(d: &LinuxDevice) -> Option<DeviceResource> {
 }
 
 fn linux_device_group_to_cgroup_device(d: &LinuxDeviceCgroup) -> Option<DeviceResource> {
-    let dev_type = match DeviceType::from_char(d.r#type.chars().next()) {
-        Some(t) => t,
+    let dev_type = match &d.r#type {
+        Some(t_s) => match DeviceType::from_char(t_s.chars().next()) {
+            Some(t_c) => t_c,
+            None => return None,
+        },
         None => return None,
     };
 
@@ -599,7 +602,7 @@ lazy_static! {
             // all mknod to all char devices
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: "c".to_string(),
+                r#type: Some("c".to_string()),
                 major: Some(WILDCARD),
                 minor: Some(WILDCARD),
                 access: "m".to_string(),
@@ -608,7 +611,7 @@ lazy_static! {
             // all mknod to all block devices
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: "b".to_string(),
+                r#type: Some("b".to_string()),
                 major: Some(WILDCARD),
                 minor: Some(WILDCARD),
                 access: "m".to_string(),
@@ -617,7 +620,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/console
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: "c".to_string(),
+                r#type: Some("c".to_string()),
                 major: Some(5),
                 minor: Some(1),
                 access: "rwm".to_string(),
@@ -626,7 +629,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/pts/<N>
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: "c".to_string(),
+                r#type: Some("c".to_string()),
                 major: Some(136),
                 minor: Some(WILDCARD),
                 access: "rwm".to_string(),
@@ -635,7 +638,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/ptmx
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: "c".to_string(),
+                r#type: Some("c".to_string()),
                 major: Some(5),
                 minor: Some(2),
                 access: "rwm".to_string(),
@@ -644,7 +647,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/net/tun
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: "c".to_string(),
+                r#type: Some("c".to_string()),
                 major: Some(10),
                 minor: Some(200),
                 access: "rwm".to_string(),

src/agent/rustjail/src/lib.rs

@@ -241,6 +241,12 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
     let devices = {
         let mut d = Vec::new();
         for dev in res.Devices.iter() {
+            let dev_type = if dev.Type.is_empty() {
+                None
+            } else {
+                Some(dev.Type.clone())
+            };
+
             let major = if dev.Major == -1 {
                 None
             } else {
@@ -254,7 +260,7 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
             };
             d.push(oci::LinuxDeviceCgroup {
                 allow: dev.Allow,
-                r#type: dev.Type.clone(),
+                r#type: dev_type,
                 major,
                 minor,
                 access: dev.Access.clone(),

src/agent/src/config.rs

@@ -11,6 +11,7 @@ use std::fs;
 use std::str::FromStr;
 use std::time;
 use tracing::instrument;
+use url::Url;
 
 use kata_types::config::default::DEFAULT_AGENT_VSOCK_PORT;
 
@@ -25,6 +26,14 @@ const LOG_VPORT_OPTION: &str = "agent.log_vport";
 const CONTAINER_PIPE_SIZE_OPTION: &str = "agent.container_pipe_size";
 const UNIFIED_CGROUP_HIERARCHY_OPTION: &str = "agent.unified_cgroup_hierarchy";
 const CONFIG_FILE: &str = "agent.config_file";
+const AA_KBC_PARAMS: &str = "agent.aa_kbc_params";
+const HTTPS_PROXY: &str = "agent.https_proxy";
+const NO_PROXY: &str = "agent.no_proxy";
+const ENABLE_DATA_INTEGRITY: &str = "agent.data_integrity";
+const ENABLE_SIGNATURE_VERIFICATION: &str = "agent.enable_signature_verification";
+const IMAGE_POLICY_FILE: &str = "agent.image_policy";
+const IMAGE_REGISTRY_AUTH_FILE: &str = "agent.image_registry_auth";
+const SIMPLE_SIGNING_SIGSTORE_CONFIG: &str = "agent.simple_signing_sigstore_config";
 
 const DEFAULT_LOG_LEVEL: slog::Level = slog::Level::Info;
 const DEFAULT_HOTPLUG_TIMEOUT: time::Duration = time::Duration::from_secs(3);
@@ -77,6 +86,15 @@ pub struct AgentConfig {
     pub tracing: bool,
     pub endpoints: AgentEndpoints,
     pub supports_seccomp: bool,
+    pub container_policy_path: String,
+    pub aa_kbc_params: String,
+    pub https_proxy: String,
+    pub no_proxy: String,
+    pub data_integrity: bool,
+    pub enable_signature_verification: bool,
+    pub image_policy_file: String,
+    pub image_registry_auth_file: String,
+    pub simple_signing_sigstore_config: String,
 }
 
 #[derive(Debug, Deserialize)]
@@ -92,6 +110,15 @@ pub struct AgentConfigBuilder {
     pub unified_cgroup_hierarchy: Option<bool>,
     pub tracing: Option<bool>,
     pub endpoints: Option<EndpointsConfig>,
+    pub container_policy_path: Option<String>,
+    pub aa_kbc_params: Option<String>,
+    pub https_proxy: Option<String>,
+    pub no_proxy: Option<String>,
+    pub data_integrity: Option<bool>,
+    pub enable_signature_verification: Option<bool>,
+    pub image_policy_file: Option<String>,
+    pub image_registry_auth_file: Option<String>,
+    pub simple_signing_sigstore_config: Option<String>,
 }
 
 macro_rules! config_override {
@@ -153,6 +180,15 @@ impl Default for AgentConfig {
             tracing: false,
             endpoints: Default::default(),
             supports_seccomp: rpc::have_seccomp(),
+            container_policy_path: String::from(""),
+            aa_kbc_params: String::from(""),
+            https_proxy: String::from(""),
+            no_proxy: String::from(""),
+            data_integrity: false,
+            enable_signature_verification: true,
+            image_policy_file: String::from(""),
+            image_registry_auth_file: String::from(""),
+            simple_signing_sigstore_config: String::from(""),
         }
     }
 }
@@ -181,6 +217,23 @@ impl FromStr for AgentConfig {
         config_override!(agent_config_builder, agent_config, server_addr);
         config_override!(agent_config_builder, agent_config, unified_cgroup_hierarchy);
         config_override!(agent_config_builder, agent_config, tracing);
+        config_override!(agent_config_builder, agent_config, container_policy_path);
+        config_override!(agent_config_builder, agent_config, aa_kbc_params);
+        config_override!(agent_config_builder, agent_config, https_proxy);
+        config_override!(agent_config_builder, agent_config, no_proxy);
+        config_override!(agent_config_builder, agent_config, data_integrity);
+        config_override!(
+            agent_config_builder,
+            agent_config,
+            enable_signature_verification
+        );
+        config_override!(agent_config_builder, agent_config, image_policy_file);
+        config_override!(agent_config_builder, agent_config, image_registry_auth_file);
+        config_override!(
+            agent_config_builder,
+            agent_config,
+            simple_signing_sigstore_config
+        );
 
         // Populate the allowed endpoints hash set, if we got any from the config file.
         if let Some(endpoints) = agent_config_builder.endpoints {
@@ -209,6 +262,10 @@ impl AgentConfig {
         let mut config: AgentConfig = Default::default();
         let cmdline = fs::read_to_string(file)?;
         let params: Vec<&str> = cmdline.split_ascii_whitespace().collect();
+
+        let mut using_config_file = false;
+        // Check if there is config file before parsing params that might
+        // override values from the config file.
         for param in params.iter() {
             // If we get a configuration file path from the command line, we
             // generate our config from it.
@@ -216,10 +273,15 @@ impl AgentConfig {
             // or if it can't be parsed properly.
             if param.starts_with(format!("{}=", CONFIG_FILE).as_str()) {
                 let config_file = get_string_value(param)?;
-                return AgentConfig::from_config_file(&config_file)
-                    .context("AgentConfig from kernel cmdline");
+                config = AgentConfig::from_config_file(&config_file)
+                    .context("AgentConfig from kernel cmdline")
+                    .unwrap();
+                using_config_file = true;
+                break;
             }
+        }
 
+        for param in params.iter() {
             // parse cmdline flags
             parse_cmdline_param!(param, DEBUG_CONSOLE_FLAG, config.debug_console);
             parse_cmdline_param!(param, DEV_MODE_FLAG, config.dev_mode);
@@ -279,6 +341,48 @@ impl AgentConfig {
                 config.unified_cgroup_hierarchy,
                 get_bool_value
             );
+
+            parse_cmdline_param!(param, AA_KBC_PARAMS, config.aa_kbc_params, get_string_value);
+            parse_cmdline_param!(param, HTTPS_PROXY, config.https_proxy, get_url_value);
+            parse_cmdline_param!(param, NO_PROXY, config.no_proxy, get_string_value);
+            parse_cmdline_param!(
+                param,
+                ENABLE_DATA_INTEGRITY,
+                config.data_integrity,
+                get_bool_value
+            );
+
+            parse_cmdline_param!(
+                param,
+                ENABLE_SIGNATURE_VERIFICATION,
+                config.enable_signature_verification,
+                get_bool_value
+            );
+
+            // URI of the image security file
+            parse_cmdline_param!(
+                param,
+                IMAGE_POLICY_FILE,
+                config.image_policy_file,
+                get_string_value
+            );
+
+            // URI of the registry auth file
+            parse_cmdline_param!(
+                param,
+                IMAGE_REGISTRY_AUTH_FILE,
+                config.image_registry_auth_file,
+                get_string_value
+            );
+
+            // URI of the simple signing sigstore file
+            // used when simple signing verification is used
+            parse_cmdline_param!(
+                param,
+                SIMPLE_SIGNING_SIGSTORE_CONFIG,
+                config.simple_signing_sigstore_config,
+                get_string_value
+            );
         }
 
         if let Ok(addr) = env::var(SERVER_ADDR_ENV_VAR) {
@@ -298,7 +402,9 @@ impl AgentConfig {
         }
 
         // We did not get a configuration file: allow all endpoints.
-        config.endpoints.all_allowed = true;
+        if !using_config_file {
+            config.endpoints.all_allowed = true;
+        }
 
         Ok(config)
     }
@@ -433,6 +539,12 @@ fn get_container_pipe_size(param: &str) -> Result<i32> {
     Ok(value)
 }
 
+#[instrument]
+fn get_url_value(param: &str) -> Result<String> {
+    let value = get_string_value(param)?;
+    Ok(Url::parse(&value)?.to_string())
+}
+
 #[cfg(test)]
 mod tests {
     use test_utils::assert_result;
@@ -451,6 +563,11 @@ mod tests {
         assert!(!config.dev_mode);
         assert_eq!(config.log_level, DEFAULT_LOG_LEVEL);
         assert_eq!(config.hotplug_timeout, DEFAULT_HOTPLUG_TIMEOUT);
+        assert_eq!(config.container_policy_path, "");
+        assert!(config.enable_signature_verification);
+        assert_eq!(config.image_policy_file, "");
+        assert_eq!(config.image_registry_auth_file, "");
+        assert_eq!(config.simple_signing_sigstore_config, "");
     }
 
     #[test]
@@ -469,6 +586,15 @@ mod tests {
             server_addr: &'a str,
             unified_cgroup_hierarchy: bool,
             tracing: bool,
+            container_policy_path: &'a str,
+            aa_kbc_params: &'a str,
+            https_proxy: &'a str,
+            no_proxy: &'a str,
+            data_integrity: bool,
+            enable_signature_verification: bool,
+            image_policy_file: &'a str,
+            image_registry_auth_file: &'a str,
+            simple_signing_sigstore_config: &'a str,
         }
 
         impl Default for TestData<'_> {
@@ -484,6 +610,15 @@ mod tests {
                     server_addr: TEST_SERVER_ADDR,
                     unified_cgroup_hierarchy: false,
                     tracing: false,
+                    container_policy_path: "",
+                    aa_kbc_params: "",
+                    https_proxy: "",
+                    no_proxy: "",
+                    data_integrity: false,
+                    enable_signature_verification: true,
+                    image_policy_file: "",
+                    image_registry_auth_file: "",
+                    simple_signing_sigstore_config: "",
                 }
             }
         }
@@ -853,6 +988,126 @@ mod tests {
                 tracing: true,
                 ..Default::default()
             },
+            TestData {
+                contents: "agent.aa_kbc_params=offline_fs_kbc::null",
+                aa_kbc_params: "offline_fs_kbc::null",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.aa_kbc_params=eaa_kbc::127.0.0.1:50000",
+                aa_kbc_params: "eaa_kbc::127.0.0.1:50000",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.https_proxy=http://proxy.url.com:81/",
+                https_proxy: "http://proxy.url.com:81/",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.https_proxy=http://192.168.1.100:81/",
+                https_proxy: "http://192.168.1.100:81/",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.no_proxy=*.internal.url.com",
+                no_proxy: "*.internal.url.com",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.no_proxy=192.168.1.0/24,172.16.0.0/12",
+                no_proxy: "192.168.1.0/24,172.16.0.0/12",
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                data_integrity: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.data_integrity=true",
+                data_integrity: true,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.data_integrity=false",
+                data_integrity: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.data_integrity=1",
+                data_integrity: true,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.data_integrity=0",
+                data_integrity: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.enable_signature_verification=false",
+                enable_signature_verification: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.enable_signature_verification=0",
+                enable_signature_verification: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.enable_signature_verification=1",
+                enable_signature_verification: true,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.enable_signature_verification=foo",
+                enable_signature_verification: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.image_policy=file:///etc/policy.json",
+                image_policy_file: "file:///etc/policy.json",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.image_policy=kbs:///default/security-policy/test",
+                image_policy_file: "kbs:///default/security-policy/test",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.image_policy=kbs://example.kbs.org/default/security-policy/test",
+                image_policy_file: "kbs://example.kbs.org/default/security-policy/test",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.image_registry_auth=file:///etc/auth.json",
+                image_registry_auth_file: "file:///etc/auth.json",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.image_registry_auth=kbs:///default/credential/test",
+                image_registry_auth_file: "kbs:///default/credential/test",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.image_registry_auth=kbs://example.kbs.org/default/credential/test",
+                image_registry_auth_file: "kbs://example.kbs.org/default/credential/test",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.simple_signing_sigstore_config=file:///etc/containers/signature/default.yml",
+                simple_signing_sigstore_config: "file:///etc/containers/signature/default.yml",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.simple_signing_sigstore_config=kbs:///default/sigstore-config/test",
+                simple_signing_sigstore_config: "kbs:///default/sigstore-config/test",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.simple_signing_sigstore_config=kbs://example.kbs.org/default/sigstore-config/test",
+                simple_signing_sigstore_config: "kbs://example.kbs.org/default/sigstore-config/test",
+                ..Default::default()
+            },
         ];
 
         let dir = tempdir().expect("failed to create tmpdir");
@@ -900,6 +1155,31 @@ mod tests {
             assert_eq!(d.container_pipe_size, config.container_pipe_size, "{}", msg);
             assert_eq!(d.server_addr, config.server_addr, "{}", msg);
             assert_eq!(d.tracing, config.tracing, "{}", msg);
+            assert_eq!(
+                d.container_policy_path, config.container_policy_path,
+                "{}",
+                msg
+            );
+            assert_eq!(d.aa_kbc_params, config.aa_kbc_params, "{}", msg);
+            assert_eq!(d.https_proxy, config.https_proxy, "{}", msg);
+            assert_eq!(d.no_proxy, config.no_proxy, "{}", msg);
+            assert_eq!(d.data_integrity, config.data_integrity, "{}", msg);
+            assert_eq!(
+                d.enable_signature_verification, config.enable_signature_verification,
+                "{}",
+                msg
+            );
+            assert_eq!(d.image_policy_file, config.image_policy_file, "{}", msg);
+            assert_eq!(
+                d.image_registry_auth_file, config.image_registry_auth_file,
+                "{}",
+                msg
+            );
+            assert_eq!(
+                d.simple_signing_sigstore_config, config.simple_signing_sigstore_config,
+                "{}",
+                msg
+            );
 
             for v in vars_to_unset {
                 env::remove_var(v);
@@ -1401,4 +1681,50 @@ Caused by:
         // Verify that the default values are valid
         assert_eq!(config.hotplug_timeout, DEFAULT_HOTPLUG_TIMEOUT);
     }
+
+    #[test]
+    fn test_config_from_cmdline_and_config_file() {
+        let dir = tempdir().expect("failed to create tmpdir");
+
+        let agent_config = r#"
+               dev_mode = false
+               server_addr = 'vsock://8:2048'
+
+               [endpoints]
+               allowed = ["CreateContainer", "StartContainer"]
+              "#;
+
+        let config_path = dir.path().join("agent-config.toml");
+        let config_filename = config_path.to_str().expect("failed to get config filename");
+
+        fs::write(config_filename, agent_config).expect("failed to write agen config");
+
+        let cmdline = format!("agent.devmode agent.config_file={}", config_filename);
+
+        let cmdline_path = dir.path().join("cmdline");
+        let cmdline_filename = cmdline_path
+            .to_str()
+            .expect("failed to get cmdline filename");
+
+        fs::write(cmdline_filename, cmdline).expect("failed to write agen config");
+
+        let config = AgentConfig::from_cmdline(cmdline_filename, vec![])
+            .expect("failed to parse command line");
+
+        // Should be overwritten by cmdline
+        assert!(config.dev_mode);
+
+        // Should be from agent config
+        assert_eq!(config.server_addr, "vsock://8:2048");
+
+        // Should be from agent config
+        assert_eq!(
+            config.endpoints.allowed,
+            vec!["CreateContainer".to_string(), "StartContainer".to_string()]
+                .iter()
+                .cloned()
+                .collect()
+        );
+        assert!(!config.endpoints.all_allowed);
+    }
 }

src/agent/src/device.rs

@@ -651,13 +651,15 @@ fn update_spec_devices(spec: &mut Spec, mut updates: HashMap<&str, DevUpdate>) -
 
     if let Some(resources) = linux.resources.as_mut() {
         for r in &mut resources.devices {
-            if let (Some(host_major), Some(host_minor)) = (r.major, r.minor) {
-                if let Some(update) = res_updates.get(&(r.r#type.as_str(), host_major, host_minor))
+            if let (Some(host_type), Some(host_major), Some(host_minor)) =
+                (r.r#type.as_ref(), r.major, r.minor)
+            {
+                if let Some(update) = res_updates.get(&(host_type.as_str(), host_major, host_minor))
                 {
                     info!(
                         sl(),
                         "update_spec_devices() updating resource";
-                        "type" => &r.r#type,
+                        "type" => &host_type,
                         "host_major" => host_major,
                         "host_minor" => host_minor,
                         "guest_major" => update.guest_major,
@@ -969,7 +971,7 @@ pub fn update_device_cgroup(spec: &mut Spec) -> Result<()> {
         allow: false,
         major: Some(major),
         minor: Some(minor),
-        r#type: String::from("b"),
+        r#type: Some(String::from("b")),
         access: String::from("rw"),
     });
 
@@ -1132,13 +1134,13 @@ mod tests {
                 resources: Some(LinuxResources {
                     devices: vec![
                         oci::LinuxDeviceCgroup {
-                            r#type: "c".to_string(),
+                            r#type: Some("c".to_string()),
                             major: Some(host_major_a),
                             minor: Some(host_minor_a),
                             ..oci::LinuxDeviceCgroup::default()
                         },
                         oci::LinuxDeviceCgroup {
-                            r#type: "c".to_string(),
+                            r#type: Some("c".to_string()),
                             major: Some(host_major_b),
                             minor: Some(host_minor_b),
                             ..oci::LinuxDeviceCgroup::default()
@@ -1231,13 +1233,13 @@ mod tests {
                 resources: Some(LinuxResources {
                     devices: vec![
                         LinuxDeviceCgroup {
-                            r#type: "c".to_string(),
+                            r#type: Some("c".to_string()),
                             major: Some(host_major),
                             minor: Some(host_minor),
                             ..LinuxDeviceCgroup::default()
                         },
                         LinuxDeviceCgroup {
-                            r#type: "b".to_string(),
+                            r#type: Some("b".to_string()),
                             major: Some(host_major),
                             minor: Some(host_minor),
                             ..LinuxDeviceCgroup::default()

src/agent/src/image_rpc.rs

@@ -0,0 +1,366 @@
+// Copyright (c) 2021 Alibaba Cloud
+// Copyright (c) 2021, 2023 IBM Corporation
+// Copyright (c) 2022 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::env;
+use std::fs;
+use std::path::Path;
+use std::process::Command;
+use std::sync::atomic::{AtomicBool, AtomicU16, Ordering};
+use std::sync::Arc;
+
+use anyhow::{anyhow, Result};
+use async_trait::async_trait;
+use protocols::image;
+use tokio::sync::Mutex;
+use ttrpc::{self, error::get_rpc_status as ttrpc_error};
+
+use crate::rpc::{verify_cid, CONTAINER_BASE};
+use crate::sandbox::Sandbox;
+use crate::AGENT_CONFIG;
+
+use image_rs::image::ImageClient;
+use std::io::Write;
+
+const AA_PATH: &str = "/usr/local/bin/attestation-agent";
+
+const AA_KEYPROVIDER_URI: &str =
+    "unix:///run/confidential-containers/attestation-agent/keyprovider.sock";
+const AA_GETRESOURCE_URI: &str =
+    "unix:///run/confidential-containers/attestation-agent/getresource.sock";
+
+const OCICRYPT_CONFIG_PATH: &str = "/tmp/ocicrypt_config.json";
+// kata rootfs is readonly, use tmpfs before CC storage is implemented.
+const KATA_CC_IMAGE_WORK_DIR: &str = "/run/image/";
+const KATA_CC_PAUSE_BUNDLE: &str = "/pause_bundle";
+const CONFIG_JSON: &str = "config.json";
+
+// Convenience function to obtain the scope logger.
+fn sl() -> slog::Logger {
+    slog_scope::logger().new(o!("subsystem" => "cgroups"))
+}
+
+pub struct ImageService {
+    sandbox: Arc<Mutex<Sandbox>>,
+    attestation_agent_started: AtomicBool,
+    image_client: Arc<Mutex<ImageClient>>,
+    container_count: Arc<AtomicU16>,
+}
+
+impl ImageService {
+    pub async fn new(sandbox: Arc<Mutex<Sandbox>>) -> Self {
+        env::set_var("CC_IMAGE_WORK_DIR", KATA_CC_IMAGE_WORK_DIR);
+        let mut image_client = ImageClient::default();
+
+        let image_policy_file = &AGENT_CONFIG.image_policy_file;
+        if !image_policy_file.is_empty() {
+            image_client.config.file_paths.sigstore_config = image_policy_file.clone();
+        }
+
+        let simple_signing_sigstore_config = &AGENT_CONFIG.simple_signing_sigstore_config;
+        if !simple_signing_sigstore_config.is_empty() {
+            image_client.config.file_paths.sigstore_config = simple_signing_sigstore_config.clone();
+        }
+
+        let image_registry_auth_file = &AGENT_CONFIG.image_registry_auth_file;
+        if !image_registry_auth_file.is_empty() {
+            image_client.config.file_paths.auth_file = image_registry_auth_file.clone();
+        }
+
+        Self {
+            sandbox,
+            attestation_agent_started: AtomicBool::new(false),
+            image_client: Arc::new(Mutex::new(image_client)),
+            container_count: Arc::new(AtomicU16::new(0)),
+        }
+    }
+
+    // pause image is packaged in rootfs for CC
+    fn unpack_pause_image(cid: &str) -> Result<()> {
+        let cc_pause_bundle = Path::new(KATA_CC_PAUSE_BUNDLE);
+        if !cc_pause_bundle.exists() {
+            return Err(anyhow!("Pause image not present in rootfs"));
+        }
+
+        info!(sl(), "use guest pause image cid {:?}", cid);
+        let pause_bundle = Path::new(CONTAINER_BASE).join(cid);
+        let pause_rootfs = pause_bundle.join("rootfs");
+        let pause_config = pause_bundle.join(CONFIG_JSON);
+        let pause_binary = pause_rootfs.join("pause");
+        fs::create_dir_all(&pause_rootfs)?;
+        if !pause_config.exists() {
+            fs::copy(
+                cc_pause_bundle.join(CONFIG_JSON),
+                pause_bundle.join(CONFIG_JSON),
+            )?;
+        }
+        if !pause_binary.exists() {
+            fs::copy(cc_pause_bundle.join("rootfs").join("pause"), pause_binary)?;
+        }
+
+        Ok(())
+    }
+
+    // If we fail to start the AA, ocicrypt won't be able to unwrap keys
+    // and container decryption will fail.
+    fn init_attestation_agent() -> Result<()> {
+        let config_path = OCICRYPT_CONFIG_PATH;
+
+        // The image will need to be encrypted using a keyprovider
+        // that has the same name (at least according to the config).
+        let ocicrypt_config = serde_json::json!({
+            "key-providers": {
+                "attestation-agent":{
+                    "ttrpc":AA_KEYPROVIDER_URI
+                }
+            }
+        });
+
+        let mut config_file = fs::File::create(config_path)?;
+        config_file.write_all(ocicrypt_config.to_string().as_bytes())?;
+
+        // The Attestation Agent will run for the duration of the guest.
+        Command::new(AA_PATH)
+            .arg("--keyprovider_sock")
+            .arg(AA_KEYPROVIDER_URI)
+            .arg("--getresource_sock")
+            .arg(AA_GETRESOURCE_URI)
+            .spawn()?;
+        Ok(())
+    }
+
+    /// Determines the container id (cid) to use for a given request.
+    ///
+    /// If the request specifies a non-empty id, use it; otherwise derive it from the image path.
+    /// In either case, verify that the chosen id is valid.
+    fn cid_from_request(&self, req: &image::PullImageRequest) -> Result<String> {
+        let req_cid = req.container_id();
+        let cid = if !req_cid.is_empty() {
+            req_cid.to_string()
+        } else if let Some(last) = req.image().rsplit('/').next() {
+            // Support multiple containers with same image
+            let index = self.container_count.fetch_add(1, Ordering::Relaxed);
+
+            // ':' not valid for container id
+            format!("{}_{}", last.replace(':', "_"), index)
+        } else {
+            return Err(anyhow!("Invalid image name. {}", req.image()));
+        };
+        verify_cid(&cid)?;
+        Ok(cid)
+    }
+
+    async fn pull_image(&self, req: &image::PullImageRequest) -> Result<String> {
+        env::set_var("OCICRYPT_KEYPROVIDER_CONFIG", OCICRYPT_CONFIG_PATH);
+
+        let https_proxy = &AGENT_CONFIG.https_proxy;
+        if !https_proxy.is_empty() {
+            env::set_var("HTTPS_PROXY", https_proxy);
+        }
+
+        let no_proxy = &AGENT_CONFIG.no_proxy;
+        if !no_proxy.is_empty() {
+            env::set_var("NO_PROXY", no_proxy);
+        }
+
+        let cid = self.cid_from_request(req)?;
+        let image = req.image();
+        if cid.starts_with("pause") {
+            Self::unpack_pause_image(&cid)?;
+
+            let mut sandbox = self.sandbox.lock().await;
+            sandbox.images.insert(String::from(image), cid);
+            return Ok(image.to_owned());
+        }
+
+        let aa_kbc_params = &AGENT_CONFIG.aa_kbc_params;
+        if !aa_kbc_params.is_empty() {
+            match self.attestation_agent_started.compare_exchange_weak(
+                false,
+                true,
+                Ordering::SeqCst,
+                Ordering::SeqCst,
+            ) {
+                Ok(_) => Self::init_attestation_agent()?,
+                Err(_) => info!(sl(), "Attestation Agent already running"),
+            }
+        }
+        // If the attestation-agent is being used, then enable the authenticated credentials support
+        info!(
+            sl(),
+            "image_client.config.auth set to: {}",
+            !aa_kbc_params.is_empty()
+        );
+        self.image_client.lock().await.config.auth = !aa_kbc_params.is_empty();
+
+        // Read enable signature verification from the agent config and set it in the image_client
+        let enable_signature_verification = &AGENT_CONFIG.enable_signature_verification;
+        info!(
+            sl(),
+            "enable_signature_verification set to: {}", enable_signature_verification
+        );
+        self.image_client.lock().await.config.security_validate = *enable_signature_verification;
+
+        let source_creds = (!req.source_creds().is_empty()).then(|| req.source_creds());
+
+        let bundle_path = Path::new(CONTAINER_BASE).join(&cid);
+        fs::create_dir_all(&bundle_path)?;
+
+        let decrypt_config = format!("provider:attestation-agent:{}", aa_kbc_params);
+
+        info!(sl(), "pull image {:?}, bundle path {:?}", cid, bundle_path);
+        // Image layers will store at KATA_CC_IMAGE_WORK_DIR, generated bundles
+        // with rootfs and config.json will store under CONTAINER_BASE/cid.
+        let res = self
+            .image_client
+            .lock()
+            .await
+            .pull_image(image, &bundle_path, &source_creds, &Some(&decrypt_config))
+            .await;
+
+        match res {
+            Ok(image) => {
+                info!(
+                    sl(),
+                    "pull and unpack image {:?}, cid: {:?}, with image-rs succeed. ", image, cid
+                );
+            }
+            Err(e) => {
+                error!(
+                    sl(),
+                    "pull and unpack image {:?}, cid: {:?}, with image-rs failed with {:?}. ",
+                    image,
+                    cid,
+                    e.to_string()
+                );
+                return Err(e);
+            }
+        };
+
+        let mut sandbox = self.sandbox.lock().await;
+        sandbox.images.insert(String::from(image), cid);
+        Ok(image.to_owned())
+    }
+}
+
+#[async_trait]
+impl protocols::image_ttrpc_async::Image for ImageService {
+    async fn pull_image(
+        &self,
+        _ctx: &ttrpc::r#async::TtrpcContext,
+        req: image::PullImageRequest,
+    ) -> ttrpc::Result<image::PullImageResponse> {
+        match self.pull_image(&req).await {
+            Ok(r) => {
+                let mut resp = image::PullImageResponse::new();
+                resp.image_ref = r;
+                return Ok(resp);
+            }
+            Err(e) => {
+                return Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()));
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::ImageService;
+    use crate::sandbox::Sandbox;
+    use protocols::image;
+    use std::sync::Arc;
+    use tokio::sync::Mutex;
+
+    #[tokio::test]
+    async fn test_cid_from_request() {
+        struct Case {
+            cid: &'static str,
+            image: &'static str,
+            result: Option<&'static str>,
+        }
+
+        let cases = [
+            Case {
+                cid: "",
+                image: "",
+                result: None,
+            },
+            Case {
+                cid: "..",
+                image: "",
+                result: None,
+            },
+            Case {
+                cid: "",
+                image: "..",
+                result: None,
+            },
+            Case {
+                cid: "",
+                image: "abc/..",
+                result: None,
+            },
+            Case {
+                cid: "",
+                image: "abc/",
+                result: None,
+            },
+            Case {
+                cid: "",
+                image: "../abc",
+                result: Some("abc_4"),
+            },
+            Case {
+                cid: "",
+                image: "../9abc",
+                result: Some("9abc_5"),
+            },
+            Case {
+                cid: "some-string.1_2",
+                image: "",
+                result: Some("some-string.1_2"),
+            },
+            Case {
+                cid: "0some-string.1_2",
+                image: "",
+                result: Some("0some-string.1_2"),
+            },
+            Case {
+                cid: "a:b",
+                image: "",
+                result: None,
+            },
+            Case {
+                cid: "",
+                image: "prefix/a:b",
+                result: Some("a_b_6"),
+            },
+            Case {
+                cid: "",
+                image: "/a/b/c/d:e",
+                result: Some("d_e_7"),
+            },
+        ];
+
+        let logger = slog::Logger::root(slog::Discard, o!());
+        let s = Sandbox::new(&logger).unwrap();
+        let image_service = ImageService::new(Arc::new(Mutex::new(s))).await;
+        for case in &cases {
+            let mut req = image::PullImageRequest::new();
+            req.set_image(case.image.to_string());
+            req.set_container_id(case.cid.to_string());
+            let ret = image_service.cid_from_request(&req);
+            match (case.result, ret) {
+                (Some(expected), Ok(actual)) => assert_eq!(expected, actual),
+                (None, Err(_)) => (),
+                (None, Ok(r)) => panic!("Expected an error, got {}", r),
+                (Some(expected), Err(e)) => {
+                    panic!("Expected {} but got an error ({})", expected, e)
+                }
+            }
+        }
+    }
+}

src/agent/src/main.rs

@@ -70,6 +70,7 @@ use tokio::{
     task::JoinHandle,
 };
 
+mod image_rpc;
 mod rpc;
 mod tracer;
 
@@ -344,7 +345,7 @@ async fn start_sandbox(
     sandbox.lock().await.sender = Some(tx);
 
     // vsock:///dev/vsock, port
-    let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str(), init_mode)?;
+    let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str(), init_mode).await?;
     server.start().await?;
 
     rx.await?;

src/agent/src/rpc.rs

@@ -37,7 +37,10 @@ use protocols::health::{
     VersionCheckResponse,
 };
 use protocols::types::Interface;
-use protocols::{agent_ttrpc_async as agent_ttrpc, health_ttrpc_async as health_ttrpc};
+use protocols::{
+    agent_ttrpc_async as agent_ttrpc, health_ttrpc_async as health_ttrpc,
+    image_ttrpc_async as image_ttrpc,
+};
 use rustjail::cgroups::notifier;
 use rustjail::container::{BaseContainer, Container, LinuxContainer, SYSTEMD_CGROUP_PATH_FORMAT};
 use rustjail::mount::parse_mount_table;
@@ -53,6 +56,7 @@ use rustjail::process::ProcessOperations;
 use crate::device::{
     add_devices, get_virtio_blk_pci_device_name, update_device_cgroup, update_env_pci,
 };
+use crate::image_rpc;
 use crate::linux_abi::*;
 use crate::metrics::get_metrics;
 use crate::mount::{add_storages, baremount, update_ephemeral_mounts, STORAGE_HANDLER_LIST};
@@ -84,8 +88,12 @@ use std::io::{BufRead, BufReader, Write};
 use std::os::unix::fs::FileExt;
 use std::path::PathBuf;
 
-const CONTAINER_BASE: &str = "/run/kata-containers";
+pub const CONTAINER_BASE: &str = "/run/kata-containers";
 const MODPROBE_PATH: &str = "/sbin/modprobe";
+const ANNO_K8S_IMAGE_NAME: &str = "io.kubernetes.cri.image-name";
+const CONFIG_JSON: &str = "config.json";
+const INIT_TRUSTED_STORAGE: &str = "/usr/bin/kata-init-trusted-storage";
+const TRUSTED_STORAGE_DEVICE: &str = "/dev/trusted_store";
 
 /// the iptables seriers binaries could appear either in /sbin
 /// or /usr/sbin, we need to check both of them
@@ -137,6 +145,41 @@ pub struct AgentService {
     init_mode: bool,
 }
 
+// A container ID must match this regex:
+//
+//     ^[a-zA-Z0-9][a-zA-Z0-9_.-]+$
+//
+pub fn verify_cid(id: &str) -> Result<()> {
+    let mut chars = id.chars();
+
+    let valid = matches!(chars.next(), Some(first) if first.is_alphanumeric()
+                && id.len() > 1
+                && chars.all(|c| c.is_alphanumeric() || ['.', '-', '_'].contains(&c)));
+
+    match valid {
+        true => Ok(()),
+        false => Err(anyhow!("invalid container ID: {:?}", id)),
+    }
+}
+
+// Partially merge an OCI process specification into another one.
+fn merge_oci_process(target: &mut oci::Process, source: &oci::Process) {
+    if target.args.is_empty() && !source.args.is_empty() {
+        target.args.append(&mut source.args.clone());
+    }
+
+    if target.cwd == "/" && source.cwd != "/" {
+        target.cwd = String::from(&source.cwd);
+    }
+
+    for source_env in &source.env {
+        let variable_name: Vec<&str> = source_env.split('=').collect();
+        if !target.env.iter().any(|i| i.contains(variable_name[0])) {
+            target.env.push(source_env.to_string());
+        }
+    }
+}
+
 impl AgentService {
     #[instrument]
     async fn do_create_container(
@@ -167,6 +210,9 @@ impl AgentService {
             "receive createcontainer, storages: {:?}", &req.storages
         );
 
+        // Merge the image bundle OCI spec into the container creation request OCI spec.
+        self.merge_bundle_oci(&mut oci).await?;
+
         // Some devices need some extra processing (the ones invoked with
         // --device for instance), and that's what this call is doing. It
         // updates the devices listed in the OCI spec, so that they actually
@@ -174,6 +220,30 @@ impl AgentService {
         // cannot predict everything from the caller.
         add_devices(&req.devices.to_vec(), &mut oci, &self.sandbox).await?;
 
+        let linux = oci
+            .linux
+            .as_mut()
+            .ok_or_else(|| anyhow!("Spec didn't contain linux field"))?;
+
+        for specdev in &mut linux.devices {
+            let dev_major_minor = format!("{}:{}", specdev.major, specdev.minor);
+
+            if specdev.path == TRUSTED_STORAGE_DEVICE {
+                let data_integrity = AGENT_CONFIG.data_integrity;
+                info!(
+                    sl(),
+                    "trusted_store device major:min {}, enable data integrity {}",
+                    dev_major_minor,
+                    data_integrity.to_string()
+                );
+
+                Command::new(INIT_TRUSTED_STORAGE)
+                    .args([&dev_major_minor, &data_integrity.to_string()])
+                    .output()
+                    .expect("Failed to initialize confidential storage");
+            }
+        }
+
         // Both rootfs and volumes (invoked with --volume for instance) will
         // be processed the same way. The idea is to always mount any provided
         // storage to the specified MountPoint, so that it will match what's
@@ -635,6 +705,54 @@ impl AgentService {
             }
         }
     }
+
+    // When being passed an image name through a container annotation, merge its
+    // corresponding bundle OCI specification into the passed container creation one.
+    async fn merge_bundle_oci(&self, container_oci: &mut oci::Spec) -> Result<()> {
+        if let Some(image_name) = container_oci
+            .annotations
+            .get(&ANNO_K8S_IMAGE_NAME.to_string())
+        {
+            if let Some(container_id) = self.sandbox.clone().lock().await.images.get(image_name) {
+                let image_oci_config_path = Path::new(CONTAINER_BASE)
+                    .join(container_id)
+                    .join(CONFIG_JSON);
+                debug!(
+                    sl(),
+                    "Image bundle config path: {:?}", image_oci_config_path
+                );
+
+                let image_oci =
+                    oci::Spec::load(image_oci_config_path.to_str().ok_or_else(|| {
+                        anyhow!(
+                            "Invalid container image OCI config path {:?}",
+                            image_oci_config_path
+                        )
+                    })?)
+                    .context("load image bundle")?;
+
+                if let Some(container_root) = container_oci.root.as_mut() {
+                    if let Some(image_root) = image_oci.root.as_ref() {
+                        let root_path = Path::new(CONTAINER_BASE)
+                            .join(container_id)
+                            .join(image_root.path.clone());
+                        container_root.path =
+                            String::from(root_path.to_str().ok_or_else(|| {
+                                anyhow!("Invalid container image root path {:?}", root_path)
+                            })?);
+                    }
+                }
+
+                if let Some(container_process) = container_oci.process.as_mut() {
+                    if let Some(image_process) = image_oci.process.as_ref() {
+                        merge_oci_process(container_process, image_process);
+                    }
+                }
+            }
+        }
+
+        Ok(())
+    }
 }
 
 #[async_trait]
@@ -1716,9 +1834,13 @@ async fn read_stream(reader: Arc<Mutex<ReadHalf<PipeStream>>>, l: usize) -> Resu
     Ok(content)
 }
 
-pub fn start(s: Arc<Mutex<Sandbox>>, server_address: &str, init_mode: bool) -> Result<TtrpcServer> {
+pub async fn start(
+    s: Arc<Mutex<Sandbox>>,
+    server_address: &str,
+    init_mode: bool,
+) -> Result<TtrpcServer> {
     let agent_service = Box::new(AgentService {
-        sandbox: s,
+        sandbox: s.clone(),
         init_mode,
     }) as Box<dyn agent_ttrpc::AgentService + Send + Sync>;
 
@@ -1727,14 +1849,20 @@ pub fn start(s: Arc<Mutex<Sandbox>>, server_address: &str, init_mode: bool) -> R
     let health_service = Box::new(HealthService {}) as Box<dyn health_ttrpc::Health + Send + Sync>;
     let health_worker = Arc::new(health_service);
 
+    let image_service = Box::new(image_rpc::ImageService::new(s).await)
+        as Box<dyn image_ttrpc::Image + Send + Sync>;
+
     let aservice = agent_ttrpc::create_agent_service(agent_worker);
 
     let hservice = health_ttrpc::create_health(health_worker);
 
+    let iservice = image_ttrpc::create_image(Arc::new(image_service));
+
     let server = TtrpcServer::new()
         .bind(server_address)?
         .register_service(aservice)
-        .register_service(hservice);
+        .register_service(hservice)
+        .register_service(iservice);
 
     info!(sl(), "ttRPC server started"; "address" => server_address);
 
@@ -1935,6 +2063,38 @@ fn do_copy_file(req: &CopyFileRequest) -> Result<()> {
         }
     }
 
+    let sflag = stat::SFlag::from_bits_truncate(req.file_mode);
+
+    if sflag.contains(stat::SFlag::S_IFDIR) {
+        fs::create_dir(path.clone()).or_else(|e| {
+            if e.kind() != std::io::ErrorKind::AlreadyExists {
+                return Err(e);
+            }
+            Ok(())
+        })?;
+
+        std::fs::set_permissions(path.clone(), std::fs::Permissions::from_mode(req.file_mode))?;
+
+        unistd::chown(
+            &path,
+            Some(Uid::from_raw(req.uid as u32)),
+            Some(Gid::from_raw(req.gid as u32)),
+        )?;
+
+        return Ok(());
+    }
+
+    if sflag.contains(stat::SFlag::S_IFLNK) {
+        let src = PathBuf::from(String::from_utf8(req.data.clone()).unwrap());
+
+        unistd::symlinkat(&src, None, &path)?;
+        let path_str = CString::new(path.to_str().unwrap())?;
+        let ret = unsafe { libc::lchown(path_str.as_ptr(), req.uid as u32, req.gid as u32) };
+        Errno::result(ret).map(drop)?;
+
+        return Ok(());
+    }
+
     let mut tmpfile = path.clone();
     tmpfile.set_extension("tmp");
 
@@ -2000,18 +2160,26 @@ pub fn setup_bundle(cid: &str, spec: &mut Spec) -> Result<PathBuf> {
     let spec_root_path = Path::new(&spec_root.path);
 
     let bundle_path = Path::new(CONTAINER_BASE).join(cid);
-    let config_path = bundle_path.join("config.json");
+    let config_path = bundle_path.join(CONFIG_JSON);
     let rootfs_path = bundle_path.join("rootfs");
 
-    fs::create_dir_all(&rootfs_path)?;
-    baremount(
-        spec_root_path,
-        &rootfs_path,
-        "bind",
-        MsFlags::MS_BIND,
-        "",
+    let rootfs_exists = Path::new(&rootfs_path).exists();
+    info!(
         &sl(),
-    )?;
+        "The rootfs_path is {:?} and exists: {}", rootfs_path, rootfs_exists
+    );
+
+    if !rootfs_exists {
+        fs::create_dir_all(&rootfs_path)?;
+        baremount(
+            spec_root_path,
+            &rootfs_path,
+            "bind",
+            MsFlags::MS_BIND,
+            "",
+            &sl(),
+        )?;
+    }
 
     let rootfs_path_name = rootfs_path
         .to_str()
@@ -2992,4 +3160,135 @@ COMMIT
             "We should see the resulting rule"
         );
     }
+
+    #[tokio::test]
+    async fn test_merge_cwd() {
+        #[derive(Debug)]
+        struct TestData<'a> {
+            container_process_cwd: &'a str,
+            image_process_cwd: &'a str,
+            expected: &'a str,
+        }
+
+        let tests = &[
+            // Image cwd should override blank container cwd
+            // TODO - how can we tell the user didn't specifically set it to `/` vs not setting at all? Is that scenario valid?
+            TestData {
+                container_process_cwd: "/",
+                image_process_cwd: "/imageDir",
+                expected: "/imageDir",
+            },
+            // Container cwd should override image cwd
+            TestData {
+                container_process_cwd: "/containerDir",
+                image_process_cwd: "/imageDir",
+                expected: "/containerDir",
+            },
+            // Container cwd should override blank image cwd
+            TestData {
+                container_process_cwd: "/containerDir",
+                image_process_cwd: "/",
+                expected: "/containerDir",
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let mut container_process = oci::Process {
+                cwd: d.container_process_cwd.to_string(),
+                ..Default::default()
+            };
+
+            let image_process = oci::Process {
+                cwd: d.image_process_cwd.to_string(),
+                ..Default::default()
+            };
+
+            merge_oci_process(&mut container_process, &image_process);
+
+            assert_eq!(d.expected, container_process.cwd, "{}", msg);
+        }
+    }
+
+    #[tokio::test]
+    async fn test_merge_env() {
+        #[derive(Debug)]
+        struct TestData {
+            container_process_env: Vec<String>,
+            image_process_env: Vec<String>,
+            expected: Vec<String>,
+        }
+
+        let tests = &[
+            // Test that the pods environment overrides the images
+            TestData {
+                container_process_env: vec!["ISPRODUCTION=true".to_string()],
+                image_process_env: vec!["ISPRODUCTION=false".to_string()],
+                expected: vec!["ISPRODUCTION=true".to_string()],
+            },
+            // Test that multiple environment variables can be overrided
+            TestData {
+                container_process_env: vec![
+                    "ISPRODUCTION=true".to_string(),
+                    "ISDEVELOPMENT=false".to_string(),
+                ],
+                image_process_env: vec![
+                    "ISPRODUCTION=false".to_string(),
+                    "ISDEVELOPMENT=true".to_string(),
+                ],
+                expected: vec![
+                    "ISPRODUCTION=true".to_string(),
+                    "ISDEVELOPMENT=false".to_string(),
+                ],
+            },
+            // Test that when none of the variables match do not override them
+            TestData {
+                container_process_env: vec!["ANOTHERENV=TEST".to_string()],
+                image_process_env: vec![
+                    "ISPRODUCTION=false".to_string(),
+                    "ISDEVELOPMENT=true".to_string(),
+                ],
+                expected: vec![
+                    "ANOTHERENV=TEST".to_string(),
+                    "ISPRODUCTION=false".to_string(),
+                    "ISDEVELOPMENT=true".to_string(),
+                ],
+            },
+            // Test a mix of both overriding and not
+            TestData {
+                container_process_env: vec![
+                    "ANOTHERENV=TEST".to_string(),
+                    "ISPRODUCTION=true".to_string(),
+                ],
+                image_process_env: vec![
+                    "ISPRODUCTION=false".to_string(),
+                    "ISDEVELOPMENT=true".to_string(),
+                ],
+                expected: vec![
+                    "ANOTHERENV=TEST".to_string(),
+                    "ISPRODUCTION=true".to_string(),
+                    "ISDEVELOPMENT=true".to_string(),
+                ],
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let mut container_process = oci::Process {
+                env: d.container_process_env.clone(),
+                ..Default::default()
+            };
+
+            let image_process = oci::Process {
+                env: d.image_process_env.clone(),
+                ..Default::default()
+            };
+
+            merge_oci_process(&mut container_process, &image_process);
+
+            assert_eq!(d.expected, container_process.env, "{}", msg);
+        }
+    }
 }

src/agent/src/sandbox.rs

@@ -62,6 +62,7 @@ pub struct Sandbox {
     pub event_tx: Option<Sender<String>>,
     pub bind_watcher: BindWatcher,
     pub pcimap: HashMap<pci::Address, pci::Address>,
+    pub images: HashMap<String, String>,
 }
 
 impl Sandbox {
@@ -95,6 +96,7 @@ impl Sandbox {
             event_tx: Some(tx),
             bind_watcher: BindWatcher::new(),
             pcimap: HashMap::new(),
+            images: HashMap::new(),
         })
     }
 

src/dragonball/src/api/v1/instance_info.rs

@@ -5,6 +5,15 @@
 
 use serde_derive::{Deserialize, Serialize};
 
+/// This struct represents the strongly typed equivalent of the json body
+/// from confidential container related requests.
+#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Serialize)]
+#[serde(deny_unknown_fields)]
+pub enum ConfidentialVmType {
+    /// Intel Trusted Domain
+    TDX = 2,
+}
+
 /// The microvm state.
 ///
 /// When Dragonball starts, the instance state is Uninitialized. Once start_microvm method is
@@ -58,10 +67,12 @@ pub struct InstanceInfo {
     pub tids: Vec<(u8, u32)>,
     /// Last instance downtime
     pub last_instance_downtime: u64,
+    /// confidential vm type
+    pub confidential_vm_type: Option<ConfidentialVmType>,
 }
 
 impl InstanceInfo {
-    /// create instance info object with given id, version, and platform type
+    /// create instance info object with given id, version, platform type and confidential vm type.
     pub fn new(id: String, vmm_version: String) -> Self {
         InstanceInfo {
             id,
@@ -72,8 +83,14 @@ impl InstanceInfo {
             async_state: AsyncState::Uninitialized,
             tids: Vec::new(),
             last_instance_downtime: 0,
+            confidential_vm_type: None,
         }
     }
+
+    /// return true if VM confidential type is TDX
+    pub fn is_tdx_enabled(&self) -> bool {
+        matches!(self.confidential_vm_type, Some(ConfidentialVmType::TDX))
+    }
 }
 
 impl Default for InstanceInfo {
@@ -87,6 +104,7 @@ impl Default for InstanceInfo {
             async_state: AsyncState::Uninitialized,
             tids: Vec::new(),
             last_instance_downtime: 0,
+            confidential_vm_type: None,
         }
     }
 }

src/dragonball/src/api/v1/mod.rs

@@ -12,7 +12,7 @@ pub use self::boot_source::{BootSourceConfig, BootSourceConfigError, DEFAULT_KER
 
 /// Wrapper over the microVM general information.
 mod instance_info;
-pub use self::instance_info::{InstanceInfo, InstanceState};
+pub use self::instance_info::{ConfidentialVmType, InstanceInfo, InstanceState};
 
 /// Wrapper for configuring the memory and CPU of the microVM.
 mod machine_config;

src/dragonball/src/error.rs

@@ -75,6 +75,10 @@ pub enum Error {
     /// Cannot open the VM file descriptor.
     #[error(transparent)]
     Vm(vm::VmError),
+
+    /// confidential vm type Error
+    #[error("confidential-vm-type can only be used in x86_64 now")]
+    ConfidentialVmType,
 }
 
 /// Errors associated with starting the instance.

src/dragonball/src/kvm_context.rs

@@ -10,7 +10,7 @@ use kvm_bindings::KVM_API_VERSION;
 use kvm_ioctls::{Cap, Kvm, VmFd};
 use std::os::unix::io::{FromRawFd, RawFd};
 
-use crate::error::{Error, Result};
+use crate::error::{Error as VmError, Result};
 
 /// Describes a KVM context that gets attached to the micro VM instance.
 /// It gives access to the functionality of the KVM wrapper as long as every required
@@ -29,11 +29,11 @@ impl KvmContext {
             // Safe because we expect kvm_fd to contain a valid fd number when is_some() == true.
             unsafe { Kvm::from_raw_fd(fd) }
         } else {
-            Kvm::new().map_err(Error::Kvm)?
+            Kvm::new().map_err(VmError::Kvm)?
         };
 
         if kvm.get_api_version() != KVM_API_VERSION as i32 {
-            return Err(Error::KvmApiVersion(kvm.get_api_version()));
+            return Err(VmError::KvmApiVersion(kvm.get_api_version()));
         }
 
         Self::check_cap(&kvm, Cap::Irqchip)?;
@@ -44,7 +44,8 @@ impl KvmContext {
         Self::check_cap(&kvm, Cap::SetTssAddr)?;
 
         #[cfg(target_arch = "x86_64")]
-        let supported_msrs = dbs_arch::msr::supported_guest_msrs(&kvm).map_err(Error::GuestMSRs)?;
+        let supported_msrs =
+            dbs_arch::msr::supported_guest_msrs(&kvm).map_err(VmError::GuestMSRs)?;
         let max_memslots = kvm.get_nr_memslots();
 
         Ok(KvmContext {
@@ -67,7 +68,7 @@ impl KvmContext {
 
     /// Create a virtual machine object.
     pub fn create_vm(&self) -> Result<VmFd> {
-        self.kvm.create_vm().map_err(Error::Kvm)
+        self.kvm.create_vm().map_err(VmError::Kvm)
     }
 
     /// Get the max vcpu count supported by kvm
@@ -75,9 +76,9 @@ impl KvmContext {
         self.kvm.get_max_vcpus()
     }
 
-    fn check_cap(kvm: &Kvm, cap: Cap) -> std::result::Result<(), Error> {
+    fn check_cap(kvm: &Kvm, cap: Cap) -> std::result::Result<(), VmError> {
         if !kvm.check_extension(cap) {
-            return Err(Error::KvmCap(cap));
+            return Err(VmError::KvmCap(cap));
         }
         Ok(())
     }
@@ -91,6 +92,18 @@ mod x86_64 {
     use std::collections::HashSet;
 
     impl KvmContext {
+        /// Create a virtual machine object with specific type.
+        /// vm_type: u64
+        ///      0: legacy vm
+        ///      2: tdx vm
+        pub fn create_vm_with_type(&self, vm_type: u64) -> Result<VmFd> {
+            let fd = self
+                .kvm
+                .create_vm_with_type(vm_type)
+                .map_err(VmError::Kvm)?;
+            Ok(fd)
+        }
+
         /// Get information about supported CPUID of x86 processor.
         pub fn supported_cpuid(
             &self,
@@ -110,7 +123,7 @@ mod x86_64 {
         // It's very sensible to manipulate MSRs, so please be careful to change code below.
         fn build_msrs_list(kvm: &Kvm) -> Result<Msrs> {
             let mut mset: HashSet<u32> = HashSet::new();
-            let supported_msr_list = kvm.get_msr_index_list().map_err(super::Error::Kvm)?;
+            let supported_msr_list = kvm.get_msr_index_list().map_err(VmError::Kvm)?;
             for msr in supported_msr_list.as_slice() {
                 mset.insert(*msr);
             }
@@ -203,7 +216,7 @@ mod x86_64 {
                 })
                 .collect();
 
-            Msrs::from_entries(&msrs).map_err(super::Error::Msr)
+            Msrs::from_entries(&msrs).map_err(VmError::Msr)
         }
     }
 }
@@ -257,4 +270,20 @@ mod tests {
 
         let _ = c.create_vm().unwrap();
     }
+
+    #[test]
+    fn test_create_vm_with_type() {
+        let c = KvmContext::new(None).unwrap();
+        #[cfg(not(target_arch = "aarch64"))]
+        let _ = c.create_vm_with_type(0_u64).unwrap();
+        #[cfg(target_arch = "aarch64")]
+        {
+            /// aarch64 is using ipa_size to create vm
+            let mut ipa_size = 0; // Create using default VM type
+            if c.check_extension(kvm_ioctls::Cap::ArmVmIPASize) {
+                ipa_size = c.kvm.get_host_ipa_limit();
+            }
+            let _ = c.create_vm_with_type(ipa_size as u64).unwrap();
+        }
+    }
 }

src/dragonball/src/vm/mod.rs

@@ -215,7 +215,26 @@ impl Vm {
         let id = api_shared_info.read().unwrap().id.clone();
         let logger = slog_scope::logger().new(slog::o!("id" => id));
         let kvm = KvmContext::new(kvm_fd)?;
-        let vm_fd = Arc::new(kvm.create_vm()?);
+        let vm_fd = match api_shared_info
+            .as_ref()
+            .read()
+            .unwrap()
+            .confidential_vm_type
+        {
+            None => Arc::new(kvm.create_vm()?),
+            Some(confidential_vm_type) => {
+                #[cfg(not(any(target_arch = "x86_64")))]
+                {
+                    error!(
+                        "confidential-vm-type {} only can be used in x86_64",
+                        confidential_vm_type as u64
+                    );
+                    return Err(Error::ConfidentialVmType);
+                }
+                #[cfg(target_arch = "x86_64")]
+                Arc::new(kvm.create_vm_with_type(confidential_vm_type as u64)?)
+            }
+        };
         let resource_manager = Arc::new(ResourceManager::new(Some(kvm.max_memslots())));
         let device_manager = DeviceManager::new(
             vm_fd.clone(),
@@ -344,6 +363,15 @@ impl Vm {
         instance_state == InstanceState::Running
     }
 
+    /// return true if VM confidential type is TDX
+    pub fn is_tdx_enabled(&self) -> bool {
+        let shared_info = self
+            .shared_info()
+            .read()
+            .expect("failed to get instance state, because shared info is poisoned lock");
+        shared_info.is_tdx_enabled()
+    }
+
     /// Save VM instance exit state
     pub fn vm_exit(&self, exit_code: i32) {
         if let Ok(mut info) = self.shared_info.write() {

src/libs/oci/src/lib.rs

@@ -512,8 +512,8 @@ pub struct LinuxDevice {
 pub struct LinuxDeviceCgroup {
     #[serde(default)]
     pub allow: bool,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
-    pub r#type: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub r#type: Option<String>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub major: Option<i64>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -1463,21 +1463,21 @@ mod tests {
                     devices: vec![
                         crate::LinuxDeviceCgroup {
                             allow: false,
-                            r#type: "".to_string(),
+                            r#type: None,
                             major: None,
                             minor: None,
                             access: "rwm".to_string(),
                         },
                         crate::LinuxDeviceCgroup {
                             allow: true,
-                            r#type: "c".to_string(),
+                            r#type: Some("c".to_string()),
                             major: Some(10),
                             minor: Some(229),
                             access: "rw".to_string(),
                         },
                         crate::LinuxDeviceCgroup {
                             allow: true,
-                            r#type: "b".to_string(),
+                            r#type: Some("b".to_string()),
                             major: Some(8),
                             minor: Some(0),
                             access: "r".to_string(),

src/libs/protocols/build.rs

@@ -198,13 +198,30 @@ fn real_main() -> Result<(), std::io::Error> {
     // generate async
     #[cfg(feature = "async")]
     {
-        codegen("src", &["protos/agent.proto", "protos/health.proto"], true)?;
+        codegen(
+            "src",
+            &[
+                "protos/agent.proto",
+                "protos/health.proto",
+                "protos/image.proto",
+            ],
+            true,
+        )?;
 
         fs::rename("src/agent_ttrpc.rs", "src/agent_ttrpc_async.rs")?;
         fs::rename("src/health_ttrpc.rs", "src/health_ttrpc_async.rs")?;
+        fs::rename("src/image_ttrpc.rs", "src/image_ttrpc_async.rs")?;
     }
 
-    codegen("src", &["protos/agent.proto", "protos/health.proto"], false)?;
+    codegen(
+        "src",
+        &[
+            "protos/agent.proto",
+            "protos/health.proto",
+            "protos/image.proto",
+        ],
+        false,
+    )?;
 
     // There is a message named 'Box' in oci.proto
     // so there is a struct named 'Box', we should replace Box<Self> to ::std::boxed::Box<Self>

src/libs/protocols/protos/image.proto

@@ -0,0 +1,31 @@
+//
+// Copyright (c) 2021 Alibaba Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+syntax = "proto3";
+
+option go_package = "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc";
+
+package grpc;
+
+// Image defines the public APIs for managing images.
+service Image {
+    // PullImage pulls an image with authentication config.
+    rpc PullImage(PullImageRequest) returns (PullImageResponse) {}
+}
+
+message PullImageRequest {
+    // Image name (e.g. docker.io/library/busybox:latest).
+    string image = 1;
+    // Unique image identifier, used to avoid duplication when unpacking the image layers.
+    string container_id = 2;
+    // Use USERNAME[:PASSWORD] for accessing the registry
+    string source_creds = 3;
+}
+
+message PullImageResponse {
+    // Reference to the image in use. For most runtimes, this should be an
+    // image ID or digest.
+    string image_ref = 1;
+}

src/libs/protocols/src/lib.rs

@@ -16,6 +16,10 @@ pub mod health;
 pub mod health_ttrpc;
 #[cfg(feature = "async")]
 pub mod health_ttrpc_async;
+pub mod image;
+pub mod image_ttrpc;
+#[cfg(feature = "async")]
+pub mod image_ttrpc_async;
 pub mod oci;
 #[cfg(feature = "with-serde")]
 mod serde_config;

src/libs/protocols/src/trans.rs

@@ -100,7 +100,7 @@ impl From<oci::LinuxDeviceCgroup> for crate::oci::LinuxDeviceCgroup {
     fn from(from: oci::LinuxDeviceCgroup) -> Self {
         crate::oci::LinuxDeviceCgroup {
             Allow: from.allow,
-            Type: from.r#type,
+            Type: from.r#type.map_or("".to_string(), |t| t as String),
             Major: from.major.map_or(0, |t| t),
             Minor: from.minor.map_or(0, |t| t),
             Access: from.access,
@@ -452,7 +452,7 @@ impl From<crate::oci::LinuxDeviceCgroup> for oci::LinuxDeviceCgroup {
 
         oci::LinuxDeviceCgroup {
             allow: from.Allow(),
-            r#type: from.take_Type(),
+            r#type: Some(from.take_Type()),
             major,
             minor,
             access: from.take_Access(),

src/runtime/Makefile

@@ -56,6 +56,7 @@ BINLIBEXECLIST :=
 BIN_PREFIX = $(PROJECT_TYPE)
 PROJECT_DIR = $(PROJECT_TAG)
 IMAGENAME = $(PROJECT_TAG).img
+IMAGETDXNAME = $(PROJECT_TAG)-tdx.img
 INITRDNAME = $(PROJECT_TAG)-initrd.img
 INITRDSEVNAME = $(PROJECT_TAG)-initrd-sev.img
 
@@ -102,6 +103,9 @@ GENERATED_VARS = \
 		CONFIG_QEMU_SNP_IN \
 		CONFIG_CLH_IN \
 		CONFIG_FC_IN \
+		CONFIG_CLH_TDX_IN \
+		CONFIG_QEMU_SE_IN \
+		CONFIG_REMOTE_IN \
 		$(USER_VARS)
 SCRIPTS += $(COLLECT_SCRIPT)
 SCRIPTS_DIR := $(BINDIR)
@@ -116,6 +120,7 @@ PKGLIBEXECDIR := $(LIBEXECDIR)/$(PROJECT_DIR)
 KERNELDIR := $(PKGDATADIR)
 
 IMAGEPATH := $(PKGDATADIR)/$(IMAGENAME)
+IMAGETDXPATH := $(PKGDATADIR)/$(IMAGETDXNAME)
 INITRDPATH := $(PKGDATADIR)/$(INITRDNAME)
 INITRDSEVPATH := $(PKGDATADIR)/$(INITRDSEVNAME)
 
@@ -126,16 +131,29 @@ DEFROOTFSTYPE := $(ROOTFSTYPE_EXT4)
 
 FIRMWAREPATH :=
 FIRMWAREVOLUMEPATH :=
+TDSHIMFIRMWAREPATH := ${PREFIXDEPS}/share/td-shim/td-shim.bin
 
-FIRMWARETDVFPATH := $(PREFIXDEPS)/share/tdvf/OVMF.fd
-FIRMWARETDVFVOLUMEPATH :=
-
-FIRMWARESEVPATH := $(PREFIXDEPS)/share/ovmf/OVMF.fd
-FIRMWARESNPPATH := $(PREFIXDEPS)/share/ovmf/OVMF.fd
+AGENTCONFIGFILEPATH := /etc/agent-config.toml
+AGENTCONFIGFILEKERNELPARAM := agent.config_file=$(AGENTCONFIGFILEPATH)
 
 ROOTMEASURECONFIG ?= ""
-KERNELPARAMS += $(ROOTMEASURECONFIG)
-KERNELTDXPARAMS += $(ROOTMEASURECONFIG)
+ROOTMEASURECONFIGTDX ?= ""
+AGENT_AA_KBC_PARAMS ?= ""
+AGENT_AA_KBC_PARAMS_TDX ?= ""
+AGENT_AA_KBC_PARAMS_SEV ?= ""
+AGENT_AA_KBC_PARAMS_SNP ?= ""
+TDXKERNELPARAMS := tdx_disable_filter agent.enable_signature_verification=false $(AGENT_AA_KBC_PARAMS_TDX)
+KERNELTDXPARAMS += $(TDXKERNELPARAMS) $(ROOTMEASURECONFIGTDX)
+TDXKERNELPARAMS_CLH += $(TDXKERNELPARAMS) $(ROOTMEASURECONFIG)
+SEVKERNELPARAMS := $(AGENTCONFIGFILEKERNELPARAM) agent.enable_signature_verification=false $(AGENT_AA_KBC_PARAMS_SEV)
+SNPKERNELPARAMS := $(AGENTCONFIGFILEKERNELPARAM) agent.enable_signature_verification=false $(AGENT_AA_KBC_PARAMS_SNP)
+KERNELPARAMS += $(ROOTMEASURECONFIG) agent.enable_signature_verification=false $(AGENT_AA_KBC_PARAMS)
+
+FIRMWARETDVFPATH := $(PREFIXDEPS)/share/tdvf/OVMF_CODE.fd
+FIRMWARETDVFVOLUMEPATH := $(PREFIXDEPS)/share/tdvf/OVMF_VARS.fd
+
+FIRMWARESEVPATH := $(PREFIXDEPS)/share/ovmf/AMDSEV.fd
+FIRMWARESNPPATH := $(PREFIXDEPS)/share/ovmf/OVMF.fd
 
 # Name of default configuration file the runtime will use.
 CONFIG_FILE = configuration.toml
@@ -144,16 +162,20 @@ HYPERVISOR_ACRN = acrn
 HYPERVISOR_FC = firecracker
 HYPERVISOR_QEMU = qemu
 HYPERVISOR_CLH = cloud-hypervisor
+HYPERVISOR_REMOTE = remote
 
 # Determines which hypervisor is specified in $(CONFIG_FILE).
 DEFAULT_HYPERVISOR ?= $(HYPERVISOR_QEMU)
 
 # List of hypervisors this build system can generate configuration for.
-HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH)
+HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH) $(HYPERVISOR_REMOTE)
 
 QEMUPATH := $(QEMUBINDIR)/$(QEMUCMD)
 QEMUVALIDHYPERVISORPATHS := [\"$(QEMUPATH)\"]
 
+QEMUSNPPATH := $(QEMUBINDIR)/$(QEMUSNPCMD)
+QEMUSNPVALIDHYPERVISORPATHS := [\"$(QEMUSNPPATH)\"]
+
 QEMUTDXPATH := $(QEMUBINDIR)/$(QEMUTDXCMD)
 QEMUTDXVALIDHYPERVISORPATHS := [\"$(QEMUTDXPATH)\"]
 
@@ -253,6 +275,20 @@ DEFBINDMOUNTS := []
 # Image Service Offload
 DEFSERVICEOFFLOAD ?= false
 
+# SEV & SEV-ES Guest Pre-Attestation
+DEFGUESTPREATTESTATION ?= false
+DEFGUESTPREATTESTATIONPROXY ?= localhost:44444
+DEFGUESTPREATTESTATIONMODE ?= online
+DEFGUESTPREATTESTATIONKEYSET ?= KEYSET-1
+DEFSEVCERTCHAIN ?= /opt/sev/cert_chain.cert
+DEFSEVGUESTPOLICY ?= 3
+
+# SNP
+DEFSNPGUEST ?= true
+# Based on SEV Secure Nested Paging Firmware ABI Specification section 4.3
+# unspecified or == 0 --> 0x30000 i.e. Bit#17 is '1' and Bit#16 is '1' (SMT is allowed)
+DEFSNPGUESTPOLICY ?= 0x30000
+
 SED = sed
 
 CLI_DIR = cmd
@@ -315,6 +351,18 @@ ifneq (,$(QEMUCMD))
 
     CONFIGS += $(CONFIG_QEMU_SEV)
 
+    CONFIG_FILE_QEMU_SE = configuration-qemu-se.toml
+    CONFIG_QEMU_SE = config/$(CONFIG_FILE_QEMU_SE)
+    CONFIG_QEMU_SE_IN = $(CONFIG_QEMU_SE).in
+
+    CONFIG_PATH_QEMU_SE = $(abspath $(CONFDIR)/$(CONFIG_FILE_QEMU_SE))
+    CONFIG_PATHS += $(CONFIG_PATH_QEMU_SE)
+
+    SYSCONFIG_QEMU_SE = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_QEMU_SE))
+    SYSCONFIG_PATHS += $(SYSCONFIG_QEMU_SE)
+
+    CONFIGS += $(CONFIG_QEMU_SE)
+
     CONFIG_FILE_QEMU_SNP = configuration-qemu-snp.toml
     CONFIG_QEMU_SNP = config/$(CONFIG_FILE_QEMU_SNP)
     CONFIG_QEMU_SNP_IN = $(CONFIG_QEMU_SNP).in
@@ -326,6 +374,18 @@ ifneq (,$(QEMUCMD))
     SYSCONFIG_PATHS_SNP += $(SYSCONFIG_QEMU_SNP)
 
     CONFIGS += $(CONFIG_QEMU_SNP)
+	
+    CONFIG_FILE_REMOTE = configuration-remote.toml
+    CONFIG_REMOTE = config/$(CONFIG_FILE_REMOTE)
+    CONFIG_REMOTE_IN = $(CONFIG_REMOTE).in
+
+    CONFIG_PATH_REMOTE = $(abspath $(CONFDIR)/$(CONFIG_FILE_REMOTE))
+    CONFIG_PATHS += $(CONFIG_PATH_REMOTE)
+
+    SYSCONFIG_REMOTE = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_REMOTE))
+    SYSCONFIG_PATHS += $(SYSCONFIG_REMOTE)
+
+    CONFIGS += $(CONFIG_REMOTE)
 
     CONFIG_FILE_QEMU_NVIDIA_GPU = configuration-qemu-nvidia-gpu.toml
     CONFIG_QEMU_NVIDIA_GPU = config/$(CONFIG_FILE_QEMU_NVIDIA_GPU)
@@ -350,10 +410,16 @@ ifneq (,$(QEMUCMD))
     KERNELTDXNAME = $(call MAKE_KERNEL_TDX_NAME,$(KERNELTDXTYPE))
     KERNELTDXPATH = $(KERNELDIR)/$(KERNELTDXNAME)
 
-    KERNELSNPTYPE = compressed
+    KERNELSEVTYPE = compressed
+    KERNELSEVNAME = $(call MAKE_KERNEL_SEV_NAME,$(KERNELSEVTYPE))
+    KERNELSEVPATH = $(KERNELDIR)/$(KERNELSEVNAME)
+
+	KERNELSNPTYPE = compressed
     KERNELSNPNAME = $(call MAKE_KERNEL_SNP_NAME,$(KERNELSNPTYPE))
     KERNELSNPPATH = $(KERNELDIR)/$(KERNELSNPNAME)
 
+    KERNELSENAME = kata-containers-secure.img
+    KERNELSEPATH = $(KERNELDIR)/$(KERNELSENAME)
 endif
 
 ifneq (,$(CLHCMD))
@@ -371,12 +437,28 @@ ifneq (,$(CLHCMD))
 
     CONFIGS += $(CONFIG_CLH)
 
+    CONFIG_FILE_CLH_TDX = configuration-clh-tdx.toml
+    CONFIG_CLH_TDX = config/$(CONFIG_FILE_CLH_TDX)
+    CONFIG_CLH_TDX_IN = $(CONFIG_CLH_TDX).in
+
+    CONFIG_PATH_CLH_TDX = $(abspath $(CONFDIR)/$(CONFIG_FILE_CLH_TDX))
+    CONFIG_PATHS += $(CONFIG_PATH_CLH_TDX)
+
+    SYSCONFIG_CLH_TDX = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_CLH_TDX))
+    SYSCONFIG_PATHS += $(SYSCONFIG_CLH_TDX)
+
+    CONFIGS += $(CONFIG_CLH_TDX)
+
     # CLH-specific options (all should be suffixed by "_CLH")
     # currently, huge pages are required for virtiofsd support
     DEFNETWORKMODEL_CLH := tcfilter
     KERNELTYPE_CLH = uncompressed
     KERNEL_NAME_CLH = $(call MAKE_KERNEL_NAME,$(KERNELTYPE_CLH))
     KERNELPATH_CLH = $(KERNELDIR)/$(KERNEL_NAME_CLH)
+
+    KERNELTDXTYPE_CLH = compressed
+    KERNELTDXNAME_CLH = $(call MAKE_KERNEL_TDX_NAME,$(KERNELTDXTYPE_CLH))
+    KERNELTDXPATH_CLH = $(KERNELDIR)/$(KERNELTDXNAME_CLH)
 endif
 
 ifneq (,$(FCCMD))
@@ -479,6 +561,7 @@ USER_VARS += CONFIG_CLH_IN
 USER_VARS += CONFIG_FC_IN
 USER_VARS += CONFIG_PATH
 USER_VARS += CONFIG_QEMU_IN
+USER_VARS += CONFIG_REMOTE_IN
 USER_VARS += DESTDIR
 USER_VARS += DEFAULT_HYPERVISOR
 USER_VARS += ACRNCMD
@@ -498,6 +581,8 @@ USER_VARS += FCVALIDJAILERPATHS
 USER_VARS += SYSCONFIG
 USER_VARS += IMAGENAME
 USER_VARS += IMAGEPATH
+USER_VARS += IMAGETDXNAME
+USER_VARS += IMAGETDXPATH
 USER_VARS += INITRDNAME
 USER_VARS += INITRDSEVNAME
 USER_VARS += INITRDPATH
@@ -513,14 +598,18 @@ USER_VARS += KERNELPATH_ACRN
 USER_VARS += KERNELPATH
 USER_VARS += KERNELSEVPATH
 USER_VARS += KERNELTDXPATH
+USER_VARS += KERNELSEVPATH
+USER_VARS += KERNELSEPATH
 USER_VARS += KERNELSNPPATH
 USER_VARS += KERNELPATH_CLH
+USER_VARS += KERNELTDXPATH_CLH
 USER_VARS += KERNELPATH_FC
 USER_VARS += KERNELVIRTIOFSPATH
 USER_VARS += FIRMWAREPATH
 USER_VARS += FIRMWARESEVPATH
 USER_VARS += FIRMWARETDVFPATH
 USER_VARS += FIRMWAREVOLUMEPATH
+USER_VARS += TDSHIMFIRMWAREPATH
 USER_VARS += FIRMWARETDVFVOLUMEPATH
 USER_VARS += FIRMWARESNPPATH
 USER_VARS += MACHINEACCELERATORS
@@ -528,6 +617,11 @@ USER_VARS += CPUFEATURES
 USER_VARS += TDXCPUFEATURES
 USER_VARS += DEFMACHINETYPE_CLH
 USER_VARS += KERNELPARAMS
+USER_VARS += TDXKERNELPARAMS
+USER_VARS += TDXKERNELPARAMS_QEMU
+USER_VARS += TDXKERNELPARAMS_CLH
+USER_VARS += SEVKERNELPARAMS
+USER_VARS += SNPKERNELPARAMS
 USER_VARS += KERNELTDXPARAMS
 USER_VARS += LIBEXECDIR
 USER_VARS += LOCALSTATEDIR
@@ -554,6 +648,10 @@ USER_VARS += QEMUTDXVALIDHYPERVISORPATHS
 USER_VARS += QEMUSNPVALIDHYPERVISORPATHS
 USER_VARS += QEMUVIRTIOFSCMD
 USER_VARS += QEMUVIRTIOFSPATH
+USER_VARS += QEMUSNPPATH
+USER_VARS += QEMUSNPVALIDHYPERVISORPATHS
+USER_VARS += QEMUTDXPATH
+USER_VARS += QEMUTDXVALIDHYPERVISORPATHS
 USER_VARS += RUNTIME_NAME
 USER_VARS += SHAREDIR
 USER_VARS += SYSCONFDIR
@@ -609,6 +707,15 @@ USER_VARS += DEFBINDMOUNTS
 USER_VARS += DEFSERVICEOFFLOAD
 USER_VARS += DEFVFIOMODE
 USER_VARS += BUILDFLAGS
+USER_VARS += DEFSERVICEOFFLOAD
+USER_VARS += DEFGUESTPREATTESTATION
+USER_VARS += DEFGUESTPREATTESTATIONPROXY
+USER_VARS += DEFGUESTPREATTESTATIONMODE
+USER_VARS += DEFGUESTPREATTESTATIONKEYSET
+USER_VARS += DEFSEVCERTCHAIN
+USER_VARS += DEFSEVGUESTPOLICY
+USER_VARS += DEFSNPGUESTPOLICY
+USER_VARS += DEFSNPGUEST
 
 
 V              = @
@@ -687,6 +794,14 @@ define MAKE_KERNEL_NAME
 $(if $(findstring uncompressed,$1),vmlinux.container,vmlinuz.container)
 endef
 
+define MAKE_KERNEL_TDX_NAME
+$(if $(findstring uncompressed,$1),vmlinux-tdx.container,vmlinuz-tdx.container)
+endef
+
+define MAKE_KERNEL_SEV_NAME
+$(if $(findstring uncompressed,$1),vmlinux-sev.container,vmlinuz-sev.container)
+endef
+
 define MAKE_KERNEL_VIRTIOFS_NAME
 $(if $(findstring uncompressed,$1),vmlinux-virtiofs.container,vmlinuz-virtiofs.container)
 endef
@@ -938,4 +1053,4 @@ ifneq (,$(findstring $(HYPERVISOR_ACRN),$(KNOWN_HYPERVISORS)))
 endif
 	@printf "\tassets path (PKGDATADIR) : %s\n" $(abspath $(PKGDATADIR))
 	@printf "\tshim path (PKGLIBEXECDIR) : %s\n" $(abspath $(PKGLIBEXECDIR))
-	@printf "\n"
+	@printf "\n"

src/runtime/arch/amd64-options.mk

@@ -11,9 +11,9 @@ MACHINEACCELERATORS :=
 CPUFEATURES := pmu=off
 
 QEMUCMD := qemu-system-x86_64
-QEMUTDXCMD := qemu-system-x86_64-tdx-experimental
-QEMUSNPCMD := qemu-system-x86_64-snp-experimental
+QEMUTDXCMD := qemu-system-x86_64-tdx
 TDXCPUFEATURES := -vmx-rdseed-exit,pmu=off
+QEMUSNPCMD := qemu-system-x86_64-snp-experimental
 
 # Firecracker binary name
 FCCMD := firecracker

src/runtime/cmd/kata-runtime/kata-check_amd64.go

@@ -319,6 +319,8 @@ func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
 		return kvmIsUsable()
 	case vc.AcrnHypervisor:
 		return acrnIsUsable()
+	case vc.RemoteHypervisor:
+		return nil
 	case vc.MockHypervisor:
 		return nil
 	default:

src/runtime/cmd/kata-runtime/kata-check_arm64.go

@@ -86,6 +86,9 @@ func checkKVMExtensions() error {
 }
 
 func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
+	if hypervisorType == "remote" {
+		return nil
+	}
 	if err := kvmIsUsable(); err != nil {
 		return err
 	}

src/runtime/cmd/kata-runtime/kata-check_ppc64le.go

@@ -61,6 +61,9 @@ func setCPUtype(hypervisorType vc.HypervisorType) error {
 }
 
 func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
+	if hypervisorType == "remote" {
+		return nil
+	}
 	return kvmIsUsable()
 }
 

src/runtime/cmd/kata-runtime/kata-check_s390x.go

@@ -55,6 +55,9 @@ func kvmIsUsable() error {
 }
 
 func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
+	if hypervisorType == "remote" {
+		return nil
+	}
 	return kvmIsUsable()
 }
 

src/runtime/config/configuration-clh-tdx.toml.in

@@ -0,0 +1,443 @@
+# Copyright (c) 2019 Ericsson Eurolab Deutschland GmbH
+# Copyright (c) 2021 Adobe Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# XXX: WARNING: this file is auto-generated.
+# XXX:
+# XXX: Source file: "@CONFIG_CLH_IN@"
+# XXX: Project:
+# XXX:   Name: @PROJECT_NAME@
+# XXX:   Type: @PROJECT_TYPE@
+
+[hypervisor.clh]
+path = "@CLHPATH@"
+kernel = "@KERNELTDXPATH_CLH@"
+image = "@IMAGEPATH@"
+
+# Enable confidential guest support.
+# Toggling that setting may trigger different hardware features, ranging
+# from memory encryption to both memory and CPU-state encryption and integrity.
+# The Kata Containers runtime dynamically detects the available feature set and
+# aims at enabling the largest possible one, returning an error if none is
+# available, or none is supported by the hypervisor.
+#
+# Known limitations:
+# * Does not work by design:
+#   - CPU Hotplug 
+#   - Memory Hotplug
+#   - NVDIMM devices
+#
+# Supported TEEs:
+# * Intel TDX
+#
+# Default false
+confidential_guest = true
+
+# disable applying SELinux on the VMM process (default false)
+disable_selinux=@DEFDISABLESELINUX@
+
+# disable applying SELinux on the container process
+# If set to false, the type `container_t` is applied to the container process by default.
+# Note: To enable guest SELinux, the guest rootfs must be CentOS that is created and built
+# with `SELINUX=yes`.
+# (default: true)
+disable_guest_selinux=@DEFDISABLEGUESTSELINUX@
+
+# Path to the firmware.
+# If you want Cloud Hypervisor to use a specific firmware, set its path below.
+# This is option is only used when confidential_guest is enabled.
+#
+# For more information about firmwared that can be used with specific TEEs,
+# please, refer to:
+# * Intel TDX:
+#   - td-shim: https://github.com/confidential-containers/td-shim
+#
+firmware = "@TDSHIMFIRMWAREPATH@"
+
+# List of valid annotation names for the hypervisor
+# Each member of the list is a regular expression, which is the base name
+# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
+enable_annotations = @DEFENABLEANNOTATIONS@
+
+# List of valid annotations values for the hypervisor
+# Each member of the list is a path pattern as described by glob(3).
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @CLHVALIDHYPERVISORPATHS@
+valid_hypervisor_paths = @CLHVALIDHYPERVISORPATHS@
+
+# Optional space-separated list of options to pass to the guest kernel.
+# For example, use `kernel_params = "vsyscall=emulate"` if you are having
+# trouble running pre-2.15 glibc.
+#
+# WARNING: - any parameter specified here will take priority over the default
+# parameter value of the same name used to start the virtual machine.
+# Do not set values here unless you understand the impact of doing so as you
+# may stop the virtual machine from booting.
+# To see the list of default parameters, enable hypervisor debug, create a
+# container and look for 'default-kernel-parameters' log entries.
+kernel_params = "@TDXKERNELPARAMS_CLH@"
+
+# Default number of vCPUs per SB/VM:
+# unspecified or 0                --> will be set to @DEFVCPUS@
+# < 0                             --> will be set to the actual number of physical cores
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores
+default_vcpus = 1
+
+# Default maximum number of vCPUs per SB/VM:
+# unspecified or == 0             --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when
+# the actual number of physical cores is greater than it.
+# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU
+# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs
+# can be added to a SB/VM, but the memory footprint will be big. Another example, with
+# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
+# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
+# unless you know what are you doing.
+default_maxvcpus = @DEFMAXVCPUS@
+
+# Default memory size in MiB for SB/VM.
+# If unspecified then it will be set @DEFMEMSZ@ MiB.
+default_memory = @DEFMEMSZ@
+
+# Default memory slots per SB/VM.
+# If unspecified then it will be set @DEFMEMSLOTS@.
+# This is will determine the times that memory will be hotadded to sandbox/VM.
+#memory_slots = @DEFMEMSLOTS@
+
+# Default maximum memory in MiB per SB / VM
+# unspecified or == 0           --> will be set to the actual amount of physical RAM
+# > 0 <= amount of physical RAM --> will be set to the specified number
+# > amount of physical RAM      --> will be set to the actual amount of physical RAM
+default_maxmemory = @DEFMAXMEMSZ@
+
+# Shared file system type:
+#   - virtio-fs (default)
+#   - virtio-fs-nydus
+shared_fs = "@DEFSHAREDFS_CLH_VIRTIOFS@"
+
+# Path to vhost-user-fs daemon.
+virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@"
+
+# List of valid annotations values for the virtiofs daemon
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDVIRTIOFSDAEMONPATHS@
+valid_virtio_fs_daemon_paths = @DEFVALIDVIRTIOFSDAEMONPATHS@
+
+# Default size of DAX cache in MiB
+virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@
+
+# Extra args for virtiofsd daemon
+#
+# Format example:
+#   ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"]
+# Examples:
+#   Set virtiofsd log level to debug : ["-o", "log_level=debug"] or ["-d"]
+# see `virtiofsd -h` for possible options.
+virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@
+
+# Cache mode:
+#
+#  - none
+#    Metadata, data, and pathname lookup are not cached in guest. They are
+#    always fetched from host and any changes are immediately pushed to host.
+#
+#  - auto
+#    Metadata and pathname lookup cache expires after a configured amount of
+#    time (default is 1 second). Data is cached while the file is open (close
+#    to open consistency).
+#
+#  - always
+#    Metadata, data, and pathname lookup are cached in guest and never expire.
+virtio_fs_cache = "@DEFVIRTIOFSCACHE@"
+
+# Block storage driver to be used for the hypervisor in case the container
+# rootfs is backed by a block device. This is virtio-blk.
+block_device_driver = "virtio-blk"
+
+# Enable huge pages for VM RAM, default false
+# Enabling this will result in the VM memory
+# being allocated using huge pages.
+#enable_hugepages = true
+
+# Disable the 'seccomp' feature from Cloud Hypervisor, default false
+# disable_seccomp = true
+
+# This option changes the default hypervisor and kernel parameters
+# to enable debug output where available.
+#
+# Default false
+#enable_debug = true
+
+# Path to OCI hook binaries in the *guest rootfs*.
+# This does not affect host-side hooks which must instead be added to
+# the OCI spec passed to the runtime.
+#
+# You can create a rootfs with hooks by customizing the osbuilder scripts:
+# https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder
+#
+# Hooks must be stored in a subdirectory of guest_hook_path according to their
+# hook type, i.e. "guest_hook_path/{prestart,poststart,poststop}".
+# The agent will scan these directories for executable files and add them, in
+# lexicographical order, to the lifecycle of the guest container.
+# Hooks are executed in the runtime namespace of the guest. See the official documentation:
+# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
+# Warnings will be logged if any error is encountered while scanning for hooks,
+# but it will not abort container execution.
+#guest_hook_path = "/usr/share/oci/hooks"
+#
+# These options are related to network rate limiter at the VMM level, and are
+# based on the Cloud Hypervisor I/O throttling.  Those are disabled by default
+# and we strongly advise users to refer the Cloud Hypervisor official
+# documentation for a better understanding of its internals:
+# https://github.com/cloud-hypervisor/cloud-hypervisor/blob/main/docs/io_throttling.md
+# 
+# Bandwidth rate limiter options
+# 
+# net_rate_limiter_bw_max_rate controls network I/O bandwidth (size in bits/sec
+# for SB/VM).
+# The same value is used for inbound and outbound bandwidth.
+# Default 0-sized value means unlimited rate.
+#net_rate_limiter_bw_max_rate = 0
+#
+# net_rate_limiter_bw_one_time_burst increases the initial max rate and this
+# initial extra credit does *NOT* affect the overall limit and can be used for
+# an *initial* burst of data.
+# This is *optional* and only takes effect if net_rate_limiter_bw_max_rate is
+# set to a non zero value.
+#net_rate_limiter_bw_one_time_burst = 0
+#
+# Operation rate limiter options
+#
+# net_rate_limiter_ops_max_rate controls network I/O bandwidth (size in ops/sec
+# for SB/VM).
+# The same value is used for inbound and outbound bandwidth.
+# Default 0-sized value means unlimited rate.
+#net_rate_limiter_ops_max_rate = 0
+#
+# net_rate_limiter_ops_one_time_burst increases the initial max rate and this
+# initial extra credit does *NOT* affect the overall limit and can be used for
+# an *initial* burst of data.
+# This is *optional* and only takes effect if net_rate_limiter_bw_max_rate is
+# set to a non zero value.
+#net_rate_limiter_ops_one_time_burst = 0
+#
+# These options are related to disk rate limiter at the VMM level, and are
+# based on the Cloud Hypervisor I/O throttling.  Those are disabled by default
+# and we strongly advise users to refer the Cloud Hypervisor official
+# documentation for a better understanding of its internals:
+# https://github.com/cloud-hypervisor/cloud-hypervisor/blob/main/docs/io_throttling.md
+# 
+# Bandwidth rate limiter options
+# 
+# disk_rate_limiter_bw_max_rate controls disk I/O bandwidth (size in bits/sec
+# for SB/VM).
+# The same value is used for inbound and outbound bandwidth.
+# Default 0-sized value means unlimited rate.
+#disk_rate_limiter_bw_max_rate = 0
+#
+# disk_rate_limiter_bw_one_time_burst increases the initial max rate and this
+# initial extra credit does *NOT* affect the overall limit and can be used for
+# an *initial* burst of data.
+# This is *optional* and only takes effect if disk_rate_limiter_bw_max_rate is
+# set to a non zero value.
+#disk_rate_limiter_bw_one_time_burst = 0
+#
+# Operation rate limiter options
+#
+# disk_rate_limiter_ops_max_rate controls disk I/O bandwidth (size in ops/sec
+# for SB/VM).
+# The same value is used for inbound and outbound bandwidth.
+# Default 0-sized value means unlimited rate.
+#disk_rate_limiter_ops_max_rate = 0
+#
+# disk_rate_limiter_ops_one_time_burst increases the initial max rate and this
+# initial extra credit does *NOT* affect the overall limit and can be used for
+# an *initial* burst of data.
+# This is *optional* and only takes effect if disk_rate_limiter_bw_max_rate is
+# set to a non zero value.
+#disk_rate_limiter_ops_one_time_burst = 0
+
+[agent.@PROJECT_TYPE@]
+# If enabled, make the agent display debug-level messages.
+# (default: disabled)
+#enable_debug = true
+
+# Enable agent tracing.
+#
+# If enabled, the agent will generate OpenTelemetry trace spans.
+#
+# Notes:
+#
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
+#
+# (default: disabled)
+#enable_tracing = true
+
+# Enable debug console.
+
+# If enabled, user can connect guest OS running inside hypervisor
+# through "kata-runtime exec <sandbox-id>" command
+
+#debug_console_enabled = true
+
+# Agent connection dialing timeout value in seconds
+# (default: 30)
+#dial_timeout = 30
+
+[runtime]
+# If enabled, the runtime will log additional debug messages to the
+# system log
+# (default: disabled)
+#enable_debug = true
+#
+# Internetworking model
+# Determines how the VM should be connected to the
+# the container network interface
+# Options:
+#
+#   - macvtap
+#     Used when the Container network interface can be bridged using
+#     macvtap.
+#
+#   - none
+#     Used when customize network. Only creates a tap device. No veth pair.
+#
+#   - tcfilter
+#     Uses tc filter rules to redirect traffic from the network interface
+#     provided by plugin to a tap interface connected to the VM.
+#
+internetworking_model="@DEFNETWORKMODEL_CLH@"
+
+# disable guest seccomp
+# Determines whether container seccomp profiles are passed to the virtual
+# machine and applied by the kata agent. If set to true, seccomp is not applied
+# within the guest
+# (default: true)
+disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
+
+# Apply a custom SELinux security policy to the container process inside the VM.
+# This is used when you want to apply a type other than the default `container_t`,
+# so general users should not uncomment and apply it.
+# (format: "user:role:type")
+# Note: You cannot specify MCS policy with the label because the sensitivity levels and
+# categories are determined automatically by high-level container runtimes such as containerd.
+#guest_selinux_label="@DEFGUESTSELINUXLABEL@"
+
+# If enabled, the runtime will create opentracing.io traces and spans.
+# (See https://www.jaegertracing.io/docs/getting-started).
+# (default: disabled)
+#enable_tracing = true
+
+# Set the full url to the Jaeger HTTP Thrift collector.
+# The default if not set will be "http://localhost:14268/api/traces"
+#jaeger_endpoint = ""
+
+# Sets the username to be used if basic auth is required for Jaeger.
+#jaeger_user = ""
+
+# Sets the password to be used if basic auth is required for Jaeger.
+#jaeger_password = ""
+
+# If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
+# This option may have some potential impacts to your host. It should only be used when you know what you're doing.
+# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only
+# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
+# (like OVS) directly.
+# (default: false)
+#disable_new_netns = true
+
+# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
+# The container cgroups in the host are not created, just one single cgroup per sandbox.
+# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
+# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
+# The sandbox cgroup is constrained if there is no container type annotation.
+# See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType
+sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
+
+# If enabled, the runtime will attempt to determine appropriate sandbox size (memory, CPU) before booting the virtual machine. In
+# this case, the runtime will not dynamically update the amount of memory and CPU in the virtual machine. This is generally helpful
+# when a hardware architecture or hypervisor solutions is utilized which does not support CPU and/or memory hotplug.
+# Compatibility for determining appropriate sandbox (VM) size:
+# - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O
+#   does not yet support sandbox sizing annotations.
+# - When running single containers using a tool like ctr, container sizing information will be available.
+static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT@
+
+# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path.
+# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory.
+# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts`
+# These will not be exposed to the container workloads, and are only provided for potential guest services.
+sandbox_bind_mounts=@DEFBINDMOUNTS@
+
+# VFIO Mode
+# Determines how VFIO devices should be be presented to the container.
+# Options:
+#
+#  - vfio
+#    Matches behaviour of OCI runtimes (e.g. runc) as much as
+#    possible.  VFIO devices will appear in the container as VFIO
+#    character devices under /dev/vfio.  The exact names may differ
+#    from the host (they need to match the VM's IOMMU group numbers
+#    rather than the host's)
+#
+#  - guest-kernel
+#    This is a Kata-specific behaviour that's useful in certain cases.
+#    The VFIO device is managed by whatever driver in the VM kernel
+#    claims it.  This means it will appear as one or more device nodes
+#    or network interfaces depending on the nature of the device.
+#    Using this mode requires specially built workloads that know how
+#    to locate the relevant device interfaces within the VM.
+#
+vfio_mode="@DEFVFIOMODE@"
+
+# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
+# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
+disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@
+
+# Enabled experimental feature list, format: ["a", "b"].
+# Experimental features are features not stable enough for production,
+# they may break compatibility, and are prepared for a big version bump.
+# Supported experimental features:
+# (default: [])
+experimental=@DEFAULTEXPFEATURES@
+
+# If enabled, user can run pprof tools with shim v2 process through kata-monitor.
+# (default: false)
+# enable_pprof = true
+
+# WARNING: All the options in the following section have not been implemented yet.
+# This section was added as a placeholder. DO NOT USE IT!
+[image]
+# Container image service.
+#
+# Offload the CRI image management service to the Kata agent.
+# (default: false)
+service_offload = @DEFSERVICEOFFLOAD@
+
+# Container image decryption keys provisioning.
+# Applies only if service_offload is true.
+# Keys can be provisioned locally (e.g. through a special command or
+# a local file) or remotely (usually after the guest is remotely attested).
+# The provision setting is a complete URL that lets the Kata agent decide
+# which method to use in order to fetch the keys.
+#
+# Keys can be stored in a local file, in a measured and attested initrd:
+#provision=data:///local/key/file
+#
+# Keys could be fetched through a special command or binary from the
+# initrd (guest) image, e.g. a firmware call:
+#provision=file:///path/to/bin/fetcher/in/guest
+#
+# Keys can be remotely provisioned. The Kata agent fetches them from e.g.
+# a HTTPS URL:
+#provision=https://my-key-broker.foo/tenant/<tenant-id>

src/runtime/config/configuration-clh.toml.in

@@ -441,7 +441,7 @@ experimental=@DEFAULTEXPFEATURES@
 #
 # Offload the CRI image management service to the Kata agent.
 # (default: false)
-#service_offload = true
+service_offload = @DEFSERVICEOFFLOAD@
 
 # Container image decryption keys provisioning.
 # Applies only if service_offload is true.

src/runtime/config/configuration-qemu-se.toml.in

@@ -0,0 +1,673 @@
+# Copyright (c) 2017-2019 Intel Corporation
+# Copyright (c) 2021 Adobe Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# XXX: WARNING: this file is auto-generated.
+# XXX:
+# XXX: Source file: "@CONFIG_QEMU_IN@"
+# XXX: Project:
+# XXX:   Name: @PROJECT_NAME@
+# XXX:   Type: @PROJECT_TYPE@
+
+[hypervisor.qemu]
+path = "@QEMUPATH@"
+kernel = "@KERNELSEPATH@"
+machine_type = "@MACHINETYPE@"
+
+# Enable confidential guest support.
+# Toggling that setting may trigger different hardware features, ranging
+# from memory encryption to both memory and CPU-state encryption and integrity.
+# The Kata Containers runtime dynamically detects the available feature set and
+# aims at enabling the largest possible one, returning an error if none is
+# available, or none is supported by the hypervisor.
+#
+# Known limitations:
+# * Does not work by design:
+#   - CPU Hotplug 
+#   - Memory Hotplug
+#   - NVDIMM devices
+#
+# Default false
+confidential_guest = true
+
+# Enable running QEMU VMM as a non-root user.
+# By default QEMU VMM run as root. When this is set to true, QEMU VMM process runs as
+# a non-root random user. See documentation for the limitations of this mode.
+# rootless = true
+
+# List of valid annotation names for the hypervisor
+# Each member of the list is a regular expression, which is the base name
+# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
+enable_annotations = @DEFENABLEANNOTATIONS@
+
+# List of valid annotations values for the hypervisor
+# Each member of the list is a path pattern as described by glob(3).
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @QEMUVALIDHYPERVISORPATHS@
+valid_hypervisor_paths = @QEMUVALIDHYPERVISORPATHS@
+
+# Optional space-separated list of options to pass to the guest kernel.
+# For example, use `kernel_params = "vsyscall=emulate"` if you are having
+# trouble running pre-2.15 glibc.
+#
+# WARNING: - any parameter specified here will take priority over the default
+# parameter value of the same name used to start the virtual machine.
+# Do not set values here unless you understand the impact of doing so as you
+# may stop the virtual machine from booting.
+# To see the list of default parameters, enable hypervisor debug, create a
+# container and look for 'default-kernel-parameters' log entries.
+kernel_params = "@KERNELPARAMS@"
+
+# Path to the firmware.
+# If you want that qemu uses the default firmware leave this option empty
+firmware = "@FIRMWAREPATH@"
+
+# Path to the firmware volume.
+# firmware TDVF or OVMF can be split into FIRMWARE_VARS.fd (UEFI variables
+# as configuration) and FIRMWARE_CODE.fd (UEFI program image). UEFI variables
+# can be customized per each user while UEFI code is kept same.
+firmware_volume = "@FIRMWAREVOLUMEPATH@"
+
+# Machine accelerators
+# comma-separated list of machine accelerators to pass to the hypervisor.
+# For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"`
+machine_accelerators="@MACHINEACCELERATORS@"
+
+# Qemu seccomp sandbox feature
+# comma-separated list of seccomp sandbox features to control the syscall access.
+# For example, `seccompsandbox= "on,obsolete=deny,spawn=deny,resourcecontrol=deny"`
+# Note: "elevateprivileges=deny" doesn't work with daemonize option, so it's removed from the seccomp sandbox
+# Another note: enabling this feature may reduce performance, you may enable
+# /proc/sys/net/core/bpf_jit_enable to reduce the impact. see https://man7.org/linux/man-pages/man8/bpfc.8.html
+#seccompsandbox="@DEFSECCOMPSANDBOXPARAM@"
+
+# CPU features
+# comma-separated list of cpu features to pass to the cpu
+# For example, `cpu_features = "pmu=off,vmx=off"
+cpu_features="@CPUFEATURES@"
+
+# Default number of vCPUs per SB/VM:
+# unspecified or 0                --> will be set to @DEFVCPUS@
+# < 0                             --> will be set to the actual number of physical cores
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores
+default_vcpus = 1
+
+# Default maximum number of vCPUs per SB/VM:
+# unspecified or == 0             --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when
+# the actual number of physical cores is greater than it.
+# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU
+# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs
+# can be added to a SB/VM, but the memory footprint will be big. Another example, with
+# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
+# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
+# unless you know what are you doing.
+# NOTICE: on arm platform with gicv2 interrupt controller, set it to 8.
+default_maxvcpus = @DEFMAXVCPUS@
+
+# Bridges can be used to hot plug devices.
+# Limitations:
+# * Currently only pci bridges are supported
+# * Until 30 devices per bridge can be hot plugged.
+# * Until 5 PCI bridges can be cold plugged per VM.
+#   This limitation could be a bug in qemu or in the kernel
+# Default number of bridges per SB/VM:
+# unspecified or 0   --> will be set to @DEFBRIDGES@
+# > 1 <= 5           --> will be set to the specified number
+# > 5                --> will be set to 5
+default_bridges = @DEFBRIDGES@
+
+# Default memory size in MiB for SB/VM.
+# If unspecified then it will be set @DEFMEMSZ@ MiB.
+default_memory = @DEFMEMSZ@
+#
+# Default memory slots per SB/VM.
+# If unspecified then it will be set @DEFMEMSLOTS@.
+# This is will determine the times that memory will be hotadded to sandbox/VM.
+#memory_slots = @DEFMEMSLOTS@
+
+# Default maximum memory in MiB per SB / VM
+# unspecified or == 0           --> will be set to the actual amount of physical RAM
+# > 0 <= amount of physical RAM --> will be set to the specified number
+# > amount of physical RAM      --> will be set to the actual amount of physical RAM
+default_maxmemory = @DEFMAXMEMSZ@
+
+# The size in MiB will be plused to max memory of hypervisor.
+# It is the memory address space for the NVDIMM devie.
+# If set block storage driver (block_device_driver) to "nvdimm",
+# should set memory_offset to the size of block device.
+# Default 0
+#memory_offset = 0
+
+# Specifies virtio-mem will be enabled or not.
+# Please note that this option should be used with the command
+# "echo 1 > /proc/sys/vm/overcommit_memory".
+# Default false
+#enable_virtio_mem = true
+
+# Disable block device from being used for a container's rootfs.
+# In case of a storage driver like devicemapper where a container's
+# root file system is backed by a block device, the block device is passed
+# directly to the hypervisor for performance reasons.
+# This flag prevents the block device from being passed to the hypervisor,
+# virtio-fs is used instead to pass the rootfs.
+disable_block_device_use = @DEFDISABLEBLOCK@
+
+# Shared file system type:
+#   - virtio-fs (default)
+#   - virtio-9p
+#   - virtio-fs-nydus
+shared_fs = "@DEFSHAREDFS_QEMU_VIRTIOFS@"
+
+# Path to vhost-user-fs daemon.
+virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@"
+
+# List of valid annotations values for the virtiofs daemon
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDVIRTIOFSDAEMONPATHS@
+valid_virtio_fs_daemon_paths = @DEFVALIDVIRTIOFSDAEMONPATHS@
+
+# Default size of DAX cache in MiB
+virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@
+
+# Default size of virtqueues
+virtio_fs_queue_size = @DEFVIRTIOFSQUEUESIZE@
+
+# Extra args for virtiofsd daemon
+#
+# Format example:
+#   ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"]
+# Examples:
+#   Set virtiofsd log level to debug : ["-o", "log_level=debug"] or ["-d"]
+#
+# see `virtiofsd -h` for possible options.
+virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@
+
+# Cache mode:
+#
+#  - never
+#    Metadata, data, and pathname lookup are not cached in guest. They are
+#    always fetched from host and any changes are immediately pushed to host.
+#
+#  - auto
+#    Metadata and pathname lookup cache expires after a configured amount of
+#    time (default is 1 second). Data is cached while the file is open (close
+#    to open consistency).
+#
+#  - always
+#    Metadata, data, and pathname lookup are cached in guest and never expire.
+virtio_fs_cache = "@DEFVIRTIOFSCACHE@"
+
+# Block storage driver to be used for the hypervisor in case the container
+# rootfs is backed by a block device. This is virtio-scsi, virtio-blk
+# or nvdimm.
+block_device_driver = "@DEFBLOCKSTORAGEDRIVER_QEMU@"
+
+# aio is the I/O mechanism used by qemu
+# Options:
+#
+#   - threads
+#     Pthread based disk I/O.
+#
+#   - native
+#     Native Linux I/O.
+#
+#   - io_uring
+#     Linux io_uring API. This provides the fastest I/O operations on Linux, requires kernel>5.1 and
+#     qemu >=5.0.
+block_device_aio = "@DEFBLOCKDEVICEAIO_QEMU@"
+
+# Specifies cache-related options will be set to block devices or not.
+# Default false
+#block_device_cache_set = true
+
+# Specifies cache-related options for block devices.
+# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled.
+# Default false
+#block_device_cache_direct = true
+
+# Specifies cache-related options for block devices.
+# Denotes whether flush requests for the device are ignored.
+# Default false
+#block_device_cache_noflush = true
+
+# Enable iothreads (data-plane) to be used. This causes IO to be
+# handled in a separate IO thread. This is currently only implemented
+# for SCSI.
+#
+enable_iothreads = @DEFENABLEIOTHREADS@
+
+# Enable pre allocation of VM RAM, default false
+# Enabling this will result in lower container density
+# as all of the memory will be allocated and locked
+# This is useful when you want to reserve all the memory
+# upfront or in the cases where you want memory latencies
+# to be very predictable
+# Default false
+#enable_mem_prealloc = true
+
+# Enable huge pages for VM RAM, default false
+# Enabling this will result in the VM memory
+# being allocated using huge pages.
+# This is useful when you want to use vhost-user network
+# stacks within the container. This will automatically
+# result in memory pre allocation
+#enable_hugepages = true
+
+# Enable vhost-user storage device, default false
+# Enabling this will result in some Linux reserved block type
+# major range 240-254 being chosen to represent vhost-user devices.
+enable_vhost_user_store = @DEFENABLEVHOSTUSERSTORE@
+
+# The base directory specifically used for vhost-user devices.
+# Its sub-path "block" is used for block devices; "block/sockets" is
+# where we expect vhost-user sockets to live; "block/devices" is where
+# simulated block device nodes for vhost-user devices to live.
+vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@"
+
+# Enable vIOMMU, default false
+# Enabling this will result in the VM having a vIOMMU device
+# This will also add the following options to the kernel's
+# command line: intel_iommu=on,iommu=pt
+#enable_iommu = true
+
+# Enable IOMMU_PLATFORM, default false
+# Enabling this will result in the VM device having iommu_platform=on set
+#enable_iommu_platform = true
+
+# List of valid annotations values for the vhost user store path
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDVHOSTUSERSTOREPATHS@
+valid_vhost_user_store_paths = @DEFVALIDVHOSTUSERSTOREPATHS@
+
+# Enable file based guest memory support. The default is an empty string which
+# will disable this feature. In the case of virtio-fs, this is enabled
+# automatically and '/dev/shm' is used as the backing folder.
+# This option will be ignored if VM templating is enabled.
+#file_mem_backend = "@DEFFILEMEMBACKEND@"
+
+# List of valid annotations values for the file_mem_backend annotation
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDFILEMEMBACKENDS@
+valid_file_mem_backends = @DEFVALIDFILEMEMBACKENDS@
+
+# -pflash can add image file to VM. The arguments of it should be in format
+# of ["/path/to/flash0.img", "/path/to/flash1.img"]
+pflashes = []
+
+# This option changes the default hypervisor and kernel parameters
+# to enable debug output where available.
+#
+# Default false
+#enable_debug = true
+
+# Disable the customizations done in the runtime when it detects
+# that it is running on top a VMM. This will result in the runtime
+# behaving as it would when running on bare metal.
+#
+#disable_nesting_checks = true
+
+# This is the msize used for 9p shares. It is the number of bytes
+# used for 9p packet payload.
+#msize_9p = @DEFMSIZE9P@
+
+# If false and nvdimm is supported, use nvdimm device to plug guest image.
+# Otherwise virtio-block device is used.
+#
+# nvdimm is not supported when `confidential_guest = true`.
+#
+# Default is false
+#disable_image_nvdimm = true
+
+# VFIO devices are hotplugged on a bridge by default.
+# Enable hotplugging on root bus. This may be required for devices with
+# a large PCI bar, as this is a current limitation with hotplugging on
+# a bridge.
+# Default false
+#hotplug_vfio_on_root_bus = true
+
+# Before hot plugging a PCIe device, you need to add a pcie_root_port device.
+# Use this parameter when using some large PCI bar devices, such as Nvidia GPU
+# The value means the number of pcie_root_port
+# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35"
+# Default 0
+#pcie_root_port = 2
+
+# If vhost-net backend for virtio-net is not desired, set to true. Default is false, which trades off
+# security (vhost-net runs ring0) for network I/O performance.
+#disable_vhost_net = true
+
+#
+# Default entropy source.
+# The path to a host source of entropy (including a real hardware RNG)
+# /dev/urandom and /dev/random are two main options.
+# Be aware that /dev/random is a blocking source of entropy.  If the host
+# runs out of entropy, the VMs boot time will increase leading to get startup
+# timeouts.
+# The source of entropy /dev/urandom is non-blocking and provides a
+# generally acceptable source of entropy. It should work well for pretty much
+# all practical purposes.
+#entropy_source= "@DEFENTROPYSOURCE@"
+
+# List of valid annotations values for entropy_source
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDENTROPYSOURCES@
+valid_entropy_sources = @DEFVALIDENTROPYSOURCES@
+
+# Path to OCI hook binaries in the *guest rootfs*.
+# This does not affect host-side hooks which must instead be added to
+# the OCI spec passed to the runtime.
+#
+# You can create a rootfs with hooks by customizing the osbuilder scripts:
+# https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder
+#
+# Hooks must be stored in a subdirectory of guest_hook_path according to their
+# hook type, i.e. "guest_hook_path/{prestart,poststart,poststop}".
+# The agent will scan these directories for executable files and add them, in
+# lexicographical order, to the lifecycle of the guest container.
+# Hooks are executed in the runtime namespace of the guest. See the official documentation:
+# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
+# Warnings will be logged if any error is encountered while scanning for hooks,
+# but it will not abort container execution.
+#guest_hook_path = "/usr/share/oci/hooks"
+#
+# Use rx Rate Limiter to control network I/O inbound bandwidth(size in bits/sec for SB/VM).
+# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) to discipline traffic.
+# Default 0-sized value means unlimited rate.
+#rx_rate_limiter_max_rate = 0
+# Use tx Rate Limiter to control network I/O outbound bandwidth(size in bits/sec for SB/VM).
+# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) and ifb(Intermediate Functional Block)
+# to discipline traffic.
+# Default 0-sized value means unlimited rate.
+#tx_rate_limiter_max_rate = 0
+
+# Set where to save the guest memory dump file.
+# If set, when GUEST_PANICKED event occurred,
+# guest memeory will be dumped to host filesystem under guest_memory_dump_path,
+# This directory will be created automatically if it does not exist.
+#
+# The dumped file(also called vmcore) can be processed with crash or gdb.
+#
+# WARNING:
+#   Dump guest’s memory can take very long depending on the amount of guest memory
+#   and use much disk space.
+#guest_memory_dump_path="/var/crash/kata"
+
+# If enable paging.
+# Basically, if you want to use "gdb" rather than "crash",
+# or need the guest-virtual addresses in the ELF vmcore,
+# then you should enable paging.
+#
+# See: https://www.qemu.org/docs/master/qemu-qmp-ref.html#Dump-guest-memory for details
+#guest_memory_dump_paging=false
+
+# Enable swap in the guest. Default false.
+# When enable_guest_swap is enabled, insert a raw file to the guest as the swap device
+# if the swappiness of a container (set by annotation "io.katacontainers.container.resource.swappiness")
+# is bigger than 0.
+# The size of the swap device should be
+# swap_in_bytes (set by annotation "io.katacontainers.container.resource.swap_in_bytes") - memory_limit_in_bytes.
+# If swap_in_bytes is not set, the size should be memory_limit_in_bytes.
+# If swap_in_bytes and memory_limit_in_bytes is not set, the size should
+# be default_memory.
+#enable_guest_swap = true
+
+# use legacy serial for guest console if available and implemented for architecture. Default false
+#use_legacy_serial = true
+
+# disable applying SELinux on the VMM process (default false)
+disable_selinux=@DEFDISABLESELINUX@
+
+# disable applying SELinux on the container process
+# If set to false, the type `container_t` is applied to the container process by default.
+# Note: To enable guest SELinux, the guest rootfs must be CentOS that is created and built
+# with `SELINUX=yes`.
+# (default: true)
+disable_guest_selinux=@DEFDISABLEGUESTSELINUX@
+
+
+[factory]
+# VM templating support. Once enabled, new VMs are created from template
+# using vm cloning. They will share the same initial kernel, initramfs and
+# agent memory by mapping it readonly. It helps speeding up new container
+# creation and saves a lot of memory if there are many kata containers running
+# on the same host.
+#
+# When disabled, new VMs are created from scratch.
+#
+# Note: Requires "initrd=" to be set ("image=" is not supported).
+#
+# Default false
+#enable_template = true
+
+# Specifies the path of template.
+#
+# Default "/run/vc/vm/template"
+#template_path = "/run/vc/vm/template"
+
+# The number of caches of VMCache:
+# unspecified or == 0   --> VMCache is disabled
+# > 0                   --> will be set to the specified number
+#
+# VMCache is a function that creates VMs as caches before using it.
+# It helps speed up new container creation.
+# The function consists of a server and some clients communicating
+# through Unix socket.  The protocol is gRPC in protocols/cache/cache.proto.
+# The VMCache server will create some VMs and cache them by factory cache.
+# It will convert the VM to gRPC format and transport it when gets
+# requestion from clients.
+# Factory grpccache is the VMCache client.  It will request gRPC format
+# VM and convert it back to a VM.  If VMCache function is enabled,
+# kata-runtime will request VM from factory grpccache when it creates
+# a new sandbox.
+#
+# Default 0
+#vm_cache_number = 0
+
+# Specify the address of the Unix socket that is used by VMCache.
+#
+# Default /var/run/kata-containers/cache.sock
+#vm_cache_endpoint = "/var/run/kata-containers/cache.sock"
+
+[agent.@PROJECT_TYPE@]
+# If enabled, make the agent display debug-level messages.
+# (default: disabled)
+#enable_debug = true
+
+# Enable agent tracing.
+#
+# If enabled, the agent will generate OpenTelemetry trace spans.
+#
+# Notes:
+#
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
+#
+# (default: disabled)
+#enable_tracing = true
+
+# Comma separated list of kernel modules and their parameters.
+# These modules will be loaded in the guest kernel using modprobe(8).
+# The following example can be used to load two kernel modules with parameters
+#  - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"]
+# The first word is considered as the module name and the rest as its parameters.
+# Container will not be started when:
+#  * A kernel module is specified and the modprobe command is not installed in the guest
+#    or it fails loading the module.
+#  * The module is not available in the guest or it doesn't met the guest kernel
+#    requirements, like architecture and version.
+#
+kernel_modules=[]
+
+# Enable debug console.
+
+# If enabled, user can connect guest OS running inside hypervisor
+# through "kata-runtime exec <sandbox-id>" command
+
+#debug_console_enabled = true
+
+# Agent connection dialing timeout value in seconds
+# (default: 30)
+#dial_timeout = 30
+
+[runtime]
+# If enabled, the runtime will log additional debug messages to the
+# system log
+# (default: disabled)
+#enable_debug = true
+#
+# Internetworking model
+# Determines how the VM should be connected to the
+# the container network interface
+# Options:
+#
+#   - macvtap
+#     Used when the Container network interface can be bridged using
+#     macvtap.
+#
+#   - none
+#     Used when customize network. Only creates a tap device. No veth pair.
+#
+#   - tcfilter
+#     Uses tc filter rules to redirect traffic from the network interface
+#     provided by plugin to a tap interface connected to the VM.
+#
+internetworking_model="@DEFNETWORKMODEL_QEMU@"
+
+# disable guest seccomp
+# Determines whether container seccomp profiles are passed to the virtual
+# machine and applied by the kata agent. If set to true, seccomp is not applied
+# within the guest
+# (default: true)
+disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
+
+# vCPUs pinning settings
+# if enabled, each vCPU thread will be scheduled to a fixed CPU
+# qualified condition: num(vCPU threads) == num(CPUs in sandbox's CPUSet)
+# enable_vcpus_pinning = false
+
+# Apply a custom SELinux security policy to the container process inside the VM.
+# This is used when you want to apply a type other than the default `container_t`,
+# so general users should not uncomment and apply it.
+# (format: "user:role:type")
+# Note: You cannot specify MCS policy with the label because the sensitivity levels and
+# categories are determined automatically by high-level container runtimes such as containerd.
+#guest_selinux_label="@DEFGUESTSELINUXLABEL@"
+
+# If enabled, the runtime will create opentracing.io traces and spans.
+# (See https://www.jaegertracing.io/docs/getting-started).
+# (default: disabled)
+#enable_tracing = true
+
+# Set the full url to the Jaeger HTTP Thrift collector.
+# The default if not set will be "http://localhost:14268/api/traces"
+#jaeger_endpoint = ""
+
+# Sets the username to be used if basic auth is required for Jaeger.
+#jaeger_user = ""
+
+# Sets the password to be used if basic auth is required for Jaeger.
+#jaeger_password = ""
+
+# If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
+# This option may have some potential impacts to your host. It should only be used when you know what you're doing.
+# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only
+# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
+# (like OVS) directly.
+# (default: false)
+#disable_new_netns = true
+
+# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
+# The container cgroups in the host are not created, just one single cgroup per sandbox.
+# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
+# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
+# The sandbox cgroup is constrained if there is no container type annotation.
+# See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType
+sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
+
+# If enabled, the runtime will attempt to determine appropriate sandbox size (memory, CPU) before booting the virtual machine. In
+# this case, the runtime will not dynamically update the amount of memory and CPU in the virtual machine. This is generally helpful
+# when a hardware architecture or hypervisor solutions is utilized which does not support CPU and/or memory hotplug.
+# Compatibility for determining appropriate sandbox (VM) size:
+# - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O
+#   does not yet support sandbox sizing annotations.
+# - When running single containers using a tool like ctr, container sizing information will be available.
+static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT@
+
+# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path.
+# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory.
+# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts`
+# These will not be exposed to the container workloads, and are only provided for potential guest services.
+sandbox_bind_mounts=@DEFBINDMOUNTS@
+
+# VFIO Mode
+# Determines how VFIO devices should be be presented to the container.
+# Options:
+#
+#  - vfio
+#    Matches behaviour of OCI runtimes (e.g. runc) as much as
+#    possible.  VFIO devices will appear in the container as VFIO
+#    character devices under /dev/vfio.  The exact names may differ
+#    from the host (they need to match the VM's IOMMU group numbers
+#    rather than the host's)
+#
+#  - guest-kernel
+#    This is a Kata-specific behaviour that's useful in certain cases.
+#    The VFIO device is managed by whatever driver in the VM kernel
+#    claims it.  This means it will appear as one or more device nodes
+#    or network interfaces depending on the nature of the device.
+#    Using this mode requires specially built workloads that know how
+#    to locate the relevant device interfaces within the VM.
+#
+vfio_mode="@DEFVFIOMODE@"
+
+# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
+# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
+disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@
+
+# Enabled experimental feature list, format: ["a", "b"].
+# Experimental features are features not stable enough for production,
+# they may break compatibility, and are prepared for a big version bump.
+# Supported experimental features:
+# (default: [])
+experimental=@DEFAULTEXPFEATURES@
+
+# If enabled, user can run pprof tools with shim v2 process through kata-monitor.
+# (default: false)
+# enable_pprof = true
+
+# WARNING: All the options in the following section have not been implemented yet.
+# This section was added as a placeholder. DO NOT USE IT!
+[image]
+# Container image service.
+#
+# Offload the CRI image management service to the Kata agent.
+# (default: false)
+service_offload = @DEFSERVICEOFFLOAD@
+
+# Container image decryption keys provisioning.
+# Applies only if service_offload is true.
+# Keys can be provisioned locally (e.g. through a special command or
+# a local file) or remotely (usually after the guest is remotely attested).
+# The provision setting is a complete URL that lets the Kata agent decide
+# which method to use in order to fetch the keys.
+#
+# Keys can be stored in a local file, in a measured and attested initrd:
+#provision=data:///local/key/file
+#
+# Keys could be fetched through a special command or binary from the
+# initrd (guest) image, e.g. a firmware call:
+#provision=file:///path/to/bin/fetcher/in/guest
+#
+# Keys can be remotely provisioned. The Kata agent fetches them from e.g.
+# a HTTPS URL:
+#provision=https://my-key-broker.foo/tenant/<tenant-id>

src/runtime/config/configuration-qemu-sev.toml.in

@@ -32,6 +32,28 @@ machine_type = "@MACHINETYPE@"
 # Default false
 confidential_guest = true
 
+# Enable pre-attestation for AMD SEV and SEV-ES guests.
+# Applies only if confidential_guest is true.
+# (default: false)
+guest_pre_attestation = true
+
+# A remote key broker server (KBS) is required to validate the attestation
+# measurement and inject a secret key.  
+# See https://github.com/confidential-containers/simple-kbs
+guest_pre_attestation_kbs_uri = "@DEFGUESTPREATTESTATIONPROXY@"
+
+# The simple-kbs can be run in "online" and "offline" modes
+guest_pre_attestation_kbs_mode="@DEFGUESTPREATTESTATIONMODE@" 
+
+# Keyset ID for injected secrets (offline kbs)
+guest_pre_attestation_keyset = "@DEFGUESTPREATTESTATIONKEYSET@"
+
+# SEV guest policy
+sev_guest_policy = @DEFSEVGUESTPOLICY@
+
+# SEV certificate chain path 
+sev_cert_chain = "@DEFSEVCERTCHAIN@"
+
 # Enable running QEMU VMM as a non-root user.
 # By default QEMU VMM run as root. When this is set to true, QEMU VMM process runs as
 # a non-root random user. See documentation for the limitations of this mode.
@@ -58,7 +80,7 @@ valid_hypervisor_paths = @QEMUVALIDHYPERVISORPATHS@
 # may stop the virtual machine from booting.
 # To see the list of default parameters, enable hypervisor debug, create a
 # container and look for 'default-kernel-parameters' log entries.
-kernel_params = "@KERNELPARAMS@"
+kernel_params = "@SEVKERNELPARAMS@"
 
 # Path to the firmware.
 # If you want that qemu uses the default firmware leave this option empty

src/runtime/config/configuration-qemu-snp.toml.in

@@ -1,5 +1,6 @@
 # Copyright (c) 2017-2019 Intel Corporation
 # Copyright (c) 2021 Adobe Inc.
+# Copyright 2022 Advanced Micro Devices, Inc.
 # Copyright (c) 2023 IBM Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
@@ -16,7 +17,7 @@
 path = "@QEMUSNPPATH@"
 kernel = "@KERNELSNPPATH@"
 #image = "@IMAGEPATH@"
-initrd = "@INITRDPATH@"
+initrd = "@INITRDSEVPATH@"
 machine_type = "@MACHINETYPE@"
 
 # rootfs filesystem type:
@@ -42,7 +43,13 @@ rootfs_type=@DEFROOTFSTYPE@
 confidential_guest = true
 
 # enable SEV SNP VMs
-sev_snp_guest = true
+sev_snp_guest = @DEFSNPGUEST@
+
+# SNP guest policy
+# Based on SEV Secure Nested Paging Firmware ABI Specification section 4.3
+# If it is unspecified or 0, it will default to 0x30000 (i.e. Bit#17 is '1' which is reserved and Bit#16 is '1' which means SMT is allowed).
+snp_guest_policy=@DEFSNPGUESTPOLICY@
+
 
 # Enable running QEMU VMM as a non-root user.
 # By default QEMU VMM run as root. When this is set to true, QEMU VMM process runs as
@@ -70,7 +77,8 @@ valid_hypervisor_paths = @QEMUSNPVALIDHYPERVISORPATHS@
 # may stop the virtual machine from booting.
 # To see the list of default parameters, enable hypervisor debug, create a
 # container and look for 'default-kernel-parameters' log entries.
-kernel_params = "@KERNELPARAMS@"
+
+kernel_params = "@SNPKERNELPARAMS@"
 
 # Path to the firmware.
 # If you want that qemu uses the default firmware leave this option empty
@@ -80,7 +88,7 @@ firmware = "@FIRMWARESNPPATH@"
 # firmware TDVF or OVMF can be split into FIRMWARE_VARS.fd (UEFI variables
 # as configuration) and FIRMWARE_CODE.fd (UEFI program image). UEFI variables
 # can be customized per each user while UEFI code is kept same.
-firmware_volume = "@FIRMWARETDVFVOLUMEPATH@"
+firmware_volume = "@FIRMWAREVOLUMEPATH@"
 
 # Machine accelerators
 # comma-separated list of machine accelerators to pass to the hypervisor.
@@ -333,8 +341,8 @@ pflashes = []
 # Disable the customizations done in the runtime when it detects
 # that it is running on top a VMM. This will result in the runtime
 # behaving as it would when running on bare metal.
-#
-#disable_nesting_checks = true
+# Needed for nested-SNP on hyper-V
+disable_nesting_checks = true
 
 # This is the msize used for 9p shares. It is the number of bytes
 # used for 9p packet payload.
@@ -454,7 +462,6 @@ disable_selinux=@DEFDISABLESELINUX@
 # (default: true)
 disable_guest_selinux=@DEFDISABLEGUESTSELINUX@
 
-
 [factory]
 # VM templating support. Once enabled, new VMs are created from template
 # using vm cloning. They will share the same initial kernel, initramfs and
@@ -674,7 +681,7 @@ experimental=@DEFAULTEXPFEATURES@
 #
 # Offload the CRI image management service to the Kata agent.
 # (default: false)
-#service_offload = true
+service_offload = @DEFSERVICEOFFLOAD@
 
 # Container image decryption keys provisioning.
 # Applies only if service_offload is true.
@@ -692,4 +699,4 @@ experimental=@DEFAULTEXPFEATURES@
 #
 # Keys can be remotely provisioned. The Kata agent fetches them from e.g.
 # a HTTPS URL:
-#provision=https://my-key-broker.foo/tenant/<tenant-id>
+#provision=https://my-key-broker.foo/tenant/<tenant-id>

src/runtime/config/configuration-qemu-tdx.toml.in

@@ -14,7 +14,7 @@
 [hypervisor.qemu]
 path = "@QEMUTDXPATH@"
 kernel = "@KERNELTDXPATH@"
-image = "@IMAGEPATH@"
+image = "@IMAGETDXPATH@"
 # initrd = "@INITRDPATH@"
 machine_type = "@MACHINETYPE@"
 
@@ -450,7 +450,6 @@ disable_selinux=@DEFDISABLESELINUX@
 # (default: true)
 disable_guest_selinux=@DEFDISABLEGUESTSELINUX@
 
-
 [factory]
 # VM templating support. Once enabled, new VMs are created from template
 # using vm cloning. They will share the same initial kernel, initramfs and
@@ -670,7 +669,7 @@ experimental=@DEFAULTEXPFEATURES@
 #
 # Offload the CRI image management service to the Kata agent.
 # (default: false)
-#service_offload = true
+service_offload = true
 
 # Container image decryption keys provisioning.
 # Applies only if service_offload is true.

src/runtime/config/configuration-qemu.toml.in

@@ -40,12 +40,40 @@ rootfs_type=@DEFROOTFSTYPE@
 # Default false
 # confidential_guest = true
 
+# Enable pre-attestation for AMD SEV and SEV-ES guests.
+# Applies only if confidential_guest is true.
+# (default: false)
+#guest_pre_attestation = true
+#
+# A remote key broker server (KBS) is required to validate the attestation
+# measurement and inject a secret key.  
+# See https://github.com/confidential-containers/simple-kbs
+#guest_pre_attestation_kbs_uri = "@DEFGUESTPREATTESTATIONPROXY@"
+#
+# The simple-kbs can be run in "online" and "offline" modes
+#guest_pre_attestation_kbs_mode="@DEFGUESTPREATTESTATIONMODE@" 
+#
+# Keyset ID of the injected secret (offline kbs)
+#guest_pre_attestation_keyset = "@DEFGUESTPREATTESTATIONKEYSET@"
+#
+# SEV guest policy
+#sev_guest_policy = @DEFSEVGUESTPOLICY@
+
+# SEV certificate chain path 
+#sev_cert_chain="@DEFSEVCERTCHAIN@"
+
 # Choose AMD SEV-SNP confidential guests
 # In case of using confidential guests on AMD hardware that supports both SEV
 # and SEV-SNP, the following enables SEV-SNP guests. SEV guests are default.
 # Default false
 # sev_snp_guest = true
 
+# SNP guest policy
+# Refer SEV Secure Nested Paging Firmware ABI Specification (sectin 4.3) to
+# change this value.
+# unspecified or == 0  -->  0x30000 i.e. Bit#17 is '1' and Bit#16 is '1' (SMT is allowed)
+# snp_guest_policy=0x30000
+
 # Enable running QEMU VMM as a non-root user.
 # By default QEMU VMM run as root. When this is set to true, QEMU VMM process runs as
 # a non-root random user. See documentation for the limitations of this mode.
@@ -688,7 +716,7 @@ experimental=@DEFAULTEXPFEATURES@
 #
 # Offload the CRI image management service to the Kata agent.
 # (default: false)
-#service_offload = true
+service_offload = @DEFSERVICEOFFLOAD@
 
 # Container image decryption keys provisioning.
 # Applies only if service_offload is true.

src/runtime/config/configuration-remote.toml.in

@@ -0,0 +1,318 @@
+# Copyright (c) 2017-2019 Intel Corporation
+# Copyright (c) 2023 IBM Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# XXX: WARNING: this file is auto-generated.
+# XXX:
+# XXX: Source file: "@CONFIG_REMOTE_IN@"
+# XXX: Project:
+# XXX:   Name: @PROJECT_NAME@
+# XXX:   Type: @PROJECT_TYPE@
+
+
+[hypervisor.remote]
+remote_hypervisor_socket = "/run/peerpod/hypervisor.sock"
+remote_hypervisor_timeout = 600
+
+
+# Enable confidential guest support.
+# Toggling that setting may trigger different hardware features, ranging
+# from memory encryption to both memory and CPU-state encryption and integrity.
+# The Kata Containers runtime dynamically detects the available feature set and
+# aims at enabling the largest possible one, returning an error if none is
+# available, or none is supported by the hypervisor.
+#
+# Known limitations:
+# * Does not work by design:
+#   - CPU Hotplug 
+#   - Memory Hotplug
+#   - NVDIMM devices
+#
+# Default false
+# confidential_guest = true
+
+
+# List of valid annotation names for the hypervisor
+# Each member of the list is a regular expression, which is the base name
+# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
+# Note: Remote hypervisor is only handling the following annotations
+enable_annotations = ["machine_type", "default_memory", "default_vcpus"]
+
+# Optional space-separated list of options to pass to the guest kernel.
+# For example, use `kernel_params = "vsyscall=emulate"` if you are having
+# trouble running pre-2.15 glibc.
+#
+# WARNING: - any parameter specified here will take priority over the default
+# parameter value of the same name used to start the virtual machine.
+# Do not set values here unless you understand the impact of doing so as you
+# may stop the virtual machine from booting.
+# To see the list of default parameters, enable hypervisor debug, create a
+# container and look for 'default-kernel-parameters' log entries.
+# NOTE: kernel_params are not currently passed over in remote hypervisor
+# kernel_params = ""
+
+# Path to the firmware.
+# If you want that qemu uses the default firmware leave this option empty
+firmware = "@FIRMWAREPATH@"
+
+# Default number of vCPUs per SB/VM:
+# unspecified or 0                --> will be set to @DEFVCPUS@
+# < 0                             --> will be set to the actual number of physical cores
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores
+# default_vcpus = 1
+
+# Default maximum number of vCPUs per SB/VM:
+# unspecified or == 0             --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when
+# the actual number of physical cores is greater than it.
+# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU
+# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs
+# can be added to a SB/VM, but the memory footprint will be big. Another example, with
+# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
+# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
+# unless you know what are you doing.
+# NOTICE: on arm platform with gicv2 interrupt controller, set it to 8.
+# default_maxvcpus = @DEFMAXVCPUS@
+
+# Bridges can be used to hot plug devices.
+# Limitations:
+# * Currently only pci bridges are supported
+# * Until 30 devices per bridge can be hot plugged.
+# * Until 5 PCI bridges can be cold plugged per VM.
+#   This limitation could be a bug in qemu or in the kernel
+# Default number of bridges per SB/VM:
+# unspecified or 0   --> will be set to @DEFBRIDGES@
+# > 1 <= 5           --> will be set to the specified number
+# > 5                --> will be set to 5
+default_bridges = @DEFBRIDGES@
+
+# Default memory size in MiB for SB/VM.
+# If unspecified then it will be set @DEFMEMSZ@ MiB.
+# Note: the remote hypervisor uses the peer pod config to determine the memory of the VM
+# default_memory = @DEFMEMSZ@
+#
+# Default memory slots per SB/VM.
+# If unspecified then it will be set @DEFMEMSLOTS@.
+# This is will determine the times that memory will be hotadded to sandbox/VM.
+# Note: the remote hypervisor uses the peer pod config to determine the memory of the VM
+#memory_slots = @DEFMEMSLOTS@
+
+# This option changes the default hypervisor and kernel parameters
+# to enable debug output where available. And Debug also enable the hmp socket.
+#
+# Default false
+#enable_debug = true
+
+# Path to OCI hook binaries in the *guest rootfs*.
+# This does not affect host-side hooks which must instead be added to
+# the OCI spec passed to the runtime.
+#
+# You can create a rootfs with hooks by customizing the osbuilder scripts:
+# https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder
+#
+# Hooks must be stored in a subdirectory of guest_hook_path according to their
+# hook type, i.e. "guest_hook_path/{prestart,poststart,poststop}".
+# The agent will scan these directories for executable files and add them, in
+# lexicographical order, to the lifecycle of the guest container.
+# Hooks are executed in the runtime namespace of the guest. See the official documentation:
+# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
+# Warnings will be logged if any error is encountered while scanning for hooks,
+# but it will not abort container execution.
+#guest_hook_path = "/usr/share/oci/hooks"
+
+# disable applying SELinux on the VMM process (default false)
+disable_selinux=@DEFDISABLESELINUX@
+
+# disable applying SELinux on the container process
+# If set to false, the type `container_t` is applied to the container process by default.
+# Note: To enable guest SELinux, the guest rootfs must be CentOS that is created and built
+# with `SELINUX=yes`.
+# (default: true)
+# Note: The remote hypervisor has a different guest, so currently requires this to be disabled
+disable_guest_selinux = true
+
+[agent.@PROJECT_TYPE@]
+# If enabled, make the agent display debug-level messages.
+# (default: disabled)
+#enable_debug = true
+
+# Enable agent tracing.
+#
+# If enabled, the agent will generate OpenTelemetry trace spans.
+#
+# Notes:
+#
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
+#
+# (default: disabled)
+#enable_tracing = true
+
+# Enable debug console.
+
+# If enabled, user can connect guest OS running inside hypervisor
+# through "kata-runtime exec <sandbox-id>" command
+
+#debug_console_enabled = true
+
+# Agent connection dialing timeout value in seconds
+# (default: 30)
+#dial_timeout = 30
+
+[runtime]
+# If enabled, the runtime will log additional debug messages to the
+# system log
+# (default: disabled)
+#enable_debug = true
+#
+# Internetworking model
+# Determines how the VM should be connected to the
+# the container network interface
+# Options:
+#
+#   - macvtap
+#     Used when the Container network interface can be bridged using
+#     macvtap.
+#
+#   - none
+#     Used when customize network. Only creates a tap device. No veth pair.
+#
+#   - tcfilter
+#     Uses tc filter rules to redirect traffic from the network interface
+#     provided by plugin to a tap interface connected to the VM.
+#
+# Note: The remote hypervisor, uses it's own network, so "none" is required
+internetworking_model="none"
+
+# disable guest seccomp
+# Determines whether container seccomp profiles are passed to the virtual
+# machine and applied by the kata agent. If set to true, seccomp is not applied
+# within the guest
+# (default: true)
+# Note: The remote hypervisor has a different guest, so currently requires this to be set to true
+disable_guest_seccomp=true
+
+
+# Apply a custom SELinux security policy to the container process inside the VM.
+# This is used when you want to apply a type other than the default `container_t`,
+# so general users should not uncomment and apply it.
+# (format: "user:role:type")
+# Note: You cannot specify MCS policy with the label because the sensitivity levels and
+# categories are determined automatically by high-level container runtimes such as containerd.
+#guest_selinux_label="@DEFGUESTSELINUXLABEL@"
+
+# If enabled, the runtime will create opentracing.io traces and spans.
+# (See https://www.jaegertracing.io/docs/getting-started).
+# (default: disabled)
+#enable_tracing = true
+
+# Set the full url to the Jaeger HTTP Thrift collector.
+# The default if not set will be "http://localhost:14268/api/traces"
+#jaeger_endpoint = ""
+
+# Sets the username to be used if basic auth is required for Jaeger.
+#jaeger_user = ""
+
+# Sets the password to be used if basic auth is required for Jaeger.
+#jaeger_password = ""
+
+# If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
+# This option may have some potential impacts to your host. It should only be used when you know what you're doing.
+# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only
+# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
+# (like OVS) directly.
+# (default: false)
+# Note: The remote hypervisor has a different networking model, which requires true 
+disable_new_netns = true
+
+# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
+# The container cgroups in the host are not created, just one single cgroup per sandbox.
+# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
+# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
+# The sandbox cgroup is constrained if there is no container type annotation.
+# See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType
+sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
+
+# If enabled, the runtime will attempt to determine appropriate sandbox size (memory, CPU) before booting the virtual machine. In
+# this case, the runtime will not dynamically update the amount of memory and CPU in the virtual machine. This is generally helpful
+# when a hardware architecture or hypervisor solutions is utilized which does not support CPU and/or memory hotplug.
+# Compatibility for determining appropriate sandbox (VM) size:
+# - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O
+#   does not yet support sandbox sizing annotations.
+# - When running single containers using a tool like ctr, container sizing information will be available.
+# Note: the remote hypervisor uses the peer pod config to determine the sandbox size, so requires this to be set to true
+static_sandbox_resource_mgmt=true
+
+# VFIO Mode
+# Determines how VFIO devices should be be presented to the container.
+# Options:
+#
+#  - vfio
+#    Matches behaviour of OCI runtimes (e.g. runc) as much as
+#    possible.  VFIO devices will appear in the container as VFIO
+#    character devices under /dev/vfio.  The exact names may differ
+#    from the host (they need to match the VM's IOMMU group numbers
+#    rather than the host's)
+#
+#  - guest-kernel
+#    This is a Kata-specific behaviour that's useful in certain cases.
+#    The VFIO device is managed by whatever driver in the VM kernel
+#    claims it.  This means it will appear as one or more device nodes
+#    or network interfaces depending on the nature of the device.
+#    Using this mode requires specially built workloads that know how
+#    to locate the relevant device interfaces within the VM.
+#
+vfio_mode="@DEFVFIOMODE@"
+
+# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
+# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
+# Note: remote hypervisor has no sharing of emptydir mounts from host to guest
+disable_guest_empty_dir=false
+
+# Enabled experimental feature list, format: ["a", "b"].
+# Experimental features are features not stable enough for production,
+# they may break compatibility, and are prepared for a big version bump.
+# Supported experimental features:
+# (default: [])
+experimental=@DEFAULTEXPFEATURES@
+
+# If enabled, user can run pprof tools with shim v2 process through kata-monitor.
+# (default: false)
+# enable_pprof = true
+
+# WARNING: All the options in the following section have not been implemented yet.
+# This section was added as a placeholder. DO NOT USE IT!
+[image]
+# Container image service.
+#
+# Offload the CRI image management service to the Kata agent.
+# (default: false)
+# Note: The remote hypervisor offloads the pulling on images on the peer pod VM, so requries this to be true
+service_offload = true
+
+# Container image decryption keys provisioning.
+# Applies only if service_offload is true.
+# Keys can be provisioned locally (e.g. through a special command or
+# a local file) or remotely (usually after the guest is remotely attested).
+# The provision setting is a complete URL that lets the Kata agent decide
+# which method to use in order to fetch the keys.
+#
+# Keys can be stored in a local file, in a measured and attested initrd:
+#provision=data:///local/key/file
+#
+# Keys could be fetched through a special command or binary from the
+# initrd (guest) image, e.g. a firmware call:
+#provision=file:///path/to/bin/fetcher/in/guest
+#
+# Keys can be remotely provisioned. The Kata agent fetches them from e.g.
+# a HTTPS URL:
+#provision=https://my-key-broker.foo/tenant/<tenant-id>

src/runtime/go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/BurntSushi/toml v1.2.0
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/blang/semver/v4 v4.0.0
+	github.com/container-orchestrated-devices/container-device-interface v0.6.0
 	github.com/containerd/cgroups v1.0.5-0.20220625035431-cf7417bca682
 	github.com/containerd/console v1.0.3
 	github.com/containerd/containerd v1.6.8
@@ -31,7 +32,7 @@ require (
 	github.com/intel-go/cpuid v0.0.0-20210602155658-5747e5cec0d9
 	github.com/mdlayher/vsock v1.1.0
 	github.com/opencontainers/runc v1.1.3
-	github.com/opencontainers/runtime-spec v1.0.3-0.20211214071223-8958f93039ab
+	github.com/opencontainers/runtime-spec v1.1.0-rc.1
 	github.com/opencontainers/selinux v1.10.1
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
 	github.com/pkg/errors v0.9.1
@@ -51,8 +52,9 @@ require (
 	go.opentelemetry.io/otel/sdk v1.3.0
 	go.opentelemetry.io/otel/trace v1.3.0
 	golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2
-	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f
+	golang.org/x/sys v0.1.0
 	google.golang.org/grpc v1.47.0
+	google.golang.org/protobuf v1.28.1
 	k8s.io/apimachinery v0.22.5
 	k8s.io/cri-api v0.23.1
 )
@@ -93,23 +95,27 @@ require (
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.3-0.20220114050600-8b9d41f48198 // indirect
+	github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rogpeppe/go-internal v1.8.1-0.20210923151022-86f73c517451 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
 	go.mongodb.org/mongo-driver v1.7.5 // indirect
 	go.opencensus.io v0.23.0 // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
 	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
 	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
 	golang.org/x/text v0.3.7 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
 replace (
+	github.com/containerd/containerd => github.com/confidential-containers/containerd v1.6.7-0.20221123142530-25f68aa818ec
 	github.com/opencontainers/image-spec => github.com/opencontainers/image-spec v1.0.2
 	github.com/opencontainers/runc => github.com/opencontainers/runc v1.1.3
 	github.com/uber-go/atomic => go.uber.org/atomic v1.5.1

src/runtime/go.sum

@@ -64,30 +64,23 @@ code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5/go.mod h1:v4VVB
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774/go.mod h1:6/0dYRLLXyJjbkIPeeGyoJ/eKOSI0eU6eTlCBYibgd0=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg=
-github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/azure-sdk-for-go v66.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
-github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
-github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=
 github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=
 github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc=
 github.com/Azure/go-autorest/autorest v0.11.27/go.mod h1:7l8ybrIdUmGqZMTD0sRtAr8NvbHjfofbf8RSP2q7w7U=
-github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg=
-github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
 github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
 github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ=
 github.com/Azure/go-autorest/autorest/azure/auth v0.5.11/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg=
 github.com/Azure/go-autorest/autorest/azure/cli v0.4.5/go.mod h1:ADQAXrkgm7acgWVUNamOgh8YNrv4p27l3Wc55oVfpzg=
 github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
-github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
 github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU=
 github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE=
 github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E=
-github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
 github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@@ -101,35 +94,18 @@ github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q
 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
 github.com/Masterminds/semver/v3 v3.0.3/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
-github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
-github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
-github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
-github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
-github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
-github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
-github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
-github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
-github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ=
-github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8=
-github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg=
-github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00=
-github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600=
 github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4=
-github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4=
 github.com/Microsoft/hcsshim v0.8.22/go.mod h1:91uVCVzvX2QD16sMCenoxxXo6L1wJnLMX2PSufFMtF0=
-github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg=
 github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc=
 github.com/Microsoft/hcsshim v0.9.3/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc=
 github.com/Microsoft/hcsshim v0.9.4 h1:mnUj0ivWy6UzbB1uLFqKR6F+ZyiDc7j4iGgHTpO+5+I=
 github.com/Microsoft/hcsshim v0.9.4/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc=
-github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU=
-github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
@@ -142,7 +118,6 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/ReneKroon/ttlcache/v2 v2.11.0/go.mod h1:mBxvsNY+BT8qLLd6CuAJubbKo6r0jh3nb5et22bbfGY=
-github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -150,7 +125,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
-github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
 github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
@@ -166,7 +140,6 @@ github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:o
 github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
 github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef h1:46PFijGLmAjMPwCCCo7Jf0W6f9slllCkkv7vyc1yOSg=
 github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
-github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
 github.com/aws/aws-sdk-go v1.44.44/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/aws/aws-sdk-go-v2 v1.16.5/go.mod h1:Wh7MEsmEApyL5hrWzpDkba4gwAPc5/piwLVLFnCxp48=
 github.com/aws/aws-sdk-go-v2/config v1.15.11/go.mod h1:mD5tNFciV7YHNjPpFYqJ6KGpoSfY107oZULvTHIxtbI=
@@ -182,30 +155,21 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.16.7/go.mod h1:lVxTdiiSHY3jb1aeg+BBF
 github.com/aws/smithy-go v1.11.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/beeker1121/goque v1.0.3-0.20191103205551-d618510128af/go.mod h1:84CWnaDz4g1tEVnFLnuBigmGK15oPohy0RfvSN8d4eg=
 github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
-github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
 github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
-github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
-github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
-github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
 github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE=
-github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
 github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
-github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
-github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50=
-github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
 github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs=
 github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs=
 github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
@@ -223,9 +187,6 @@ github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAc
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg=
-github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc=
-github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs=
 github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
 github.com/cilium/ebpf v0.7.0 h1:1k/q3ATgxSXRdrmPfH8d7YK0GfqVsEKZAX9dQZvs56k=
 github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
@@ -242,127 +203,54 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
-github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
 github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
 github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4=
+github.com/confidential-containers/containerd v1.6.7-0.20221123142530-25f68aa818ec h1:eFjq9ReTZ77/j29LoI68rW+utQ/oglOS9liDQAO8aCI=
+github.com/confidential-containers/containerd v1.6.7-0.20221123142530-25f68aa818ec/go.mod h1:By6p5KqPK0/7/CgO/A6t/Gz+CUYUu2zf1hUaaymVXB0=
 github.com/container-orchestrated-devices/container-device-interface v0.4.0/go.mod h1:E1zcucIkq9P3eyNmY+68dBQsTcsXJh9cgRo2IVNScKQ=
-github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE=
-github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU=
-github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=
+github.com/container-orchestrated-devices/container-device-interface v0.6.0 h1:aWwcz/Ep0Fd7ZuBjQGjU/jdPloM7ydhMW13h85jZNvk=
+github.com/container-orchestrated-devices/container-device-interface v0.6.0/go.mod h1:OQlgtJtDrOxSQ1BWODC8OZK1tzi9W69wek+Jy17ndzo=
 github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=
-github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E=
-github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss=
 github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss=
-github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI=
-github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko=
-github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM=
-github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo=
-github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo=
-github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE=
 github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU=
 github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8=
 github.com/containerd/cgroups v1.0.5-0.20220625035431-cf7417bca682 h1:d/YjAAP6A6fT0vpMhbYSDkE+K1ww/DZodOIamD8Pr/E=
 github.com/containerd/cgroups v1.0.5-0.20220625035431-cf7417bca682/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA=
-github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
-github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
-github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE=
 github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw=
 github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ=
 github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw=
 github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
-github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ=
-github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU=
-github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI=
-github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s=
-github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g=
-github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c=
-github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s=
-github.com/containerd/containerd v1.5.9/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ=
-github.com/containerd/containerd v1.6.1/go.mod h1:1nJz5xCZPusx6jJU8Frfct988y0NpumIq9ODB0kLtoE=
-github.com/containerd/containerd v1.6.6/go.mod h1:ZoP1geJldzCVY3Tonoz7b1IXk8rIX0Nltt5QE4OMNk0=
-github.com/containerd/containerd v1.6.8 h1:h4dOFDwzHmqFEP754PgfgTeVXFnLiRc6kiqC7tplDJs=
-github.com/containerd/containerd v1.6.8/go.mod h1:By6p5KqPK0/7/CgO/A6t/Gz+CUYUu2zf1hUaaymVXB0=
-github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
-github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
-github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
-github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo=
-github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y=
 github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ=
 github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM=
 github.com/containerd/continuity v0.2.2 h1:QSqfxcn8c+12slxwu00AtzXrsami0MJb/MQs9lOLHLA=
 github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk=
 github.com/containerd/cri-containerd v1.19.0 h1:PcTvvl+SHaekCMQZFQkYjn1RKlYrK6khYbuhOeF68k0=
 github.com/containerd/cri-containerd v1.19.0/go.mod h1:wxbGdReWGCalzGOEpifoHeYCK4xAgnj4o/4bVB+9voU=
-github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
-github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
-github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0=
-github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0=
-github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4=
 github.com/containerd/fifo v1.0.0 h1:6PirWBr9/L7GDamKr+XM0IeUFXu5mf3M/BPpH9gaLBU=
 github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4=
-github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU=
-github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk=
-github.com/containerd/go-cni v1.1.0/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA=
 github.com/containerd/go-cni v1.1.3/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA=
 github.com/containerd/go-cni v1.1.6/go.mod h1:BWtoWl5ghVymxu6MBjg79W9NZrCRyHIdUtk4cauMe34=
-github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
-github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
-github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g=
-github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok=
 github.com/containerd/go-runc v1.0.0 h1:oU+lLv1ULm5taqgV/CJivypVODI4SUz1znWjv3nNYS0=
 github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok=
-github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0=
-github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA=
-github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow=
-github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms=
-github.com/containerd/imgcrypt v1.1.3/go.mod h1:/TPA1GIDXMzbj01yd8pIbQiLdQxed5ue1wb8bP7PQu4=
 github.com/containerd/imgcrypt v1.1.4/go.mod h1:LorQnPtzL/T0IyCeftcsMEO7AqxUDbdO8j/tSUpgxvo=
-github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c=
-github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
 github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
 github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM=
 github.com/containerd/stargz-snapshotter/estargz v0.9.0/go.mod h1:aE5PCyhFMwR8sbrErO5eM2GcvkyXTTJremG883D4qF0=
 github.com/containerd/stargz-snapshotter/estargz v0.10.1/go.mod h1:aE5PCyhFMwR8sbrErO5eM2GcvkyXTTJremG883D4qF0=
 github.com/containerd/stargz-snapshotter/estargz v0.11.4/go.mod h1:7vRJIcImfY8bpifnMjt+HTJoQxASq7T28MYbP15/Nf0=
 github.com/containerd/stargz-snapshotter/estargz v0.12.0/go.mod h1:AIQ59TewBFJ4GOPEQXujcrJ/EKxh5xXZegW1rkR1P/M=
-github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
-github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
-github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8=
-github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y=
 github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y=
 github.com/containerd/ttrpc v1.1.0 h1:GbtyLRxb0gOLR0TYQWt3O6B0NvT8tMdorEHqIQo/lWI=
 github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ=
-github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc=
-github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk=
-github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg=
 github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY=
 github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s=
-github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw=
-github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y=
-github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY=
-github.com/containerd/zfs v0.0.0-20210324211415-d5c4544f0433/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY=
 github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY=
-github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
-github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
-github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
 github.com/containernetworking/cni v1.0.1/go.mod h1:AKuhXbN5EzmD4yTNtfSsX3tPcmtrBI6QcRV0NiNt15Y=
 github.com/containernetworking/cni v1.1.1/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw=
 github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl31EQbXALQ=
 github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw=
-github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM=
-github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8=
-github.com/containernetworking/plugins v1.0.1/go.mod h1:QHCfGpaTwYTbbH+nZXKVTxNBDZcxSOplJT5ico8/FLE=
 github.com/containernetworking/plugins v1.1.1 h1:+AGfFigZ5TiQH00vhR8qPeSatj53eNGz0C1d3wVYlHE=
 github.com/containernetworking/plugins v1.1.1/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19sZPp3ry5uHSkI4LPxV8=
 github.com/containers/buildah v1.27.0/go.mod h1:anH3ExvDXRNP9zLQCrOc1vWb5CrhqLF/aYFim4tslvA=
@@ -370,10 +258,6 @@ github.com/containers/common v0.49.1/go.mod h1:ueM5hT0itKqCQvVJDs+EtjornAQtrHYxQ
 github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I=
 github.com/containers/image/v5 v5.22.0/go.mod h1:D8Ksv2RNB8qLJ7xe1P3rgJJOSQpahA6amv2Ax++/YO4=
 github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY=
-github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc=
-github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4=
-github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY=
-github.com/containers/ocicrypt v1.1.2/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY=
 github.com/containers/ocicrypt v1.1.3/go.mod h1:xpdkbVAuaH3WzbEabUd5yDsl9SwJA5pABH85425Es2g=
 github.com/containers/ocicrypt v1.1.5/go.mod h1:WgjxPWdTJMqYMjf3M6cuIFFA1/MpyyhIM99YInA+Rvc=
 github.com/containers/podman/v4 v4.2.0 h1:mqQ0CtdSOTfsl6IEcO0UiA/Yi/9Yxoe/mSEC4h21CK8=
@@ -385,22 +269,16 @@ github.com/containers/storage v1.42.0/go.mod h1:JiUJwOgOo1dr2DdOUc1MRe2GCAXABYoY
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
-github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
 github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
 github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
 github.com/coreos/go-oidc/v3 v3.2.0/go.mod h1:rEJ/idjfUyfkBit1eI1fvyr+64/g9dcKpAm8MJMesvo=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
 github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
 github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/coreos/stream-metadata-go v0.0.0-20210225230131-70edb9eb47b3/go.mod h1:RTjQyHgO/G37oJ3qnqYK6Z4TPZ5EsaabOtfMjVXmgko=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
@@ -416,13 +294,10 @@ github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxG
 github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
 github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
 github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8=
-github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I=
 github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0=
-github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
@@ -430,11 +305,8 @@ github.com/digitalocean/go-libvirt v0.0.0-20201209184759-e2a69bcd5bd1/go.mod h1:
 github.com/digitalocean/go-qemu v0.0.0-20210326154740-ac9e0b687001/go.mod h1:IetBE52JfFxK46p2n2Rqm+p5Gx1gpu2hRHsrbnPOWZQ=
 github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
 github.com/disiqueira/gotree/v3 v3.0.2/go.mod h1:ZuyjE4+mUQZlbpkI24AmruZKhg3VHEgPLDY8Qk+uUu8=
-github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
 github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/cli v20.10.16+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
-github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY=
-github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
@@ -445,22 +317,17 @@ github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avu
 github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11/go.mod h1:a6bNUGTbQBsY6VRHTr4h/rkOXjl244DyRD0tx3fgq4Q=
-github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
 github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8=
 github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
-github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI=
 github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
 github.com/docker/go-plugins-helpers v0.0.0-20211224144127-6eecb7beb651/go.mod h1:LFyLie6XcDbyKGeVK6bHe+9aJTYCxWLBg5IrJZOaXKA=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/libnetwork v0.8.0-dev.2.0.20190625141545-5a177b73e316/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8=
-github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
 github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
-github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dtylman/scp v0.0.0-20181017070807-f3000a34aef4/go.mod h1:jN1ZaUPSNA8jm10nmaRLky84qV/iCeiHmcEf3EbP+dc=
-github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eggsampler/acme/v3 v3.2.1/go.mod h1:/qh0rKC/Dh7Jj+p4So7DbWmFNzC4dpcpK53r226Fhuo=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
@@ -476,7 +343,6 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m
 github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
 github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch/v5 v5.5.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
 github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA=
@@ -507,8 +373,6 @@ github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwV
 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
 github.com/fsouza/go-dockerclient v1.7.7/go.mod h1:njNCXvoZj3sLPjf3yO0DPHf1mdLdCPDYPc14GskKA4Y=
 github.com/fsouza/go-dockerclient v1.8.1/go.mod h1:zmA2ogSxRnXmbZcy0Aq7yhRoCdP/bDns/qghCK9SWtM=
-github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA=
-github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
 github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -521,7 +385,6 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gorp/gorp/v3 v3.0.2/go.mod h1:BJ3q1ejpV8cVALtcXvXaXyTOlMmJhWDxTmncaR6rwBY=
-github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-ini/ini v1.28.2 h1:drmmYv7psRpoGZkPtPKKTB+ZFSnvmwCMfNj5o1nLh2Y=
 github.com/go-ini/ini v1.28.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -685,9 +548,6 @@ github.com/gobuffalo/tags/v3 v3.0.2/go.mod h1:ZQeN6TCTiwAFnS0dNcbDtSgZDwNKSpqajv
 github.com/gobuffalo/tags/v3 v3.1.0/go.mod h1:ZQeN6TCTiwAFnS0dNcbDtSgZDwNKSpqajvVtt6mlYpA=
 github.com/gobuffalo/validate/v3 v3.0.0/go.mod h1:HFpjq+AIiA2RHoQnQVTFKF/ZpUPXwyw82LgyDPxQ9r0=
 github.com/gobuffalo/validate/v3 v3.1.0/go.mod h1:HFpjq+AIiA2RHoQnQVTFKF/ZpUPXwyw82LgyDPxQ9r0=
-github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
-github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
-github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -695,19 +555,15 @@ github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
 github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
-github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU=
 github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
-github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -813,26 +669,20 @@ github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97Dwqy
 github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
 github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q=
-github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/schema v1.2.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU=
-github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
-github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -847,7 +697,6 @@ github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjh
 github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-kms-wrapping/entropy v0.1.0/go.mod h1:d1g9WGtAunDNpek8jUIEJnBlbgKS1N2Q61QkHiZyR1g=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
-github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
@@ -894,9 +743,6 @@ github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714/go.mod h1:2Go
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
-github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
@@ -904,8 +750,6 @@ github.com/insomniacslk/dhcp v0.0.0-20220119180841-3c283ff8b7dd/go.mod h1:h+MxyH
 github.com/intel-go/cpuid v0.0.0-20210602155658-5747e5cec0d9 h1:x9HFDMDCsaxTvC4X3o0ZN6mw99dT/wYnTItGwhBRmg0=
 github.com/intel-go/cpuid v0.0.0-20210602155658-5747e5cec0d9/go.mod h1:RmeVYf9XrPRbRc3XIx0gLYA8qOFvNoPOfaEZduRlEp4=
 github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ=
-github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
-github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw=
 github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
 github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
 github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
@@ -940,8 +784,6 @@ github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=
 github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
-github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
-github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548/go.mod h1:hGT6jSUVzF6no3QaDSMLGLEHtHSBSefs+MgcDWnmhmo=
@@ -980,7 +822,6 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
@@ -1035,7 +876,6 @@ github.com/markbates/errx v1.1.0/go.mod h1:PLa46Oex9KNbVDZhKel8v1OT7hD5JZ2eI7AHh
 github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
 github.com/markbates/oncer v1.0.0/go.mod h1:Z59JA581E9GP6w96jai+TGqafHPW+cPfRxz2aSZ0mcI=
 github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
-github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
@@ -1045,7 +885,6 @@ github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope
 github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
@@ -1054,10 +893,8 @@ github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcME
 github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
-github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
 github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
 github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
 github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
@@ -1084,7 +921,6 @@ github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N
 github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
 github.com/miekg/dns v1.1.45/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/miekg/pkcs11 v1.0.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
-github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
@@ -1105,7 +941,6 @@ github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
 github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs=
@@ -1120,9 +955,7 @@ github.com/moby/sys/mountinfo v0.6.1/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdx
 github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78=
 github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
 github.com/moby/sys/signal v0.6.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg=
-github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ=
 github.com/moby/sys/symlink v0.2.0/go.mod h1:7uZVF2dqJjG/NsClqul95CqKOBRQyYSNnJ6BMgR/gFs=
-github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
 github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A=
 github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw=
 github.com/moby/vpnkit v0.5.0/go.mod h1:KyjUrL9cb6ZSNNAUwZfqRjhwwgJ3BJN+kXh0t43WTUQ=
@@ -1140,7 +973,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
-github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
 github.com/networkplumbing/go-nft v0.2.0/go.mod h1:HnnM+tYvlGAsMU7yoYwXEVLLiDW9gdMmb5HoGcwpuQs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
@@ -1150,16 +982,11 @@ github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQ
 github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
 github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
-github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0=
@@ -1171,15 +998,12 @@ github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3
 github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
 github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY=
 github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU=
-github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
 github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
 github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
@@ -1188,31 +1012,24 @@ github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9
 github.com/onsi/gomega v1.20.0 h1:8W0cWlwFkflGPLltQvLRB7ZVD5HuP6ng320w2IS245Q=
 github.com/onsi/gomega v1.20.0/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
-github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
-github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
-github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
-github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
 github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/opencontainers/runc v1.1.3 h1:vIXrkId+0/J2Ymu2m7VjGvbSlAId9XNRPhn2p4b+d8w=
 github.com/opencontainers/runc v1.1.3/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg=
-github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.3-0.20201121164853-7413a7f753e1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-spec v1.0.3-0.20211214071223-8958f93039ab h1:YQZXa3elcHgKXAa2GjVFC9M3JeP7ZPyFD1YByDx/dgQ=
 github.com/opencontainers/runtime-spec v1.0.3-0.20211214071223-8958f93039ab/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
+github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.1.0-rc.1 h1:wHa9jroFfKGQqFHj0I1fMRKLl0pfj+ynAqBxo3v6u9w=
+github.com/opencontainers/runtime-spec v1.1.0-rc.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-tools v0.0.0-20190417131837-cd1349b7c47e/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
 github.com/opencontainers/runtime-tools v0.9.1-0.20220714195903-17b3287fafb7/go.mod h1:/tgP02fPXGHkU3/qKK1Y0Db4yqNyGm03vLq/mzHzcS4=
-github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE=
-github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo=
-github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=
+github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 h1:DmNGcqH3WDbV5k8OJ+esPWbqUOX5rMLR2PMvziDMJi0=
+github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626/go.mod h1:BRHJJd0E+cx42OybVYSgUvZmU0B8P9gZuRXlZUP7TKI=
 github.com/opencontainers/selinux v1.8.5/go.mod h1:HTvjPFoGMbpQsG886e3lQwnsRWtE4TC1OF3OUvG9FAo=
 github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
 github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
@@ -1229,7 +1046,6 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
-github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
@@ -1237,7 +1053,6 @@ github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -1248,7 +1063,6 @@ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndr
 github.com/poy/onpar v0.0.0-20190519213022-ee068f8ea4d1/go.mod h1:nSbFQvMj97ZyhFRSJYtut+msi4sOY6zJDGCdSc+/rZU=
 github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
 github.com/proglottis/gpgme v0.1.3/go.mod h1:fPbW/EZ0LvwQtH8Hy7eixhp1eF3G39dtx7GUN+0Gmy0=
-github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
@@ -1259,13 +1073,11 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr
 github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
 github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
-github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
@@ -1276,16 +1088,13 @@ github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9
 github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
 github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4=
 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
-github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
@@ -1311,7 +1120,6 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
-github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
 github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1 h1:ZFfeKAhIQiiOrQaI3/znw0gOmYpO28Tcu1YaqMa/jtQ=
 github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
@@ -1331,8 +1139,6 @@ github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNX
 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sigstore/sigstore v1.3.1-0.20220629021053-b95fc0d626c1/go.mod h1:y83NePRM98MJpbGgBgi54UZduhG0aD7lYngAVCx+i/E=
-github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
-github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
@@ -1346,7 +1152,6 @@ github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs
 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
@@ -1372,7 +1177,6 @@ github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb6
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
@@ -1380,13 +1184,11 @@ github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5q
 github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns=
 github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8=
 github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
-github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -1399,8 +1201,7 @@ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PK
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/sylabs/sif/v2 v2.7.1/go.mod h1:bBse2nEFd3yHkmq6KmAOFEWQg5LdFYiQUdVcgamxlc8=
-github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
-github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
+github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc=
@@ -1411,7 +1212,6 @@ github.com/theupdateframework/go-tuf v0.3.1/go.mod h1:lhHZ3Vt2pdAh15h0Cc6gWdlI+O
 github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399/go.mod h1:LdwHTNJT99C5fTAzDz0ud328OgXz+gierycbcIx2fRs=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
@@ -1422,9 +1222,7 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
-github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
-github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli v1.22.4 h1:u7tSpNPPswAFymm8IehJhy4uJMlUuU/GmqSkvJ1InXA=
@@ -1435,14 +1233,12 @@ github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPU
 github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
 github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI=
 github.com/vbauerster/mpb/v7 v7.4.2/go.mod h1:UmOiIUI8aPqWXIps0ciik3RKMdzx7+ooQpq+fBcXwBA=
-github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
 github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
 github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
 github.com/vishvananda/netlink v1.1.1-0.20220115184804-dd687eb2f2d4/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
 github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs=
 github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
-github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=
@@ -1451,17 +1247,17 @@ github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+
 github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
 github.com/weppos/publicsuffix-go v0.15.1-0.20210807195340-dc689ff0bb59/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE=
 github.com/weppos/publicsuffix-go v0.15.1-0.20220329081811-9a40b608a236/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE=
-github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
-github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
 github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
 github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
-github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
+github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
@@ -1477,9 +1273,6 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
-github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA=
-github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
 github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
 github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
 github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE=
@@ -1489,10 +1282,7 @@ github.com/zmap/zlint/v3 v3.3.1-0.20211019173530-cb17369b4628/go.mod h1:O+4OXRfN
 gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20220601114329-47893b162965 h1:EXE1ZsUqiUWGV5Dw2oTYpXx24ffxj0//yhTB0Ppv+4s=
 gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20220601114329-47893b162965/go.mod h1:TBB3sR7/jg4RCThC/cgT4fB8mAbbMO307TycfgeR59w=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
-go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg=
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
@@ -1563,9 +1353,7 @@ go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
 goji.io/v3 v3.0.0/go.mod h1:c02FFnNiVNCDo+DpR2IhBQpM9r5G1BG/MkHNTPUJ13U=
-golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -1576,14 +1364,12 @@ golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
@@ -1629,13 +1415,13 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o=
 golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
 golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1650,13 +1436,11 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -1676,7 +1460,6 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
@@ -1770,10 +1553,7 @@ golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606122018-79a91cf218c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1782,15 +1562,12 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1802,7 +1579,6 @@ golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1816,12 +1592,10 @@ golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1831,11 +1605,8 @@ golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1853,7 +1624,6 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1893,8 +1663,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1911,12 +1681,10 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -2007,7 +1775,6 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
-google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
@@ -2057,18 +1824,14 @@ google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
@@ -2115,20 +1878,16 @@ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/alexcesaro/statsd.v2 v2.0.0/go.mod h1:i0ubccKGzBVNBpdGV5MocxyA/XlLUJzA7SLonnE4drU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
 gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
@@ -2137,7 +1896,6 @@ gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
-gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
@@ -2172,33 +1930,14 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo=
-k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ=
-k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8=
 k8s.io/api v0.22.5/go.mod h1:mEhXyLaSD1qTOf40rRiKXkc+2iCem09rWLlFwhCEiAs=
-k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
-k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
-k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc=
 k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0=
 k8s.io/apimachinery v0.22.5 h1:cIPwldOYm1Slq9VLBRPtEYpyhjIm1C6aAMAoENuvN9s=
 k8s.io/apimachinery v0.22.5/go.mod h1:xziclGKwuuJ2RM5/rSFQSYAj0zdbci3DH8kj+WvyN0U=
-k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU=
-k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM=
-k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q=
 k8s.io/apiserver v0.22.5/go.mod h1:s2WbtgZAkTKt679sYtSudEQrTGWUSQAPe6MupLnlmaQ=
-k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y=
-k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k=
-k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0=
 k8s.io/client-go v0.22.5/go.mod h1:cs6yf/61q2T1SdQL5Rdcjg9J1ElXSwbjSrW2vFImM4Y=
 k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0=
-k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk=
-k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI=
-k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM=
 k8s.io/component-base v0.22.5/go.mod h1:VK3I+TjuF9eaa+Ln67dKxhGar5ynVbwnGrUiNF4MqCI=
-k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM=
-k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=
-k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=
-k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc=
 k8s.io/cri-api v0.23.1 h1:0DHL/hpTf4Fp+QkUXFefWcp1fhjXr9OlNdY9X99c+O8=
 k8s.io/cri-api v0.23.1/go.mod h1:REJE3PSU0h/LOV1APBrupxrEJqnoxZC8KWzkBUHwrK4=
 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
@@ -2211,23 +1950,18 @@ k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
 k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o=
-k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=
 k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
 k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
-k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
-k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
-sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
+sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
 sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

src/runtime/hack/update-generated-hypervisor-proto.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+# (C) Copyright IBM Corp. 2022.
+# SPDX-License-Identifier: Apache-2.0
+
+set -o errexit -o pipefail -o nounset
+
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+
+protoc --gogottrpc_out=protocols/hypervisor \
+	--gogottrpc_opt=plugins=ttrpc+fieldpath,paths=source_relative \
+	-Iprotocols/hypervisor \
+	-I../libs/protocols/protos/gogo/protobuf \
+	protocols/hypervisor/hypervisor.proto

src/runtime/pkg/containerd-shim-v2/create.go

@@ -18,6 +18,7 @@ import (
 	"strconv"
 	"syscall"
 
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	containerd_types "github.com/containerd/containerd/api/types"
 	"github.com/containerd/containerd/mount"
 	taskAPI "github.com/containerd/containerd/runtime/v2/task"
@@ -48,6 +49,62 @@ var defaultStartManagementServerFunc startManagementServerFunc = func(s *service
 	shimLog.Info("management server started")
 }
 
+// CDI (Container Device Interface), is a specification, for container- runtimes,
+// to support third-party devices.
+// It introduces an abstract notion of a device as a resource. Such devices are
+// uniquely specified by a fully-qualified name that is constructed from a
+// vendor ID, a device class, and a name that is unique per vendor ID-device
+// class pair.
+//
+// vendor.com/class=unique_name
+//
+// The combination of vendor ID and device class (vendor.com/class in the
+// above example) is referred to as the device kind.
+// CDI concerns itself only with enabling containers to be device aware.
+// Areas like resource management are explicitly left out of CDI (and are
+// expected to be handled by the orchestrator). Because of this focus, the CDI
+// specification is simple to implement and allows great flexibility for
+// runtimes and orchestrators.
+func withCDI(annotations map[string]string, cdiSpecDirs []string, spec *specs.Spec) (*specs.Spec, error) {
+	// Add devices from CDI annotations
+	_, devsFromAnnotations, err := cdi.ParseAnnotations(annotations)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse CDI device annotations: %w", err)
+	}
+	if len(devsFromAnnotations) == 0 {
+		// No devices found, skip device injection
+		return spec, nil
+	}
+
+	var registry cdi.Registry
+	if len(cdiSpecDirs) > 0 {
+		// We can override the directories where to search for CDI specs
+		// if needed, the default is /etc/cdi /var/run/cdi
+		registry = cdi.GetRegistry(cdi.WithSpecDirs(cdiSpecDirs...))
+	} else {
+		registry = cdi.GetRegistry()
+	}
+
+	if err = registry.Refresh(); err != nil {
+		// We don't consider registry refresh failure a fatal error.
+		// For instance, a dynamically generated invalid CDI Spec file for
+		// any particular vendor shouldn't prevent injection of devices of
+		// different vendors. CDI itself knows better and it will fail the
+		// injection if necessary.
+		return nil, fmt.Errorf("CDI registry refresh failed: %w", err)
+	}
+
+	if _, err := registry.InjectDevices(spec, devsFromAnnotations...); err != nil {
+		return nil, fmt.Errorf("CDI device injection failed: %w", err)
+	}
+
+	// One crucial thing to keep in mind is that CDI device injection
+	// might add OCI Spec environment variables, hooks, and mounts as
+	// well. Therefore it is important that none of the corresponding
+	// OCI Spec fields are reset up in the call stack once we return.
+	return spec, nil
+}
+
 func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*container, error) {
 	rootFs := vc.RootFs{}
 	if len(r.Rootfs) == 1 {
@@ -81,6 +138,16 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con
 		if s.sandbox != nil {
 			return nil, fmt.Errorf("cannot create another sandbox in sandbox: %s", s.sandbox.ID())
 		}
+		// We can provide additional directories where to search for
+		// CDI specs if needed. immutable OS's only have specific
+		// directories where applications can write too. For instance /opt/cdi
+		//
+		// _, err = withCDI(ociSpec.Annotations, []string{"/opt/cdi"}, ociSpec)
+		//
+		_, err = withCDI(ociSpec.Annotations, []string{}, ociSpec)
+		if err != nil {
+			return nil, fmt.Errorf("adding CDI devices failed")
+		}
 
 		s.config = runtimeConfig
 
@@ -261,6 +328,11 @@ func loadRuntimeConfig(s *service, r *taskAPI.CreateTaskRequest, anno map[string
 }
 
 func checkAndMount(s *service, r *taskAPI.CreateTaskRequest) (bool, error) {
+	// In the confidential computing, there is no Image information on the host,
+	// so there is no Rootfs.
+	if s.config.ServiceOffload && len(r.Rootfs) == 0 {
+		return false, nil
+	}
 	if len(r.Rootfs) == 1 {
 		m := r.Rootfs[0]
 

src/runtime/pkg/containerd-shim-v2/image.go

@@ -0,0 +1,84 @@
+// Copyright (c) 2021 Alibaba Cloud
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"context"
+	"time"
+
+	"github.com/containerd/containerd/plugin"
+	"github.com/containerd/containerd/runtime/v2/shim"
+	"github.com/containerd/containerd/runtime/v2/task"
+	"github.com/containerd/ttrpc"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/image"
+)
+
+func init() {
+	plugin.Register(&plugin.Registration{
+		Type:     plugin.TTRPCPlugin,
+		ID:       "image",
+		Requires: []plugin.Type{"*"},
+		InitFn:   initImageService,
+	})
+}
+
+type ImageService struct {
+	s *service
+}
+
+func initImageService(ic *plugin.InitContext) (interface{}, error) {
+	i, err := ic.GetByID(plugin.TTRPCPlugin, "task")
+	if err != nil {
+		return nil, errors.Errorf("get task plugin error. %v", err)
+	}
+	task := i.(shim.TaskService)
+	s := task.TaskService.(*service)
+	is := &ImageService{s: s}
+	return is, nil
+}
+
+func (is *ImageService) RegisterTTRPC(server *ttrpc.Server) error {
+	task.RegisterImageService(server, is)
+	return nil
+}
+
+// Pull image and unbundle ready for container creation
+func (is *ImageService) PullImage(ctx context.Context, req *task.PullImageRequest) (_ *task.PullImageResponse, err error) {
+	shimLog.WithField("image", req.Image).Debug("PullImage() start")
+	defer shimLog.WithField("image", req.Image).Debug("PullImage() end")
+	span, spanCtx := katatrace.Trace(is.s.rootCtx, shimLog, "PullImage", shimTracingTags)
+	defer span.End()
+
+	start := time.Now()
+	defer func() {
+		err = toGRPC(err)
+		rpcDurationsHistogram.WithLabelValues("pullimage").Observe(float64(time.Since(start).Nanoseconds() / int64(time.Millisecond)))
+	}()
+
+	is.s.mu.Lock()
+	defer is.s.mu.Unlock()
+
+	shimLog.WithFields(logrus.Fields{
+		"image": req.Image,
+	}).Debug("Making image pull request")
+
+	r := &image.PullImageReq{
+		Image: req.Image,
+	}
+
+	resp, err := is.s.sandbox.PullImage(spanCtx, r)
+	if err != nil {
+		shimLog.Errorf("kata runtime PullImage err. %v", err)
+		return nil, err
+	}
+	return &task.PullImageResponse{
+		ImageRef: resp.ImageRef,
+	}, err
+}

src/runtime/pkg/govmm/qemu/qemu.go

@@ -283,11 +283,11 @@ type Object struct {
 	FirmwareVolume string
 
 	// CBitPos is the location of the C-bit in a guest page table entry
-	// This is only relevant for sev-guest objects
+	// This is only relevant for sev-guest and sev-snp-guest objects
 	CBitPos uint32
 
 	// ReducedPhysBits is the reduction in the guest physical address space
-	// This is only relevant for sev-guest objects
+	// This is only relevant for sev-guest and sev-snp-guest objects
 	ReducedPhysBits uint32
 
 	// ReadOnly specifies whether `MemPath` is opened read-only or read/write (default)
@@ -295,6 +295,26 @@ type Object struct {
 
 	// Prealloc enables memory preallocation
 	Prealloc bool
+
+	// SevPolicy is the policy for the SEV instance. For more info, see AMD document 55766
+	// This is only relevant for sev-guest objects
+	SevPolicy uint32
+
+	// SnpPolicy is the policy for the SNP instance. For more info, see AMD document 56860
+	// This is only relevant for sev-snp-guest objects
+	SnpPolicy uint64
+
+	// SevCertFilePath is the path to the guest Diffie–Hellman key
+	// This is only relevant for sev-guest objects
+	SevCertFilePath string
+
+	// SevSessionFilePath is the path to the launch blog
+	// This is only relevant for sev-guest objects
+	SevSessionFilePath string
+
+	// SevKernelHashes specifies whether the hashes of the kernel, initrd, & cmdline are included in the measurement
+	// This is only relevant for sev-guest objects
+	SevKernelHashes bool
 }
 
 // Valid returns true if the Object structure is valid and complete.
@@ -352,20 +372,41 @@ func (object Object) QemuParams(config *Config) []string {
 
 	case TDXGuest:
 		objectParams = append(objectParams, string(object.Type))
-		objectParams = append(objectParams, "sept-ve-disable=on")
 		objectParams = append(objectParams, fmt.Sprintf("id=%s", object.ID))
 		if object.Debug {
 			objectParams = append(objectParams, "debug=on")
 		}
-		config.Bios = object.File
+		deviceParams = append(deviceParams, string(object.Driver))
+		deviceParams = append(deviceParams, fmt.Sprintf("id=%s", object.DeviceID))
+		deviceParams = append(deviceParams, fmt.Sprintf("file=%s", object.File))
+		if object.FirmwareVolume != "" {
+			deviceParams = append(deviceParams, fmt.Sprintf("config-firmware-volume=%s", object.FirmwareVolume))
+		}
 	case SEVGuest:
-		fallthrough
+		objectParams = append(objectParams, string(object.Type))
+		objectParams = append(objectParams, fmt.Sprintf("id=%s", object.ID))
+		objectParams = append(objectParams, fmt.Sprintf("cbitpos=%d", object.CBitPos))
+		objectParams = append(objectParams, fmt.Sprintf("reduced-phys-bits=%d", object.ReducedPhysBits))
+		objectParams = append(objectParams, fmt.Sprintf("policy=%d", object.SevPolicy))
+		if object.SevCertFilePath != "" {
+			objectParams = append(objectParams, fmt.Sprintf("dh-cert-file=%s", object.SevCertFilePath))
+		}
+		if object.SevSessionFilePath != "" {
+			objectParams = append(objectParams, fmt.Sprintf("session-file=%s", object.SevSessionFilePath))
+		}
+		if object.SevKernelHashes {
+			objectParams = append(objectParams, "kernel-hashes=on")
+		}
+		// Add OVMF firmware as pflash drive
+		driveParams = append(driveParams, "if=pflash,format=raw,readonly=on")
+		driveParams = append(driveParams, fmt.Sprintf("file=%s", object.File))
 	case SNPGuest:
 		objectParams = append(objectParams, string(object.Type))
 		objectParams = append(objectParams, fmt.Sprintf("id=%s", object.ID))
 		objectParams = append(objectParams, fmt.Sprintf("cbitpos=%d", object.CBitPos))
 		objectParams = append(objectParams, fmt.Sprintf("reduced-phys-bits=%d", object.ReducedPhysBits))
-
+		objectParams = append(objectParams, fmt.Sprintf("policy=0x%x", object.SnpPolicy))
+		// Add OVMF firmware as pflash drive
 		driveParams = append(driveParams, "if=pflash,format=raw,readonly=on")
 		driveParams = append(driveParams, fmt.Sprintf("file=%s", object.File))
 	case SecExecGuest:

src/runtime/pkg/govmm/qemu/qmp.go

@@ -257,6 +257,22 @@ type StatusInfo struct {
 	Status     string `json:"status"`
 }
 
+// SEVInfo represents the SEV guest information
+type SEVInfo struct {
+	State    string `json:"state"`
+	Enabled  bool   `json:"enabled"`
+	APIMajor uint32 `json:"api-major"`
+	APIMinor uint32 `json:"api-minor"`
+	BuildId  uint32 `json:"build-id"`
+	Policy   uint32 `json:"policy"`
+	Handle   uint32 `json:"handle"`
+}
+
+// SEVLaunchMeasurement represents the SEV prelaunch measurement
+type SEVLaunchMeasurement struct {
+	Measurement string `json:"data"`
+}
+
 func (q *QMP) readLoop(fromVMCh chan<- []byte) {
 	scanner := bufio.NewScanner(q.conn)
 	if q.cfg.MaxCapacity > 0 {
@@ -1673,3 +1689,53 @@ func (q *QMP) ExecuteDumpGuestMemory(ctx context.Context, protocol string, pagin
 
 	return q.executeCommand(ctx, "dump-guest-memory", args, nil)
 }
+
+// ExecuteQuerySEV queries SEV hardware details
+func (q *QMP) ExecuteQuerySEV(ctx context.Context) (SEVInfo, error) {
+	response, err := q.executeCommandWithResponse(ctx, "query-sev", nil, nil, nil)
+	if err != nil {
+		return SEVInfo{}, err
+	}
+
+	data, err := json.Marshal(response)
+	if err != nil {
+		return SEVInfo{}, fmt.Errorf("unable to extract SEV information: %v", err)
+	}
+
+	var info SEVInfo
+	if err = json.Unmarshal(data, &info); err != nil {
+		return SEVInfo{}, fmt.Errorf("unable to convert SEV information: %v", err)
+	}
+
+	return info, nil
+}
+
+// ExecuteQuerySEVLaunchMeasure queries SEV launch measurement
+func (q *QMP) ExecuteQuerySEVLaunchMeasure(ctx context.Context) (SEVLaunchMeasurement, error) {
+	response, err := q.executeCommandWithResponse(ctx, "query-sev-launch-measure", nil, nil, nil)
+	if err != nil {
+		return SEVLaunchMeasurement{}, err
+	}
+
+	data, err := json.Marshal(response)
+	if err != nil {
+		return SEVLaunchMeasurement{}, fmt.Errorf("unable to extract launch measurement: %v", err)
+	}
+
+	var measurement SEVLaunchMeasurement
+	if err = json.Unmarshal(data, &measurement); err != nil {
+		return SEVLaunchMeasurement{}, fmt.Errorf("unable to convert launch measurement: %v", err)
+	}
+
+	return measurement, nil
+}
+
+// ExecuteSEVInjectLaunchSecret injects launch secret bundle into SEV guest
+func (q *QMP) ExecuteSEVInjectLaunchSecret(ctx context.Context, packetHeader string, secret string) error {
+	args := map[string]interface{}{
+		"packet-header": packetHeader,
+		"secret":        secret,
+	}
+
+	return q.executeCommand(ctx, "sev-inject-launch-secret", args, nil)
+}

src/runtime/pkg/katautils/config-settings.go.in

@@ -98,6 +98,13 @@ const defaultDisableSeccomp = false
 const defaultDisableGuestSeLinux = true
 const defaultVfioMode = "guest-kernel"
 const defaultLegacySerial = false
+const defaultGuestPreAttestation = false
+const defaultGuestPreAttestationURI string = ""
+const defaultGuestPreAttestationMode string = ""
+const defaultGuestPreAttestationKeyset string = ""
+const defaultSEVCertChainPath string = ""
+const defaultSEVGuestPolicy uint32 = 0
+const defaultSNPGuestPolicy uint64 = 0x30000
 
 var defaultSGXEPCSize = int64(0)
 

src/runtime/pkg/katautils/config.go

@@ -51,6 +51,7 @@ const (
 	qemuHypervisorTableType        = "qemu"
 	acrnHypervisorTableType        = "acrn"
 	dragonballHypervisorTableType  = "dragonball"
+	remoteHypervisorTableType      = "remote"
 
 	// the maximum amount of PCI bridges that can be cold plugged in a VM
 	maxPCIBridges uint32 = 5
@@ -100,7 +101,12 @@ type hypervisor struct {
 	GuestHookPath                  string          `toml:"guest_hook_path"`
 	GuestMemoryDumpPath            string          `toml:"guest_memory_dump_path"`
 	SeccompSandbox                 string          `toml:"seccompsandbox"`
+	GuestPreAttestationURI         string          `toml:"guest_pre_attestation_kbs_uri"`
+	GuestPreAttestationMode        string          `toml:"guest_pre_attestation_kbs_mode"`
+	GuestPreAttestationKeyset      string          `toml:"guest_pre_attestation_keyset"`
+	SEVCertChainPath               string          `toml:"sev_cert_chain"`
 	BlockDeviceAIO                 string          `toml:"block_device_aio"`
+	RemoteHypervisorSocket         string          `toml:"remote_hypervisor_socket"`
 	HypervisorPathList             []string        `toml:"valid_hypervisor_paths"`
 	JailerPathList                 []string        `toml:"valid_jailer_paths"`
 	CtlPathList                    []string        `toml:"valid_ctlpaths"`
@@ -130,6 +136,10 @@ type hypervisor struct {
 	MemSlots                       uint32          `toml:"memory_slots"`
 	DefaultBridges                 uint32          `toml:"default_bridges"`
 	Msize9p                        uint32          `toml:"msize_9p"`
+	GuestPreAttestationGRPCTimeout uint32          `toml:"guest_pre_attestation_grpc_timeout"`
+	SEVGuestPolicy                 uint32          `toml:"sev_guest_policy"`
+	SNPGuestPolicy                 uint64          `toml:"snp_guest_policy"`
+	RemoteHypervisorTimeout        uint32          `toml:"remote_hypervisor_timeout"`
 	NumVCPUs                       int32           `toml:"default_vcpus"`
 	BlockDeviceCacheSet            bool            `toml:"block_device_cache_set"`
 	BlockDeviceCacheDirect         bool            `toml:"block_device_cache_direct"`
@@ -159,6 +169,7 @@ type hypervisor struct {
 	DisableSeLinux                 bool            `toml:"disable_selinux"`
 	DisableGuestSeLinux            bool            `toml:"disable_guest_selinux"`
 	LegacySerial                   bool            `toml:"use_legacy_serial"`
+	GuestPreAttestation            bool            `toml:"guest_pre_attestation"`
 }
 
 type runtime struct {
@@ -647,6 +658,13 @@ func (a agent) kernelModules() []string {
 	return a.KernelModules
 }
 
+func (h hypervisor) getSnpGuestPolicy() uint64 {
+	if h.SNPGuestPolicy == 0 { // or unspecified
+		return defaultSNPGuestPolicy
+	}
+	return h.SNPGuestPolicy
+}
+
 func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 	hypervisor, err := h.path()
 	if err != nil {
@@ -820,74 +838,81 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 	txRateLimiterMaxRate := h.getTxRateLimiterCfg()
 
 	return vc.HypervisorConfig{
-		HypervisorPath:          hypervisor,
-		HypervisorPathList:      h.HypervisorPathList,
-		KernelPath:              kernel,
-		InitrdPath:              initrd,
-		ImagePath:               image,
-		RootfsType:              rootfsType,
-		FirmwarePath:            firmware,
-		FirmwareVolumePath:      firmwareVolume,
-		PFlash:                  pflashes,
-		MachineAccelerators:     machineAccelerators,
-		CPUFeatures:             cpuFeatures,
-		KernelParams:            vc.DeserializeParams(strings.Fields(kernelParams)),
-		HypervisorMachineType:   machineType,
-		NumVCPUs:                h.defaultVCPUs(),
-		DefaultMaxVCPUs:         h.defaultMaxVCPUs(),
-		MemorySize:              h.defaultMemSz(),
-		MemSlots:                h.defaultMemSlots(),
-		MemOffset:               h.defaultMemOffset(),
-		DefaultMaxMemorySize:    h.defaultMaxMemSz(),
-		VirtioMem:               h.VirtioMem,
-		EntropySource:           h.GetEntropySource(),
-		EntropySourceList:       h.EntropySourceList,
-		DefaultBridges:          h.defaultBridges(),
-		DisableBlockDeviceUse:   h.DisableBlockDeviceUse,
-		SharedFS:                sharedFS,
-		VirtioFSDaemon:          h.VirtioFSDaemon,
-		VirtioFSDaemonList:      h.VirtioFSDaemonList,
-		VirtioFSCacheSize:       h.VirtioFSCacheSize,
-		VirtioFSCache:           h.defaultVirtioFSCache(),
-		VirtioFSQueueSize:       h.VirtioFSQueueSize,
-		VirtioFSExtraArgs:       h.VirtioFSExtraArgs,
-		MemPrealloc:             h.MemPrealloc,
-		HugePages:               h.HugePages,
-		IOMMU:                   h.IOMMU,
-		IOMMUPlatform:           h.getIOMMUPlatform(),
-		FileBackedMemRootDir:    h.FileBackedMemRootDir,
-		FileBackedMemRootList:   h.FileBackedMemRootList,
-		Debug:                   h.Debug,
-		DisableNestingChecks:    h.DisableNestingChecks,
-		BlockDeviceDriver:       blockDriver,
-		BlockDeviceAIO:          blockAIO,
-		BlockDeviceCacheSet:     h.BlockDeviceCacheSet,
-		BlockDeviceCacheDirect:  h.BlockDeviceCacheDirect,
-		BlockDeviceCacheNoflush: h.BlockDeviceCacheNoflush,
-		EnableIOThreads:         h.EnableIOThreads,
-		Msize9p:                 h.msize9p(),
-		DisableImageNvdimm:      h.DisableImageNvdimm,
-		HotplugVFIOOnRootBus:    h.HotplugVFIOOnRootBus,
-		HotPlugVFIO:             h.hotPlugVFIO(),
-		ColdPlugVFIO:            h.coldPlugVFIO(),
-		DisableVhostNet:         h.DisableVhostNet,
-		EnableVhostUserStore:    h.EnableVhostUserStore,
-		VhostUserStorePath:      h.vhostUserStorePath(),
-		VhostUserStorePathList:  h.VhostUserStorePathList,
-		SeccompSandbox:          h.SeccompSandbox,
-		GuestHookPath:           h.guestHookPath(),
-		RxRateLimiterMaxRate:    rxRateLimiterMaxRate,
-		TxRateLimiterMaxRate:    txRateLimiterMaxRate,
-		EnableAnnotations:       h.EnableAnnotations,
-		GuestMemoryDumpPath:     h.GuestMemoryDumpPath,
-		GuestMemoryDumpPaging:   h.GuestMemoryDumpPaging,
-		ConfidentialGuest:       h.ConfidentialGuest,
-		SevSnpGuest:             h.SevSnpGuest,
-		GuestSwap:               h.GuestSwap,
-		Rootless:                h.Rootless,
-		LegacySerial:            h.LegacySerial,
-		DisableSeLinux:          h.DisableSeLinux,
-		DisableGuestSeLinux:     h.DisableGuestSeLinux,
+		HypervisorPath:            hypervisor,
+		HypervisorPathList:        h.HypervisorPathList,
+		KernelPath:                kernel,
+		InitrdPath:                initrd,
+		ImagePath:                 image,
+		RootfsType:                rootfsType,
+		FirmwarePath:              firmware,
+		FirmwareVolumePath:        firmwareVolume,
+		PFlash:                    pflashes,
+		MachineAccelerators:       machineAccelerators,
+		CPUFeatures:               cpuFeatures,
+		KernelParams:              vc.DeserializeParams(strings.Fields(kernelParams)),
+		HypervisorMachineType:     machineType,
+		NumVCPUs:                  h.defaultVCPUs(),
+		DefaultMaxVCPUs:           h.defaultMaxVCPUs(),
+		MemorySize:                h.defaultMemSz(),
+		MemSlots:                  h.defaultMemSlots(),
+		MemOffset:                 h.defaultMemOffset(),
+		DefaultMaxMemorySize:      h.defaultMaxMemSz(),
+		VirtioMem:                 h.VirtioMem,
+		EntropySource:             h.GetEntropySource(),
+		EntropySourceList:         h.EntropySourceList,
+		DefaultBridges:            h.defaultBridges(),
+		DisableBlockDeviceUse:     h.DisableBlockDeviceUse,
+		SharedFS:                  sharedFS,
+		VirtioFSDaemon:            h.VirtioFSDaemon,
+		VirtioFSDaemonList:        h.VirtioFSDaemonList,
+		VirtioFSCacheSize:         h.VirtioFSCacheSize,
+		VirtioFSCache:             h.defaultVirtioFSCache(),
+		VirtioFSQueueSize:         h.VirtioFSQueueSize,
+		VirtioFSExtraArgs:         h.VirtioFSExtraArgs,
+		MemPrealloc:               h.MemPrealloc,
+		HugePages:                 h.HugePages,
+		IOMMU:                     h.IOMMU,
+		IOMMUPlatform:             h.getIOMMUPlatform(),
+		FileBackedMemRootDir:      h.FileBackedMemRootDir,
+		FileBackedMemRootList:     h.FileBackedMemRootList,
+		Debug:                     h.Debug,
+		DisableNestingChecks:      h.DisableNestingChecks,
+		BlockDeviceDriver:         blockDriver,
+		BlockDeviceAIO:            blockAIO,
+		BlockDeviceCacheSet:       h.BlockDeviceCacheSet,
+		BlockDeviceCacheDirect:    h.BlockDeviceCacheDirect,
+		BlockDeviceCacheNoflush:   h.BlockDeviceCacheNoflush,
+		EnableIOThreads:           h.EnableIOThreads,
+		Msize9p:                   h.msize9p(),
+		DisableImageNvdimm:        h.DisableImageNvdimm,
+		HotplugVFIOOnRootBus:      h.HotplugVFIOOnRootBus,
+		HotPlugVFIO:               h.hotPlugVFIO(),
+		ColdPlugVFIO:              h.coldPlugVFIO(),
+		DisableVhostNet:           h.DisableVhostNet,
+		EnableVhostUserStore:      h.EnableVhostUserStore,
+		VhostUserStorePath:        h.vhostUserStorePath(),
+		VhostUserStorePathList:    h.VhostUserStorePathList,
+		SeccompSandbox:            h.SeccompSandbox,
+		GuestHookPath:             h.guestHookPath(),
+		RxRateLimiterMaxRate:      rxRateLimiterMaxRate,
+		TxRateLimiterMaxRate:      txRateLimiterMaxRate,
+		EnableAnnotations:         h.EnableAnnotations,
+		GuestMemoryDumpPath:       h.GuestMemoryDumpPath,
+		GuestMemoryDumpPaging:     h.GuestMemoryDumpPaging,
+		ConfidentialGuest:         h.ConfidentialGuest,
+		SevSnpGuest:               h.SevSnpGuest,
+		GuestSwap:                 h.GuestSwap,
+		Rootless:                  h.Rootless,
+		LegacySerial:              h.LegacySerial,
+		DisableSeLinux:            h.DisableSeLinux,
+		GuestPreAttestation:       h.GuestPreAttestation,
+		GuestPreAttestationURI:    h.GuestPreAttestationURI,
+		GuestPreAttestationMode:   h.GuestPreAttestationMode,
+		GuestPreAttestationKeyset: h.GuestPreAttestationKeyset,
+		SEVGuestPolicy:            h.SEVGuestPolicy,
+		SNPGuestPolicy:            h.getSnpGuestPolicy(),
+		SEVCertChainPath:          h.SEVCertChainPath,
+		DisableGuestSeLinux:       h.DisableGuestSeLinux,
 	}, nil
 }
 
@@ -1119,6 +1144,19 @@ func newDragonballHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 	}, nil
 }
 
+func newRemoteHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
+
+	return vc.HypervisorConfig{
+		RemoteHypervisorSocket:  h.RemoteHypervisorSocket,
+		RemoteHypervisorTimeout: h.RemoteHypervisorTimeout,
+		DisableGuestSeLinux:     h.DisableGuestSeLinux,
+
+		// No valid value so avoid to append block device to list in kata_agent.appendDevices
+		BlockDeviceDriver: "dummy",
+		EnableAnnotations: h.EnableAnnotations,
+	}, nil
+}
+
 func newFactoryConfig(f factory) (oci.FactoryConfig, error) {
 	if f.TemplatePath == "" {
 		f.TemplatePath = defaultTemplatePath
@@ -1155,6 +1193,9 @@ func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, confi
 		case dragonballHypervisorTableType:
 			config.HypervisorType = vc.DragonballHypervisor
 			hConfig, err = newDragonballHypervisorConfig(hypervisor)
+		case remoteHypervisorTableType:
+			config.HypervisorType = vc.RemoteHypervisor
+			hConfig, err = newRemoteHypervisorConfig(hypervisor)
 		}
 
 		if err != nil {
@@ -1262,56 +1303,63 @@ func updateRuntimeConfig(configPath string, tomlConf tomlConfig, config *oci.Run
 
 func GetDefaultHypervisorConfig() vc.HypervisorConfig {
 	return vc.HypervisorConfig{
-		HypervisorPath:           defaultHypervisorPath,
-		JailerPath:               defaultJailerPath,
-		KernelPath:               defaultKernelPath,
-		ImagePath:                defaultImagePath,
-		InitrdPath:               defaultInitrdPath,
-		RootfsType:               defaultRootfsType,
-		FirmwarePath:             defaultFirmwarePath,
-		FirmwareVolumePath:       defaultFirmwareVolumePath,
-		MachineAccelerators:      defaultMachineAccelerators,
-		CPUFeatures:              defaultCPUFeatures,
-		HypervisorMachineType:    defaultMachineType,
-		NumVCPUs:                 defaultVCPUCount,
-		DefaultMaxVCPUs:          defaultMaxVCPUCount,
-		MemorySize:               defaultMemSize,
-		MemOffset:                defaultMemOffset,
-		VirtioMem:                defaultVirtioMem,
-		DisableBlockDeviceUse:    defaultDisableBlockDeviceUse,
-		DefaultBridges:           defaultBridgesCount,
-		MemPrealloc:              defaultEnableMemPrealloc,
-		HugePages:                defaultEnableHugePages,
-		IOMMU:                    defaultEnableIOMMU,
-		IOMMUPlatform:            defaultEnableIOMMUPlatform,
-		FileBackedMemRootDir:     defaultFileBackedMemRootDir,
-		Debug:                    defaultEnableDebug,
-		DisableNestingChecks:     defaultDisableNestingChecks,
-		BlockDeviceDriver:        defaultBlockDeviceDriver,
-		BlockDeviceAIO:           defaultBlockDeviceAIO,
-		BlockDeviceCacheSet:      defaultBlockDeviceCacheSet,
-		BlockDeviceCacheDirect:   defaultBlockDeviceCacheDirect,
-		BlockDeviceCacheNoflush:  defaultBlockDeviceCacheNoflush,
-		EnableIOThreads:          defaultEnableIOThreads,
-		Msize9p:                  defaultMsize9p,
-		HotplugVFIOOnRootBus:     defaultHotplugVFIOOnRootBus,
-		ColdPlugVFIO:             defaultColdPlugVFIO,
-		HotPlugVFIO:              defaultHotPlugVFIO,
-		GuestHookPath:            defaultGuestHookPath,
-		VhostUserStorePath:       defaultVhostUserStorePath,
-		VhostUserDeviceReconnect: defaultVhostUserDeviceReconnect,
-		VirtioFSCache:            defaultVirtioFSCacheMode,
-		DisableImageNvdimm:       defaultDisableImageNvdimm,
-		RxRateLimiterMaxRate:     defaultRxRateLimiterMaxRate,
-		TxRateLimiterMaxRate:     defaultTxRateLimiterMaxRate,
-		SGXEPCSize:               defaultSGXEPCSize,
-		ConfidentialGuest:        defaultConfidentialGuest,
-		SevSnpGuest:              defaultSevSnpGuest,
-		GuestSwap:                defaultGuestSwap,
-		Rootless:                 defaultRootlessHypervisor,
-		DisableSeccomp:           defaultDisableSeccomp,
-		DisableGuestSeLinux:      defaultDisableGuestSeLinux,
-		LegacySerial:             defaultLegacySerial,
+		HypervisorPath:            defaultHypervisorPath,
+		JailerPath:                defaultJailerPath,
+		KernelPath:                defaultKernelPath,
+		ImagePath:                 defaultImagePath,
+		InitrdPath:                defaultInitrdPath,
+		RootfsType:                defaultRootfsType,
+		FirmwarePath:              defaultFirmwarePath,
+		FirmwareVolumePath:        defaultFirmwareVolumePath,
+		MachineAccelerators:       defaultMachineAccelerators,
+		CPUFeatures:               defaultCPUFeatures,
+		HypervisorMachineType:     defaultMachineType,
+		NumVCPUs:                  defaultVCPUCount,
+		DefaultMaxVCPUs:           defaultMaxVCPUCount,
+		MemorySize:                defaultMemSize,
+		MemOffset:                 defaultMemOffset,
+		VirtioMem:                 defaultVirtioMem,
+		DisableBlockDeviceUse:     defaultDisableBlockDeviceUse,
+		DefaultBridges:            defaultBridgesCount,
+		MemPrealloc:               defaultEnableMemPrealloc,
+		HugePages:                 defaultEnableHugePages,
+		IOMMU:                     defaultEnableIOMMU,
+		IOMMUPlatform:             defaultEnableIOMMUPlatform,
+		FileBackedMemRootDir:      defaultFileBackedMemRootDir,
+		Debug:                     defaultEnableDebug,
+		DisableNestingChecks:      defaultDisableNestingChecks,
+		BlockDeviceDriver:         defaultBlockDeviceDriver,
+		BlockDeviceAIO:            defaultBlockDeviceAIO,
+		BlockDeviceCacheSet:       defaultBlockDeviceCacheSet,
+		BlockDeviceCacheDirect:    defaultBlockDeviceCacheDirect,
+		BlockDeviceCacheNoflush:   defaultBlockDeviceCacheNoflush,
+		EnableIOThreads:           defaultEnableIOThreads,
+		Msize9p:                   defaultMsize9p,
+		HotplugVFIOOnRootBus:      defaultHotplugVFIOOnRootBus,
+		ColdPlugVFIO:              defaultColdPlugVFIO,
+		HotPlugVFIO:               defaultHotPlugVFIO,
+		GuestHookPath:             defaultGuestHookPath,
+		VhostUserStorePath:        defaultVhostUserStorePath,
+		VhostUserDeviceReconnect:  defaultVhostUserDeviceReconnect,
+		VirtioFSCache:             defaultVirtioFSCacheMode,
+		DisableImageNvdimm:        defaultDisableImageNvdimm,
+		RxRateLimiterMaxRate:      defaultRxRateLimiterMaxRate,
+		TxRateLimiterMaxRate:      defaultTxRateLimiterMaxRate,
+		SGXEPCSize:                defaultSGXEPCSize,
+		ConfidentialGuest:         defaultConfidentialGuest,
+		SevSnpGuest:               defaultSevSnpGuest,
+		GuestSwap:                 defaultGuestSwap,
+		Rootless:                  defaultRootlessHypervisor,
+		DisableSeccomp:            defaultDisableSeccomp,
+		DisableGuestSeLinux:       defaultDisableGuestSeLinux,
+		LegacySerial:              defaultLegacySerial,
+		GuestPreAttestation:       defaultGuestPreAttestation,
+		GuestPreAttestationURI:    defaultGuestPreAttestationURI,
+		GuestPreAttestationMode:   defaultGuestPreAttestationMode,
+		GuestPreAttestationKeyset: defaultGuestPreAttestationKeyset,
+		SEVGuestPolicy:            defaultSEVGuestPolicy,
+		SNPGuestPolicy:            defaultSNPGuestPolicy,
+		SEVCertChainPath:          defaultSEVCertChainPath,
 	}
 }
 
@@ -1407,6 +1455,7 @@ func LoadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat
 	config.JaegerEndpoint = tomlConf.Runtime.JaegerEndpoint
 	config.JaegerUser = tomlConf.Runtime.JaegerUser
 	config.JaegerPassword = tomlConf.Runtime.JaegerPassword
+	config.ServiceOffload = tomlConf.Image.ServiceOffload
 	for _, f := range tomlConf.Runtime.Experimental {
 		feature := exp.Get(f)
 		if feature == nil {
@@ -1741,6 +1790,11 @@ func checkFactoryConfig(config oci.RuntimeConfig) error {
 // checkHypervisorConfig performs basic "sanity checks" on the hypervisor
 // config.
 func checkHypervisorConfig(config vc.HypervisorConfig) error {
+
+	if config.RemoteHypervisorSocket != "" {
+		return nil
+	}
+
 	type image struct {
 		path   string
 		initrd bool

src/runtime/pkg/katautils/config_test.go

@@ -185,6 +185,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (testConfig testRuntime
 		VirtioFSCache:         defaultVirtioFSCacheMode,
 		PFlash:                []string{},
 		SGXEPCSize:            epcSize,
+		SNPGuestPolicy:        defaultSNPGuestPolicy,
 	}
 
 	if goruntime.GOARCH == "arm64" && len(hypervisorConfig.PFlash) == 0 && hypervisorConfig.FirmwarePath == "" {
@@ -569,6 +570,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 		VirtioFSCache:         defaultVirtioFSCacheMode,
 		BlockDeviceAIO:        defaultBlockDeviceAIO,
 		DisableGuestSeLinux:   defaultDisableGuestSeLinux,
+		SNPGuestPolicy:        defaultSNPGuestPolicy,
 		HotPlugVFIO:           defaultHotPlugVFIO,
 		ColdPlugVFIO:          defaultColdPlugVFIO,
 	}

src/runtime/pkg/oci/utils.go

@@ -154,6 +154,9 @@ type RuntimeConfig struct {
 
 	// Determines if Kata creates emptyDir on the guest
 	DisableGuestEmptyDir bool
+
+	// Offload the CRI image management service to the Kata agent.
+	ServiceOffload bool
 }
 
 // AddKernelParam allows the addition of new kernel parameters to an existing
@@ -453,6 +456,10 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig,
 		return err
 	}
 
+	if err := addConfidentialComputingOverrides(ocispec, config); err != nil {
+		return err
+	}
+
 	if err := addHypervisorHotColdPlugVfioOverrides(ocispec, config); err != nil {
 		return err
 	}
@@ -938,6 +945,29 @@ func addAgentConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig) error
 	return nil
 }
 
+func addConfidentialComputingOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
+
+	if err := newAnnotationConfiguration(ocispec, vcAnnotations.GuestPreAttestation).setBool(func(guestPreAttestation bool) {
+		sbConfig.HypervisorConfig.GuestPreAttestation = guestPreAttestation
+	}); err != nil {
+		return err
+	}
+
+	if value, ok := ocispec.Annotations[vcAnnotations.GuestPreAttestationURI]; ok {
+		if value != "" {
+			sbConfig.HypervisorConfig.GuestPreAttestationURI = value
+		}
+	}
+
+	if err := newAnnotationConfiguration(ocispec, vcAnnotations.SEVGuestPolicy).setUint(func(sevGuestPolicy uint64) {
+		sbConfig.HypervisorConfig.SEVGuestPolicy = uint32(sevGuestPolicy)
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // SandboxConfig converts an OCI compatible runtime configuration file
 // to a virtcontainers sandbox configuration structure.
 func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid string, detach, systemdCgroup bool) (vc.SandboxConfig, error) {
@@ -997,6 +1027,8 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st
 		GuestSeLinuxLabel: runtime.GuestSeLinuxLabel,
 
 		Experimental: runtime.Experimental,
+
+		ServiceOffload: runtime.ServiceOffload,
 	}
 
 	if err := addAnnotations(ocispec, &sandboxConfig, runtime); err != nil {

src/runtime/pkg/oci/utils_test.go

@@ -1211,8 +1211,8 @@ func TestNewMount(t *testing.T) {
 	assert := assert.New(t)
 
 	testCases := []struct {
-		out vc.Mount
 		in  specs.Mount
+		out vc.Mount
 	}{
 		{
 			in: specs.Mount{

src/runtime/protocols/hypervisor/hypervisor.proto

@@ -0,0 +1,46 @@
+// (C) Copyright IBM Corp. 2022.
+// SPDX-License-Identifier: Apache-2.0
+
+syntax = "proto3";
+
+package hypervisor;
+
+service Hypervisor {
+	rpc CreateVM(CreateVMRequest) returns (CreateVMResponse) {}
+	rpc StartVM(StartVMRequest) returns (StartVMResponse) {}
+	rpc StopVM(StopVMRequest) returns (StopVMResponse) {}
+	rpc Version(VersionRequest) returns (VersionResponse) {}
+}
+
+
+message VersionRequest {
+	string version = 1;
+}
+
+message VersionResponse {
+	string version = 1;
+}
+
+message CreateVMRequest {
+	string id = 1;
+	map<string, string> annotations = 2;
+	string networkNamespacePath = 3;
+}
+
+message CreateVMResponse {
+	string agentSocketPath = 1;
+}
+
+message StartVMRequest {
+	string id = 1;
+}
+
+message StartVMResponse {
+}
+
+message StopVMRequest {
+	string id = 1;
+}
+
+message StopVMResponse {
+}

src/runtime/protocols/simple-kbs/keybroker.pb.go

@@ -0,0 +1,573 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.28.0
+// 	protoc        v3.6.1
+// source: keybroker.proto
+
+package simple_kbs
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type BundleRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// for platform verification
+	// the cert chain includes the PDH
+	CertificateChain string `protobuf:"bytes,1,opt,name=CertificateChain,proto3" json:"CertificateChain,omitempty"`
+	// Required to construct launch blob
+	Policy uint32 `protobuf:"varint,2,opt,name=Policy,proto3" json:"Policy,omitempty"`
+}
+
+func (x *BundleRequest) Reset() {
+	*x = BundleRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_keybroker_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *BundleRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*BundleRequest) ProtoMessage() {}
+
+func (x *BundleRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_keybroker_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use BundleRequest.ProtoReflect.Descriptor instead.
+func (*BundleRequest) Descriptor() ([]byte, []int) {
+	return file_keybroker_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *BundleRequest) GetCertificateChain() string {
+	if x != nil {
+		return x.CertificateChain
+	}
+	return ""
+}
+
+func (x *BundleRequest) GetPolicy() uint32 {
+	if x != nil {
+		return x.Policy
+	}
+	return 0
+}
+
+type BundleResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	GuestOwnerPublicKey string `protobuf:"bytes,1,opt,name=GuestOwnerPublicKey,proto3" json:"GuestOwnerPublicKey,omitempty"`
+	LaunchBlob          string `protobuf:"bytes,2,opt,name=LaunchBlob,proto3" json:"LaunchBlob,omitempty"`
+	// GUID
+	LaunchId string `protobuf:"bytes,3,opt,name=LaunchId,proto3" json:"LaunchId,omitempty"`
+}
+
+func (x *BundleResponse) Reset() {
+	*x = BundleResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_keybroker_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *BundleResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*BundleResponse) ProtoMessage() {}
+
+func (x *BundleResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_keybroker_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use BundleResponse.ProtoReflect.Descriptor instead.
+func (*BundleResponse) Descriptor() ([]byte, []int) {
+	return file_keybroker_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *BundleResponse) GetGuestOwnerPublicKey() string {
+	if x != nil {
+		return x.GuestOwnerPublicKey
+	}
+	return ""
+}
+
+func (x *BundleResponse) GetLaunchBlob() string {
+	if x != nil {
+		return x.LaunchBlob
+	}
+	return ""
+}
+
+func (x *BundleResponse) GetLaunchId() string {
+	if x != nil {
+		return x.LaunchId
+	}
+	return ""
+}
+
+type RequestDetails struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Guid       string `protobuf:"bytes,1,opt,name=Guid,proto3" json:"Guid,omitempty"`
+	Format     string `protobuf:"bytes,2,opt,name=Format,proto3" json:"Format,omitempty"`
+	SecretType string `protobuf:"bytes,3,opt,name=SecretType,proto3" json:"SecretType,omitempty"`
+	Id         string `protobuf:"bytes,4,opt,name=Id,proto3" json:"Id,omitempty"`
+}
+
+func (x *RequestDetails) Reset() {
+	*x = RequestDetails{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_keybroker_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *RequestDetails) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*RequestDetails) ProtoMessage() {}
+
+func (x *RequestDetails) ProtoReflect() protoreflect.Message {
+	mi := &file_keybroker_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use RequestDetails.ProtoReflect.Descriptor instead.
+func (*RequestDetails) Descriptor() ([]byte, []int) {
+	return file_keybroker_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *RequestDetails) GetGuid() string {
+	if x != nil {
+		return x.Guid
+	}
+	return ""
+}
+
+func (x *RequestDetails) GetFormat() string {
+	if x != nil {
+		return x.Format
+	}
+	return ""
+}
+
+func (x *RequestDetails) GetSecretType() string {
+	if x != nil {
+		return x.SecretType
+	}
+	return ""
+}
+
+func (x *RequestDetails) GetId() string {
+	if x != nil {
+		return x.Id
+	}
+	return ""
+}
+
+type SecretRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// b64 encoded string
+	LaunchMeasurement string `protobuf:"bytes,1,opt,name=LaunchMeasurement,proto3" json:"LaunchMeasurement,omitempty"`
+	LaunchId          string `protobuf:"bytes,2,opt,name=LaunchId,proto3" json:"LaunchId,omitempty"`
+	Policy            uint32 `protobuf:"varint,3,opt,name=Policy,proto3" json:"Policy,omitempty"`
+	// Hints for secret validation,
+	ApiMajor uint32 `protobuf:"varint,4,opt,name=ApiMajor,proto3" json:"ApiMajor,omitempty"`
+	ApiMinor uint32 `protobuf:"varint,5,opt,name=ApiMinor,proto3" json:"ApiMinor,omitempty"`
+	BuildId  uint32 `protobuf:"varint,6,opt,name=BuildId,proto3" json:"BuildId,omitempty"`
+	// The fw digest that the guest was launched with. Hopefully we can
+	// get this from QEMU. Pass this in as a base64 string.
+	FwDigest string `protobuf:"bytes,7,opt,name=FwDigest,proto3" json:"FwDigest,omitempty"`
+	// Flexible description of launch provided by orchestrator
+	// and logged by KBS.
+	LaunchDescription string            `protobuf:"bytes,8,opt,name=LaunchDescription,proto3" json:"LaunchDescription,omitempty"`
+	SecretRequests    []*RequestDetails `protobuf:"bytes,9,rep,name=SecretRequests,proto3" json:"SecretRequests,omitempty"`
+}
+
+func (x *SecretRequest) Reset() {
+	*x = SecretRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_keybroker_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *SecretRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*SecretRequest) ProtoMessage() {}
+
+func (x *SecretRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_keybroker_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use SecretRequest.ProtoReflect.Descriptor instead.
+func (*SecretRequest) Descriptor() ([]byte, []int) {
+	return file_keybroker_proto_rawDescGZIP(), []int{3}
+}
+
+func (x *SecretRequest) GetLaunchMeasurement() string {
+	if x != nil {
+		return x.LaunchMeasurement
+	}
+	return ""
+}
+
+func (x *SecretRequest) GetLaunchId() string {
+	if x != nil {
+		return x.LaunchId
+	}
+	return ""
+}
+
+func (x *SecretRequest) GetPolicy() uint32 {
+	if x != nil {
+		return x.Policy
+	}
+	return 0
+}
+
+func (x *SecretRequest) GetApiMajor() uint32 {
+	if x != nil {
+		return x.ApiMajor
+	}
+	return 0
+}
+
+func (x *SecretRequest) GetApiMinor() uint32 {
+	if x != nil {
+		return x.ApiMinor
+	}
+	return 0
+}
+
+func (x *SecretRequest) GetBuildId() uint32 {
+	if x != nil {
+		return x.BuildId
+	}
+	return 0
+}
+
+func (x *SecretRequest) GetFwDigest() string {
+	if x != nil {
+		return x.FwDigest
+	}
+	return ""
+}
+
+func (x *SecretRequest) GetLaunchDescription() string {
+	if x != nil {
+		return x.LaunchDescription
+	}
+	return ""
+}
+
+func (x *SecretRequest) GetSecretRequests() []*RequestDetails {
+	if x != nil {
+		return x.SecretRequests
+	}
+	return nil
+}
+
+type SecretResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	LaunchSecretHeader string `protobuf:"bytes,1,opt,name=LaunchSecretHeader,proto3" json:"LaunchSecretHeader,omitempty"`
+	LaunchSecretData   string `protobuf:"bytes,2,opt,name=LaunchSecretData,proto3" json:"LaunchSecretData,omitempty"`
+}
+
+func (x *SecretResponse) Reset() {
+	*x = SecretResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_keybroker_proto_msgTypes[4]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *SecretResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*SecretResponse) ProtoMessage() {}
+
+func (x *SecretResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_keybroker_proto_msgTypes[4]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use SecretResponse.ProtoReflect.Descriptor instead.
+func (*SecretResponse) Descriptor() ([]byte, []int) {
+	return file_keybroker_proto_rawDescGZIP(), []int{4}
+}
+
+func (x *SecretResponse) GetLaunchSecretHeader() string {
+	if x != nil {
+		return x.LaunchSecretHeader
+	}
+	return ""
+}
+
+func (x *SecretResponse) GetLaunchSecretData() string {
+	if x != nil {
+		return x.LaunchSecretData
+	}
+	return ""
+}
+
+var File_keybroker_proto protoreflect.FileDescriptor
+
+var file_keybroker_proto_rawDesc = []byte{
+	0x0a, 0x0f, 0x6b, 0x65, 0x79, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x12, 0x09, 0x6b, 0x65, 0x79, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x22, 0x53, 0x0a, 0x0d,
+	0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2a, 0x0a,
+	0x10, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, 0x69,
+	0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69,
+	0x63, 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x6f, 0x6c,
+	0x69, 0x63, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x50, 0x6f, 0x6c, 0x69, 0x63,
+	0x79, 0x22, 0x7e, 0x0a, 0x0e, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f,
+	0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x13, 0x47, 0x75, 0x65, 0x73, 0x74, 0x4f, 0x77, 0x6e, 0x65,
+	0x72, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x13, 0x47, 0x75, 0x65, 0x73, 0x74, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x50, 0x75, 0x62, 0x6c,
+	0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1e, 0x0a, 0x0a, 0x4c, 0x61, 0x75, 0x6e, 0x63, 0x68, 0x42,
+	0x6c, 0x6f, 0x62, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x4c, 0x61, 0x75, 0x6e, 0x63,
+	0x68, 0x42, 0x6c, 0x6f, 0x62, 0x12, 0x1a, 0x0a, 0x08, 0x4c, 0x61, 0x75, 0x6e, 0x63, 0x68, 0x49,
+	0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x4c, 0x61, 0x75, 0x6e, 0x63, 0x68, 0x49,
+	0x64, 0x22, 0x6c, 0x0a, 0x0e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x44, 0x65, 0x74, 0x61,
+	0x69, 0x6c, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x47, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x04, 0x47, 0x75, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x46, 0x6f, 0x72, 0x6d, 0x61,
+	0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12,
+	0x1e, 0x0a, 0x0a, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x0a, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12,
+	0x0e, 0x0a, 0x02, 0x49, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x64, 0x22,
+	0xd0, 0x02, 0x0a, 0x0d, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x12, 0x2c, 0x0a, 0x11, 0x4c, 0x61, 0x75, 0x6e, 0x63, 0x68, 0x4d, 0x65, 0x61, 0x73, 0x75,
+	0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x4c, 0x61,
+	0x75, 0x6e, 0x63, 0x68, 0x4d, 0x65, 0x61, 0x73, 0x75, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12,
+	0x1a, 0x0a, 0x08, 0x4c, 0x61, 0x75, 0x6e, 0x63, 0x68, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x08, 0x4c, 0x61, 0x75, 0x6e, 0x63, 0x68, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x50,
+	0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x50, 0x6f, 0x6c,
+	0x69, 0x63, 0x79, 0x12, 0x1a, 0x0a, 0x08, 0x41, 0x70, 0x69, 0x4d, 0x61, 0x6a, 0x6f, 0x72, 0x18,
+	0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x41, 0x70, 0x69, 0x4d, 0x61, 0x6a, 0x6f, 0x72, 0x12,
+	0x1a, 0x0a, 0x08, 0x41, 0x70, 0x69, 0x4d, 0x69, 0x6e, 0x6f, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28,
+	0x0d, 0x52, 0x08, 0x41, 0x70, 0x69, 0x4d, 0x69, 0x6e, 0x6f, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x42,
+	0x75, 0x69, 0x6c, 0x64, 0x49, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x42, 0x75,
+	0x69, 0x6c, 0x64, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x46, 0x77, 0x44, 0x69, 0x67, 0x65, 0x73,
+	0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x46, 0x77, 0x44, 0x69, 0x67, 0x65, 0x73,
+	0x74, 0x12, 0x2c, 0x0a, 0x11, 0x4c, 0x61, 0x75, 0x6e, 0x63, 0x68, 0x44, 0x65, 0x73, 0x63, 0x72,
+	0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x4c, 0x61,
+	0x75, 0x6e, 0x63, 0x68, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12,
+	0x41, 0x0a, 0x0e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6b, 0x65, 0x79, 0x62, 0x72, 0x6f,
+	0x6b, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x44, 0x65, 0x74, 0x61, 0x69,
+	0x6c, 0x73, 0x52, 0x0e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x73, 0x22, 0x6c, 0x0a, 0x0e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70,
+	0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x12, 0x4c, 0x61, 0x75, 0x6e, 0x63, 0x68, 0x53, 0x65,
+	0x63, 0x72, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x12, 0x4c, 0x61, 0x75, 0x6e, 0x63, 0x68, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x48, 0x65,
+	0x61, 0x64, 0x65, 0x72, 0x12, 0x2a, 0x0a, 0x10, 0x4c, 0x61, 0x75, 0x6e, 0x63, 0x68, 0x53, 0x65,
+	0x63, 0x72, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10,
+	0x4c, 0x61, 0x75, 0x6e, 0x63, 0x68, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61,
+	0x32, 0x9a, 0x01, 0x0a, 0x10, 0x4b, 0x65, 0x79, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x53, 0x65,
+	0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x42, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x42, 0x75, 0x6e, 0x64,
+	0x6c, 0x65, 0x12, 0x18, 0x2e, 0x6b, 0x65, 0x79, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x42,
+	0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x6b,
+	0x65, 0x79, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x09, 0x47, 0x65, 0x74,
+	0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x18, 0x2e, 0x6b, 0x65, 0x79, 0x62, 0x72, 0x6f, 0x6b,
+	0x65, 0x72, 0x2e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x1a, 0x19, 0x2e, 0x6b, 0x65, 0x79, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x63,
+	0x72, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x0e, 0x5a,
+	0x0c, 0x2e, 0x2f, 0x73, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x2d, 0x6b, 0x62, 0x73, 0x62, 0x06, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_keybroker_proto_rawDescOnce sync.Once
+	file_keybroker_proto_rawDescData = file_keybroker_proto_rawDesc
+)
+
+func file_keybroker_proto_rawDescGZIP() []byte {
+	file_keybroker_proto_rawDescOnce.Do(func() {
+		file_keybroker_proto_rawDescData = protoimpl.X.CompressGZIP(file_keybroker_proto_rawDescData)
+	})
+	return file_keybroker_proto_rawDescData
+}
+
+var file_keybroker_proto_msgTypes = make([]protoimpl.MessageInfo, 5)
+var file_keybroker_proto_goTypes = []interface{}{
+	(*BundleRequest)(nil),  // 0: keybroker.BundleRequest
+	(*BundleResponse)(nil), // 1: keybroker.BundleResponse
+	(*RequestDetails)(nil), // 2: keybroker.RequestDetails
+	(*SecretRequest)(nil),  // 3: keybroker.SecretRequest
+	(*SecretResponse)(nil), // 4: keybroker.SecretResponse
+}
+var file_keybroker_proto_depIdxs = []int32{
+	2, // 0: keybroker.SecretRequest.SecretRequests:type_name -> keybroker.RequestDetails
+	0, // 1: keybroker.KeyBrokerService.GetBundle:input_type -> keybroker.BundleRequest
+	3, // 2: keybroker.KeyBrokerService.GetSecret:input_type -> keybroker.SecretRequest
+	1, // 3: keybroker.KeyBrokerService.GetBundle:output_type -> keybroker.BundleResponse
+	4, // 4: keybroker.KeyBrokerService.GetSecret:output_type -> keybroker.SecretResponse
+	3, // [3:5] is the sub-list for method output_type
+	1, // [1:3] is the sub-list for method input_type
+	1, // [1:1] is the sub-list for extension type_name
+	1, // [1:1] is the sub-list for extension extendee
+	0, // [0:1] is the sub-list for field type_name
+}
+
+func init() { file_keybroker_proto_init() }
+func file_keybroker_proto_init() {
+	if File_keybroker_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_keybroker_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*BundleRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_keybroker_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*BundleResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_keybroker_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*RequestDetails); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_keybroker_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SecretRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_keybroker_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SecretResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_keybroker_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   5,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_keybroker_proto_goTypes,
+		DependencyIndexes: file_keybroker_proto_depIdxs,
+		MessageInfos:      file_keybroker_proto_msgTypes,
+	}.Build()
+	File_keybroker_proto = out.File
+	file_keybroker_proto_rawDesc = nil
+	file_keybroker_proto_goTypes = nil
+	file_keybroker_proto_depIdxs = nil
+}

src/runtime/protocols/simple-kbs/keybroker.proto

@@ -0,0 +1,68 @@
+// Copyright (c) 2022 IBM 
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+syntax = "proto3";
+
+package keybroker;
+
+option go_package = "./simple-kbs";
+
+message BundleRequest {
+
+    // for platform verification
+    // the cert chain includes the PDH
+    string CertificateChain = 1;
+    // Required to construct launch blob
+    uint32 Policy = 2;
+
+}
+
+message BundleResponse {
+    string GuestOwnerPublicKey = 1;
+    string LaunchBlob = 2;
+
+    // GUID
+    string LaunchId = 3;
+}
+
+message RequestDetails {
+    string Guid = 1;
+    string Format = 2;
+    string SecretType = 3;
+    string Id = 4;
+}
+
+message SecretRequest {
+    // b64 encoded string
+    string LaunchMeasurement = 1;
+    string LaunchId = 2;
+
+    uint32 Policy = 3;
+
+    // Hints for secret validation,
+    uint32 ApiMajor = 4;
+    uint32 ApiMinor = 5;
+    uint32 BuildId = 6;
+
+    // The fw digest that the guest was launched with. Hopefully we can
+    // get this from QEMU. Pass this in as a base64 string.
+    string FwDigest = 7;
+
+    // Flexible description of launch provided by orchestrator
+    // and logged by KBS.
+    string LaunchDescription = 8;
+
+    repeated RequestDetails SecretRequests = 9;
+}
+
+message SecretResponse {
+    string LaunchSecretHeader = 1;
+    string LaunchSecretData = 2;
+}
+
+service KeyBrokerService {
+    rpc GetBundle(BundleRequest) returns (BundleResponse) {};
+    rpc GetSecret(SecretRequest) returns (SecretResponse) {};
+}
+

src/runtime/protocols/simple-kbs/keybroker_grpc.pb.go

@@ -0,0 +1,141 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.2.0
+// - protoc             v3.6.1
+// source: keybroker.proto
+
+package simple_kbs
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
+const _ = grpc.SupportPackageIsVersion7
+
+// KeyBrokerServiceClient is the client API for KeyBrokerService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type KeyBrokerServiceClient interface {
+	GetBundle(ctx context.Context, in *BundleRequest, opts ...grpc.CallOption) (*BundleResponse, error)
+	GetSecret(ctx context.Context, in *SecretRequest, opts ...grpc.CallOption) (*SecretResponse, error)
+}
+
+type keyBrokerServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewKeyBrokerServiceClient(cc grpc.ClientConnInterface) KeyBrokerServiceClient {
+	return &keyBrokerServiceClient{cc}
+}
+
+func (c *keyBrokerServiceClient) GetBundle(ctx context.Context, in *BundleRequest, opts ...grpc.CallOption) (*BundleResponse, error) {
+	out := new(BundleResponse)
+	err := c.cc.Invoke(ctx, "/keybroker.KeyBrokerService/GetBundle", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *keyBrokerServiceClient) GetSecret(ctx context.Context, in *SecretRequest, opts ...grpc.CallOption) (*SecretResponse, error) {
+	out := new(SecretResponse)
+	err := c.cc.Invoke(ctx, "/keybroker.KeyBrokerService/GetSecret", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// KeyBrokerServiceServer is the server API for KeyBrokerService service.
+// All implementations must embed UnimplementedKeyBrokerServiceServer
+// for forward compatibility
+type KeyBrokerServiceServer interface {
+	GetBundle(context.Context, *BundleRequest) (*BundleResponse, error)
+	GetSecret(context.Context, *SecretRequest) (*SecretResponse, error)
+	mustEmbedUnimplementedKeyBrokerServiceServer()
+}
+
+// UnimplementedKeyBrokerServiceServer must be embedded to have forward compatible implementations.
+type UnimplementedKeyBrokerServiceServer struct {
+}
+
+func (UnimplementedKeyBrokerServiceServer) GetBundle(context.Context, *BundleRequest) (*BundleResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetBundle not implemented")
+}
+func (UnimplementedKeyBrokerServiceServer) GetSecret(context.Context, *SecretRequest) (*SecretResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetSecret not implemented")
+}
+func (UnimplementedKeyBrokerServiceServer) mustEmbedUnimplementedKeyBrokerServiceServer() {}
+
+// UnsafeKeyBrokerServiceServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to KeyBrokerServiceServer will
+// result in compilation errors.
+type UnsafeKeyBrokerServiceServer interface {
+	mustEmbedUnimplementedKeyBrokerServiceServer()
+}
+
+func RegisterKeyBrokerServiceServer(s grpc.ServiceRegistrar, srv KeyBrokerServiceServer) {
+	s.RegisterService(&KeyBrokerService_ServiceDesc, srv)
+}
+
+func _KeyBrokerService_GetBundle_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(BundleRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(KeyBrokerServiceServer).GetBundle(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/keybroker.KeyBrokerService/GetBundle",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(KeyBrokerServiceServer).GetBundle(ctx, req.(*BundleRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _KeyBrokerService_GetSecret_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(SecretRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(KeyBrokerServiceServer).GetSecret(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/keybroker.KeyBrokerService/GetSecret",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(KeyBrokerServiceServer).GetSecret(ctx, req.(*SecretRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+// KeyBrokerService_ServiceDesc is the grpc.ServiceDesc for KeyBrokerService service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var KeyBrokerService_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "keybroker.KeyBrokerService",
+	HandlerType: (*KeyBrokerServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "GetBundle",
+			Handler:    _KeyBrokerService_GetBundle_Handler,
+		},
+		{
+			MethodName: "GetSecret",
+			Handler:    _KeyBrokerService_GetSecret_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "keybroker.proto",
+}

src/runtime/vendor/github.com/container-orchestrated-devices/container-device-interface/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

src/runtime/vendor/github.com/container-orchestrated-devices/container-device-interface/internal/multierror/multierror.go

@@ -0,0 +1,82 @@
+/*
+   Copyright © 2022 The CDI Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package multierror
+
+import (
+	"strings"
+)
+
+// New combines several errors into a single error. Parameters that are nil are
+// ignored. If no errors are passed in or all parameters are nil, then the
+// result is also nil.
+func New(errors ...error) error {
+	// Filter out nil entries.
+	numErrors := 0
+	for _, err := range errors {
+		if err != nil {
+			errors[numErrors] = err
+			numErrors++
+		}
+	}
+	if numErrors == 0 {
+		return nil
+	}
+	return multiError(errors[0:numErrors])
+}
+
+// multiError is the underlying implementation used by New.
+//
+// Beware that a null multiError is not the same as a nil error.
+type multiError []error
+
+// multiError returns all individual error strings concatenated with "\n"
+func (e multiError) Error() string {
+	var builder strings.Builder
+	for i, err := range e {
+		if i > 0 {
+			_, _ = builder.WriteString("\n")
+		}
+		_, _ = builder.WriteString(err.Error())
+	}
+	return builder.String()
+}
+
+// Append returns a new multi error all errors concatenated. Errors that are
+// multi errors get flattened, nil is ignored.
+func Append(err error, errors ...error) error {
+	var result multiError
+	if m, ok := err.(multiError); ok {
+		result = m
+	} else if err != nil {
+		result = append(result, err)
+	}
+
+	for _, e := range errors {
+		if e == nil {
+			continue
+		}
+		if m, ok := e.(multiError); ok {
+			result = append(result, m...)
+		} else {
+			result = append(result, e)
+		}
+	}
+	if len(result) == 0 {
+		return nil
+	}
+	return result
+}

src/runtime/vendor/github.com/container-orchestrated-devices/container-device-interface/internal/validation/k8s/objectmeta.go

@@ -0,0 +1,57 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Adapted from k8s.io/apimachinery/pkg/api/validation:
+// https://github.com/kubernetes/apimachinery/blob/7687996c715ee7d5c8cf1e3215e607eb065a4221/pkg/api/validation/objectmeta.go
+
+package k8s
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/container-orchestrated-devices/container-device-interface/internal/multierror"
+)
+
+// TotalAnnotationSizeLimitB defines the maximum size of all annotations in characters.
+const TotalAnnotationSizeLimitB int = 256 * (1 << 10) // 256 kB
+
+// ValidateAnnotations validates that a set of annotations are correctly defined.
+func ValidateAnnotations(annotations map[string]string, path string) error {
+	errors := multierror.New()
+	for k := range annotations {
+		// The rule is QualifiedName except that case doesn't matter, so convert to lowercase before checking.
+		for _, msg := range IsQualifiedName(strings.ToLower(k)) {
+			errors = multierror.Append(errors, fmt.Errorf("%v.%v is invalid: %v", path, k, msg))
+		}
+	}
+	if err := ValidateAnnotationsSize(annotations); err != nil {
+		errors = multierror.Append(errors, fmt.Errorf("%v is too long: %v", path, err))
+	}
+	return errors
+}
+
+// ValidateAnnotationsSize validates that a set of annotations is not too large.
+func ValidateAnnotationsSize(annotations map[string]string) error {
+	var totalSize int64
+	for k, v := range annotations {
+		totalSize += (int64)(len(k)) + (int64)(len(v))
+	}
+	if totalSize > (int64)(TotalAnnotationSizeLimitB) {
+		return fmt.Errorf("annotations size %d is larger than limit %d", totalSize, TotalAnnotationSizeLimitB)
+	}
+	return nil
+}

src/runtime/vendor/github.com/container-orchestrated-devices/container-device-interface/internal/validation/k8s/validation.go

@@ -0,0 +1,217 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Adapted from k8s.io/apimachinery/pkg/util/validation:
+// https://github.com/kubernetes/apimachinery/blob/7687996c715ee7d5c8cf1e3215e607eb065a4221/pkg/util/validation/validation.go
+
+package k8s
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+const qnameCharFmt string = "[A-Za-z0-9]"
+const qnameExtCharFmt string = "[-A-Za-z0-9_.]"
+const qualifiedNameFmt string = "(" + qnameCharFmt + qnameExtCharFmt + "*)?" + qnameCharFmt
+const qualifiedNameErrMsg string = "must consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character"
+const qualifiedNameMaxLength int = 63
+
+var qualifiedNameRegexp = regexp.MustCompile("^" + qualifiedNameFmt + "$")
+
+// IsQualifiedName tests whether the value passed is what Kubernetes calls a
+// "qualified name".  This is a format used in various places throughout the
+// system.  If the value is not valid, a list of error strings is returned.
+// Otherwise an empty list (or nil) is returned.
+func IsQualifiedName(value string) []string {
+	var errs []string
+	parts := strings.Split(value, "/")
+	var name string
+	switch len(parts) {
+	case 1:
+		name = parts[0]
+	case 2:
+		var prefix string
+		prefix, name = parts[0], parts[1]
+		if len(prefix) == 0 {
+			errs = append(errs, "prefix part "+EmptyError())
+		} else if msgs := IsDNS1123Subdomain(prefix); len(msgs) != 0 {
+			errs = append(errs, prefixEach(msgs, "prefix part ")...)
+		}
+	default:
+		return append(errs, "a qualified name "+RegexError(qualifiedNameErrMsg, qualifiedNameFmt, "MyName", "my.name", "123-abc")+
+			" with an optional DNS subdomain prefix and '/' (e.g. 'example.com/MyName')")
+	}
+
+	if len(name) == 0 {
+		errs = append(errs, "name part "+EmptyError())
+	} else if len(name) > qualifiedNameMaxLength {
+		errs = append(errs, "name part "+MaxLenError(qualifiedNameMaxLength))
+	}
+	if !qualifiedNameRegexp.MatchString(name) {
+		errs = append(errs, "name part "+RegexError(qualifiedNameErrMsg, qualifiedNameFmt, "MyName", "my.name", "123-abc"))
+	}
+	return errs
+}
+
+const labelValueFmt string = "(" + qualifiedNameFmt + ")?"
+const labelValueErrMsg string = "a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character"
+
+// LabelValueMaxLength is a label's max length
+const LabelValueMaxLength int = 63
+
+var labelValueRegexp = regexp.MustCompile("^" + labelValueFmt + "$")
+
+// IsValidLabelValue tests whether the value passed is a valid label value.  If
+// the value is not valid, a list of error strings is returned.  Otherwise an
+// empty list (or nil) is returned.
+func IsValidLabelValue(value string) []string {
+	var errs []string
+	if len(value) > LabelValueMaxLength {
+		errs = append(errs, MaxLenError(LabelValueMaxLength))
+	}
+	if !labelValueRegexp.MatchString(value) {
+		errs = append(errs, RegexError(labelValueErrMsg, labelValueFmt, "MyValue", "my_value", "12345"))
+	}
+	return errs
+}
+
+const dns1123LabelFmt string = "[a-z0-9]([-a-z0-9]*[a-z0-9])?"
+const dns1123LabelErrMsg string = "a lowercase RFC 1123 label must consist of lower case alphanumeric characters or '-', and must start and end with an alphanumeric character"
+
+// DNS1123LabelMaxLength is a label's max length in DNS (RFC 1123)
+const DNS1123LabelMaxLength int = 63
+
+var dns1123LabelRegexp = regexp.MustCompile("^" + dns1123LabelFmt + "$")
+
+// IsDNS1123Label tests for a string that conforms to the definition of a label in
+// DNS (RFC 1123).
+func IsDNS1123Label(value string) []string {
+	var errs []string
+	if len(value) > DNS1123LabelMaxLength {
+		errs = append(errs, MaxLenError(DNS1123LabelMaxLength))
+	}
+	if !dns1123LabelRegexp.MatchString(value) {
+		errs = append(errs, RegexError(dns1123LabelErrMsg, dns1123LabelFmt, "my-name", "123-abc"))
+	}
+	return errs
+}
+
+const dns1123SubdomainFmt string = dns1123LabelFmt + "(\\." + dns1123LabelFmt + ")*"
+const dns1123SubdomainErrorMsg string = "a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character"
+
+// DNS1123SubdomainMaxLength is a subdomain's max length in DNS (RFC 1123)
+const DNS1123SubdomainMaxLength int = 253
+
+var dns1123SubdomainRegexp = regexp.MustCompile("^" + dns1123SubdomainFmt + "$")
+
+// IsDNS1123Subdomain tests for a string that conforms to the definition of a
+// subdomain in DNS (RFC 1123).
+func IsDNS1123Subdomain(value string) []string {
+	var errs []string
+	if len(value) > DNS1123SubdomainMaxLength {
+		errs = append(errs, MaxLenError(DNS1123SubdomainMaxLength))
+	}
+	if !dns1123SubdomainRegexp.MatchString(value) {
+		errs = append(errs, RegexError(dns1123SubdomainErrorMsg, dns1123SubdomainFmt, "example.com"))
+	}
+	return errs
+}
+
+const dns1035LabelFmt string = "[a-z]([-a-z0-9]*[a-z0-9])?"
+const dns1035LabelErrMsg string = "a DNS-1035 label must consist of lower case alphanumeric characters or '-', start with an alphabetic character, and end with an alphanumeric character"
+
+// DNS1035LabelMaxLength is a label's max length in DNS (RFC 1035)
+const DNS1035LabelMaxLength int = 63
+
+var dns1035LabelRegexp = regexp.MustCompile("^" + dns1035LabelFmt + "$")
+
+// IsDNS1035Label tests for a string that conforms to the definition of a label in
+// DNS (RFC 1035).
+func IsDNS1035Label(value string) []string {
+	var errs []string
+	if len(value) > DNS1035LabelMaxLength {
+		errs = append(errs, MaxLenError(DNS1035LabelMaxLength))
+	}
+	if !dns1035LabelRegexp.MatchString(value) {
+		errs = append(errs, RegexError(dns1035LabelErrMsg, dns1035LabelFmt, "my-name", "abc-123"))
+	}
+	return errs
+}
+
+// wildcard definition - RFC 1034 section 4.3.3.
+// examples:
+// - valid: *.bar.com, *.foo.bar.com
+// - invalid: *.*.bar.com, *.foo.*.com, *bar.com, f*.bar.com, *
+const wildcardDNS1123SubdomainFmt = "\\*\\." + dns1123SubdomainFmt
+const wildcardDNS1123SubdomainErrMsg = "a wildcard DNS-1123 subdomain must start with '*.', followed by a valid DNS subdomain, which must consist of lower case alphanumeric characters, '-' or '.' and end with an alphanumeric character"
+
+// IsWildcardDNS1123Subdomain tests for a string that conforms to the definition of a
+// wildcard subdomain in DNS (RFC 1034 section 4.3.3).
+func IsWildcardDNS1123Subdomain(value string) []string {
+	wildcardDNS1123SubdomainRegexp := regexp.MustCompile("^" + wildcardDNS1123SubdomainFmt + "$")
+
+	var errs []string
+	if len(value) > DNS1123SubdomainMaxLength {
+		errs = append(errs, MaxLenError(DNS1123SubdomainMaxLength))
+	}
+	if !wildcardDNS1123SubdomainRegexp.MatchString(value) {
+		errs = append(errs, RegexError(wildcardDNS1123SubdomainErrMsg, wildcardDNS1123SubdomainFmt, "*.example.com"))
+	}
+	return errs
+}
+
+// MaxLenError returns a string explanation of a "string too long" validation
+// failure.
+func MaxLenError(length int) string {
+	return fmt.Sprintf("must be no more than %d characters", length)
+}
+
+// RegexError returns a string explanation of a regex validation failure.
+func RegexError(msg string, fmt string, examples ...string) string {
+	if len(examples) == 0 {
+		return msg + " (regex used for validation is '" + fmt + "')"
+	}
+	msg += " (e.g. "
+	for i := range examples {
+		if i > 0 {
+			msg += " or "
+		}
+		msg += "'" + examples[i] + "', "
+	}
+	msg += "regex used for validation is '" + fmt + "')"
+	return msg
+}
+
+// EmptyError returns a string explanation of a "must not be empty" validation
+// failure.
+func EmptyError() string {
+	return "must be non-empty"
+}
+
+func prefixEach(msgs []string, prefix string) []string {
+	for i := range msgs {
+		msgs[i] = prefix + msgs[i]
+	}
+	return msgs
+}
+
+// InclusiveRangeError returns a string explanation of a numeric "must be
+// between" validation failure.
+func InclusiveRangeError(lo, hi int) string {
+	return fmt.Sprintf(`must be between %d and %d, inclusive`, lo, hi)
+}

src/runtime/vendor/github.com/container-orchestrated-devices/container-device-interface/internal/validation/validate.go

@@ -0,0 +1,56 @@
+/*
+   Copyright © The CDI Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package validation
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/container-orchestrated-devices/container-device-interface/internal/validation/k8s"
+)
+
+// ValidateSpecAnnotations checks whether spec annotations are valid.
+func ValidateSpecAnnotations(name string, any interface{}) error {
+	if any == nil {
+		return nil
+	}
+
+	switch v := any.(type) {
+	case map[string]interface{}:
+		annotations := make(map[string]string)
+		for k, v := range v {
+			if s, ok := v.(string); ok {
+				annotations[k] = s
+			} else {
+				return fmt.Errorf("invalid annotation %v.%v; %v is not a string", name, k, any)
+			}
+		}
+		return validateSpecAnnotations(name, annotations)
+	}
+
+	return nil
+}
+
+// validateSpecAnnotations checks whether spec annotations are valid.
+func validateSpecAnnotations(name string, annotations map[string]string) error {
+	path := "annotations"
+	if name != "" {
+		path = strings.Join([]string{name, path}, ".")
+	}
+
+	return k8s.ValidateAnnotations(annotations, path)
+}

src/runtime/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/annotations.go

@@ -0,0 +1,141 @@
+/*
+   Copyright © 2021-2022 The CDI Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package cdi
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/parser"
+)
+
+const (
+	// AnnotationPrefix is the prefix for CDI container annotation keys.
+	AnnotationPrefix = "cdi.k8s.io/"
+)
+
+// UpdateAnnotations updates annotations with a plugin-specific CDI device
+// injection request for the given devices. Upon any error a non-nil error
+// is returned and annotations are left intact. By convention plugin should
+// be in the format of "vendor.device-type".
+func UpdateAnnotations(annotations map[string]string, plugin string, deviceID string, devices []string) (map[string]string, error) {
+	key, err := AnnotationKey(plugin, deviceID)
+	if err != nil {
+		return annotations, fmt.Errorf("CDI annotation failed: %w", err)
+	}
+	if _, ok := annotations[key]; ok {
+		return annotations, fmt.Errorf("CDI annotation failed, key %q used", key)
+	}
+	value, err := AnnotationValue(devices)
+	if err != nil {
+		return annotations, fmt.Errorf("CDI annotation failed: %w", err)
+	}
+
+	if annotations == nil {
+		annotations = make(map[string]string)
+	}
+	annotations[key] = value
+
+	return annotations, nil
+}
+
+// ParseAnnotations parses annotations for CDI device injection requests.
+// The keys and devices from all such requests are collected into slices
+// which are returned as the result. All devices are expected to be fully
+// qualified CDI device names. If any device fails this check empty slices
+// are returned along with a non-nil error. The annotations are expected
+// to be formatted by, or in a compatible fashion to UpdateAnnotations().
+func ParseAnnotations(annotations map[string]string) ([]string, []string, error) {
+	var (
+		keys    []string
+		devices []string
+	)
+
+	for key, value := range annotations {
+		if !strings.HasPrefix(key, AnnotationPrefix) {
+			continue
+		}
+		for _, d := range strings.Split(value, ",") {
+			if !IsQualifiedName(d) {
+				return nil, nil, fmt.Errorf("invalid CDI device name %q", d)
+			}
+			devices = append(devices, d)
+		}
+		keys = append(keys, key)
+	}
+
+	return keys, devices, nil
+}
+
+// AnnotationKey returns a unique annotation key for an device allocation
+// by a K8s device plugin. pluginName should be in the format of
+// "vendor.device-type". deviceID is the ID of the device the plugin is
+// allocating. It is used to make sure that the generated key is unique
+// even if multiple allocations by a single plugin needs to be annotated.
+func AnnotationKey(pluginName, deviceID string) (string, error) {
+	const maxNameLen = 63
+
+	if pluginName == "" {
+		return "", errors.New("invalid plugin name, empty")
+	}
+	if deviceID == "" {
+		return "", errors.New("invalid deviceID, empty")
+	}
+
+	name := pluginName + "_" + strings.ReplaceAll(deviceID, "/", "_")
+
+	if len(name) > maxNameLen {
+		return "", fmt.Errorf("invalid plugin+deviceID %q, too long", name)
+	}
+
+	if c := rune(name[0]); !parser.IsAlphaNumeric(c) {
+		return "", fmt.Errorf("invalid name %q, first '%c' should be alphanumeric",
+			name, c)
+	}
+	if len(name) > 2 {
+		for _, c := range name[1 : len(name)-1] {
+			switch {
+			case parser.IsAlphaNumeric(c):
+			case c == '_' || c == '-' || c == '.':
+			default:
+				return "", fmt.Errorf("invalid name %q, invalid character '%c'",
+					name, c)
+			}
+		}
+	}
+	if c := rune(name[len(name)-1]); !parser.IsAlphaNumeric(c) {
+		return "", fmt.Errorf("invalid name %q, last '%c' should be alphanumeric",
+			name, c)
+	}
+
+	return AnnotationPrefix + name, nil
+}
+
+// AnnotationValue returns an annotation value for the given devices.
+func AnnotationValue(devices []string) (string, error) {
+	value, sep := "", ""
+	for _, d := range devices {
+		if _, _, _, err := ParseQualifiedName(d); err != nil {
+			return "", err
+		}
+		value += sep + d
+		sep = ","
+	}
+
+	return value, nil
+}

src/runtime/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/cache.go

@@ -0,0 +1,581 @@
+/*
+   Copyright © 2021 The CDI Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package cdi
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/container-orchestrated-devices/container-device-interface/internal/multierror"
+	cdi "github.com/container-orchestrated-devices/container-device-interface/specs-go"
+	"github.com/fsnotify/fsnotify"
+	oci "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+// Option is an option to change some aspect of default CDI behavior.
+type Option func(*Cache) error
+
+// Cache stores CDI Specs loaded from Spec directories.
+type Cache struct {
+	sync.Mutex
+	specDirs  []string
+	specs     map[string][]*Spec
+	devices   map[string]*Device
+	errors    map[string][]error
+	dirErrors map[string]error
+
+	autoRefresh bool
+	watch       *watch
+}
+
+// WithAutoRefresh returns an option to control automatic Cache refresh.
+// By default auto-refresh is enabled, the list of Spec directories are
+// monitored and the Cache is automatically refreshed whenever a change
+// is detected. This option can be used to disable this behavior when a
+// manually refreshed mode is preferable.
+func WithAutoRefresh(autoRefresh bool) Option {
+	return func(c *Cache) error {
+		c.autoRefresh = autoRefresh
+		return nil
+	}
+}
+
+// NewCache creates a new CDI Cache. The cache is populated from a set
+// of CDI Spec directories. These can be specified using a WithSpecDirs
+// option. The default set of directories is exposed in DefaultSpecDirs.
+func NewCache(options ...Option) (*Cache, error) {
+	c := &Cache{
+		autoRefresh: true,
+		watch:       &watch{},
+	}
+
+	WithSpecDirs(DefaultSpecDirs...)(c)
+	c.Lock()
+	defer c.Unlock()
+
+	return c, c.configure(options...)
+}
+
+// Configure applies options to the Cache. Updates and refreshes the
+// Cache if options have changed.
+func (c *Cache) Configure(options ...Option) error {
+	if len(options) == 0 {
+		return nil
+	}
+
+	c.Lock()
+	defer c.Unlock()
+
+	return c.configure(options...)
+}
+
+// Configure the Cache. Start/stop CDI Spec directory watch, refresh
+// the Cache if necessary.
+func (c *Cache) configure(options ...Option) error {
+	var err error
+
+	for _, o := range options {
+		if err = o(c); err != nil {
+			return fmt.Errorf("failed to apply cache options: %w", err)
+		}
+	}
+
+	c.dirErrors = make(map[string]error)
+
+	c.watch.stop()
+	if c.autoRefresh {
+		c.watch.setup(c.specDirs, c.dirErrors)
+		c.watch.start(&c.Mutex, c.refresh, c.dirErrors)
+	}
+	c.refresh()
+
+	return nil
+}
+
+// Refresh rescans the CDI Spec directories and refreshes the Cache.
+// In manual refresh mode the cache is always refreshed. In auto-
+// refresh mode the cache is only refreshed if it is out of date.
+func (c *Cache) Refresh() error {
+	c.Lock()
+	defer c.Unlock()
+
+	// force a refresh in manual mode
+	if refreshed, err := c.refreshIfRequired(!c.autoRefresh); refreshed {
+		return err
+	}
+
+	// collect and return cached errors, much like refresh() does it
+	var result error
+	for _, errors := range c.errors {
+		result = multierror.Append(result, errors...)
+	}
+	return result
+}
+
+// Refresh the Cache by rescanning CDI Spec directories and files.
+func (c *Cache) refresh() error {
+	var (
+		specs      = map[string][]*Spec{}
+		devices    = map[string]*Device{}
+		conflicts  = map[string]struct{}{}
+		specErrors = map[string][]error{}
+		result     []error
+	)
+
+	// collect errors per spec file path and once globally
+	collectError := func(err error, paths ...string) {
+		result = append(result, err)
+		for _, path := range paths {
+			specErrors[path] = append(specErrors[path], err)
+		}
+	}
+	// resolve conflicts based on device Spec priority (order of precedence)
+	resolveConflict := func(name string, dev *Device, old *Device) bool {
+		devSpec, oldSpec := dev.GetSpec(), old.GetSpec()
+		devPrio, oldPrio := devSpec.GetPriority(), oldSpec.GetPriority()
+		switch {
+		case devPrio > oldPrio:
+			return false
+		case devPrio == oldPrio:
+			devPath, oldPath := devSpec.GetPath(), oldSpec.GetPath()
+			collectError(fmt.Errorf("conflicting device %q (specs %q, %q)",
+				name, devPath, oldPath), devPath, oldPath)
+			conflicts[name] = struct{}{}
+		}
+		return true
+	}
+
+	_ = scanSpecDirs(c.specDirs, func(path string, priority int, spec *Spec, err error) error {
+		path = filepath.Clean(path)
+		if err != nil {
+			collectError(fmt.Errorf("failed to load CDI Spec %w", err), path)
+			return nil
+		}
+
+		vendor := spec.GetVendor()
+		specs[vendor] = append(specs[vendor], spec)
+
+		for _, dev := range spec.devices {
+			qualified := dev.GetQualifiedName()
+			other, ok := devices[qualified]
+			if ok {
+				if resolveConflict(qualified, dev, other) {
+					continue
+				}
+			}
+			devices[qualified] = dev
+		}
+
+		return nil
+	})
+
+	for conflict := range conflicts {
+		delete(devices, conflict)
+	}
+
+	c.specs = specs
+	c.devices = devices
+	c.errors = specErrors
+
+	return multierror.New(result...)
+}
+
+// RefreshIfRequired triggers a refresh if necessary.
+func (c *Cache) refreshIfRequired(force bool) (bool, error) {
+	// We need to refresh if
+	// - it's forced by an explicitly call to Refresh() in manual mode
+	// - a missing Spec dir appears (added to watch) in auto-refresh mode
+	if force || (c.autoRefresh && c.watch.update(c.dirErrors)) {
+		return true, c.refresh()
+	}
+	return false, nil
+}
+
+// InjectDevices injects the given qualified devices to an OCI Spec. It
+// returns any unresolvable devices and an error if injection fails for
+// any of the devices.
+func (c *Cache) InjectDevices(ociSpec *oci.Spec, devices ...string) ([]string, error) {
+	var unresolved []string
+
+	if ociSpec == nil {
+		return devices, fmt.Errorf("can't inject devices, nil OCI Spec")
+	}
+
+	c.Lock()
+	defer c.Unlock()
+
+	c.refreshIfRequired(false)
+
+	edits := &ContainerEdits{}
+	specs := map[*Spec]struct{}{}
+
+	for _, device := range devices {
+		d := c.devices[device]
+		if d == nil {
+			unresolved = append(unresolved, device)
+			continue
+		}
+		if _, ok := specs[d.GetSpec()]; !ok {
+			specs[d.GetSpec()] = struct{}{}
+			edits.Append(d.GetSpec().edits())
+		}
+		edits.Append(d.edits())
+	}
+
+	if unresolved != nil {
+		return unresolved, fmt.Errorf("unresolvable CDI devices %s",
+			strings.Join(devices, ", "))
+	}
+
+	if err := edits.Apply(ociSpec); err != nil {
+		return nil, fmt.Errorf("failed to inject devices: %w", err)
+	}
+
+	return nil, nil
+}
+
+// highestPrioritySpecDir returns the Spec directory with highest priority
+// and its priority.
+func (c *Cache) highestPrioritySpecDir() (string, int) {
+	if len(c.specDirs) == 0 {
+		return "", -1
+	}
+
+	prio := len(c.specDirs) - 1
+	dir := c.specDirs[prio]
+
+	return dir, prio
+}
+
+// WriteSpec writes a Spec file with the given content into the highest
+// priority Spec directory. If name has a "json" or "yaml" extension it
+// choses the encoding. Otherwise the default YAML encoding is used.
+func (c *Cache) WriteSpec(raw *cdi.Spec, name string) error {
+	var (
+		specDir string
+		path    string
+		prio    int
+		spec    *Spec
+		err     error
+	)
+
+	specDir, prio = c.highestPrioritySpecDir()
+	if specDir == "" {
+		return errors.New("no Spec directories to write to")
+	}
+
+	path = filepath.Join(specDir, name)
+	if ext := filepath.Ext(path); ext != ".json" && ext != ".yaml" {
+		path += defaultSpecExt
+	}
+
+	spec, err = newSpec(raw, path, prio)
+	if err != nil {
+		return err
+	}
+
+	return spec.write(true)
+}
+
+// RemoveSpec removes a Spec with the given name from the highest
+// priority Spec directory. This function can be used to remove a
+// Spec previously written by WriteSpec(). If the file exists and
+// its removal fails RemoveSpec returns an error.
+func (c *Cache) RemoveSpec(name string) error {
+	var (
+		specDir string
+		path    string
+		err     error
+	)
+
+	specDir, _ = c.highestPrioritySpecDir()
+	if specDir == "" {
+		return errors.New("no Spec directories to remove from")
+	}
+
+	path = filepath.Join(specDir, name)
+	if ext := filepath.Ext(path); ext != ".json" && ext != ".yaml" {
+		path += defaultSpecExt
+	}
+
+	err = os.Remove(path)
+	if err != nil && errors.Is(err, fs.ErrNotExist) {
+		err = nil
+	}
+
+	return err
+}
+
+// GetDevice returns the cached device for the given qualified name.
+func (c *Cache) GetDevice(device string) *Device {
+	c.Lock()
+	defer c.Unlock()
+
+	c.refreshIfRequired(false)
+
+	return c.devices[device]
+}
+
+// ListDevices lists all cached devices by qualified name.
+func (c *Cache) ListDevices() []string {
+	var devices []string
+
+	c.Lock()
+	defer c.Unlock()
+
+	c.refreshIfRequired(false)
+
+	for name := range c.devices {
+		devices = append(devices, name)
+	}
+	sort.Strings(devices)
+
+	return devices
+}
+
+// ListVendors lists all vendors known to the cache.
+func (c *Cache) ListVendors() []string {
+	var vendors []string
+
+	c.Lock()
+	defer c.Unlock()
+
+	c.refreshIfRequired(false)
+
+	for vendor := range c.specs {
+		vendors = append(vendors, vendor)
+	}
+	sort.Strings(vendors)
+
+	return vendors
+}
+
+// ListClasses lists all device classes known to the cache.
+func (c *Cache) ListClasses() []string {
+	var (
+		cmap    = map[string]struct{}{}
+		classes []string
+	)
+
+	c.Lock()
+	defer c.Unlock()
+
+	c.refreshIfRequired(false)
+
+	for _, specs := range c.specs {
+		for _, spec := range specs {
+			cmap[spec.GetClass()] = struct{}{}
+		}
+	}
+	for class := range cmap {
+		classes = append(classes, class)
+	}
+	sort.Strings(classes)
+
+	return classes
+}
+
+// GetVendorSpecs returns all specs for the given vendor.
+func (c *Cache) GetVendorSpecs(vendor string) []*Spec {
+	c.Lock()
+	defer c.Unlock()
+
+	c.refreshIfRequired(false)
+
+	return c.specs[vendor]
+}
+
+// GetSpecErrors returns all errors encountered for the spec during the
+// last cache refresh.
+func (c *Cache) GetSpecErrors(spec *Spec) []error {
+	var errors []error
+
+	c.Lock()
+	defer c.Unlock()
+
+	if errs, ok := c.errors[spec.GetPath()]; ok {
+		errors = make([]error, len(errs))
+		copy(errors, errs)
+	}
+
+	return errors
+}
+
+// GetErrors returns all errors encountered during the last
+// cache refresh.
+func (c *Cache) GetErrors() map[string][]error {
+	c.Lock()
+	defer c.Unlock()
+
+	errors := map[string][]error{}
+	for path, errs := range c.errors {
+		errors[path] = errs
+	}
+	for path, err := range c.dirErrors {
+		errors[path] = []error{err}
+	}
+
+	return errors
+}
+
+// GetSpecDirectories returns the CDI Spec directories currently in use.
+func (c *Cache) GetSpecDirectories() []string {
+	c.Lock()
+	defer c.Unlock()
+
+	dirs := make([]string, len(c.specDirs))
+	copy(dirs, c.specDirs)
+	return dirs
+}
+
+// GetSpecDirErrors returns any errors related to configured Spec directories.
+func (c *Cache) GetSpecDirErrors() map[string]error {
+	if c.dirErrors == nil {
+		return nil
+	}
+
+	c.Lock()
+	defer c.Unlock()
+
+	errors := make(map[string]error)
+	for dir, err := range c.dirErrors {
+		errors[dir] = err
+	}
+	return errors
+}
+
+// Our fsnotify helper wrapper.
+type watch struct {
+	watcher *fsnotify.Watcher
+	tracked map[string]bool
+}
+
+// Setup monitoring for the given Spec directories.
+func (w *watch) setup(dirs []string, dirErrors map[string]error) {
+	var (
+		dir string
+		err error
+	)
+	w.tracked = make(map[string]bool)
+	for _, dir = range dirs {
+		w.tracked[dir] = false
+	}
+
+	w.watcher, err = fsnotify.NewWatcher()
+	if err != nil {
+		for _, dir := range dirs {
+			dirErrors[dir] = fmt.Errorf("failed to create watcher: %w", err)
+		}
+		return
+	}
+
+	w.update(dirErrors)
+}
+
+// Start watching Spec directories for relevant changes.
+func (w *watch) start(m *sync.Mutex, refresh func() error, dirErrors map[string]error) {
+	go w.watch(w.watcher, m, refresh, dirErrors)
+}
+
+// Stop watching directories.
+func (w *watch) stop() {
+	if w.watcher == nil {
+		return
+	}
+
+	w.watcher.Close()
+	w.tracked = nil
+}
+
+// Watch Spec directory changes, triggering a refresh if necessary.
+func (w *watch) watch(fsw *fsnotify.Watcher, m *sync.Mutex, refresh func() error, dirErrors map[string]error) {
+	watch := fsw
+	if watch == nil {
+		return
+	}
+	for {
+		select {
+		case event, ok := <-watch.Events:
+			if !ok {
+				return
+			}
+
+			if (event.Op & (fsnotify.Rename | fsnotify.Remove | fsnotify.Write)) == 0 {
+				continue
+			}
+			if event.Op == fsnotify.Write {
+				if ext := filepath.Ext(event.Name); ext != ".json" && ext != ".yaml" {
+					continue
+				}
+			}
+
+			m.Lock()
+			if event.Op == fsnotify.Remove && w.tracked[event.Name] {
+				w.update(dirErrors, event.Name)
+			} else {
+				w.update(dirErrors)
+			}
+			refresh()
+			m.Unlock()
+
+		case _, ok := <-watch.Errors:
+			if !ok {
+				return
+			}
+		}
+	}
+}
+
+// Update watch with pending/missing or removed directories.
+func (w *watch) update(dirErrors map[string]error, removed ...string) bool {
+	var (
+		dir    string
+		ok     bool
+		err    error
+		update bool
+	)
+
+	for dir, ok = range w.tracked {
+		if ok {
+			continue
+		}
+
+		err = w.watcher.Add(dir)
+		if err == nil {
+			w.tracked[dir] = true
+			delete(dirErrors, dir)
+			update = true
+		} else {
+			w.tracked[dir] = false
+			dirErrors[dir] = fmt.Errorf("failed to monitor for changes: %w", err)
+		}
+	}
+
+	for _, dir = range removed {
+		w.tracked[dir] = false
+		dirErrors[dir] = errors.New("directory removed")
+		update = true
+	}
+
+	return update
+}

src/runtime/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/cache_test_unix.go

@@ -0,0 +1,26 @@
+//go:build !windows
+// +build !windows
+
+/*
+   Copyright © 2021 The CDI Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package cdi
+
+import "syscall"
+
+func osSync() {
+	syscall.Sync()
+}

src/runtime/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/cache_test_windows.go

@@ -0,0 +1,22 @@
+//go:build windows
+// +build windows
+
+/*
+   Copyright © 2021 The CDI Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package cdi
+
+func osSync() {}

src/runtime/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/container-edits.go

@@ -0,0 +1,332 @@
+/*
+   Copyright © 2021 The CDI Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package cdi
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+	oci "github.com/opencontainers/runtime-spec/specs-go"
+	ocigen "github.com/opencontainers/runtime-tools/generate"
+)
+
+const (
+	// PrestartHook is the name of the OCI "prestart" hook.
+	PrestartHook = "prestart"
+	// CreateRuntimeHook is the name of the OCI "createRuntime" hook.
+	CreateRuntimeHook = "createRuntime"
+	// CreateContainerHook is the name of the OCI "createContainer" hook.
+	CreateContainerHook = "createContainer"
+	// StartContainerHook is the name of the OCI "startContainer" hook.
+	StartContainerHook = "startContainer"
+	// PoststartHook is the name of the OCI "poststart" hook.
+	PoststartHook = "poststart"
+	// PoststopHook is the name of the OCI "poststop" hook.
+	PoststopHook = "poststop"
+)
+
+var (
+	// Names of recognized hooks.
+	validHookNames = map[string]struct{}{
+		PrestartHook:        {},
+		CreateRuntimeHook:   {},
+		CreateContainerHook: {},
+		StartContainerHook:  {},
+		PoststartHook:       {},
+		PoststopHook:        {},
+	}
+)
+
+// ContainerEdits represent updates to be applied to an OCI Spec.
+// These updates can be specific to a CDI device, or they can be
+// specific to a CDI Spec. In the former case these edits should
+// be applied to all OCI Specs where the corresponding CDI device
+// is injected. In the latter case, these edits should be applied
+// to all OCI Specs where at least one devices from the CDI Spec
+// is injected.
+type ContainerEdits struct {
+	*specs.ContainerEdits
+}
+
+// Apply edits to the given OCI Spec. Updates the OCI Spec in place.
+// Returns an error if the update fails.
+func (e *ContainerEdits) Apply(spec *oci.Spec) error {
+	if spec == nil {
+		return errors.New("can't edit nil OCI Spec")
+	}
+	if e == nil || e.ContainerEdits == nil {
+		return nil
+	}
+
+	specgen := ocigen.NewFromSpec(spec)
+	if len(e.Env) > 0 {
+		specgen.AddMultipleProcessEnv(e.Env)
+	}
+
+	for _, d := range e.DeviceNodes {
+		dn := DeviceNode{d}
+
+		err := dn.fillMissingInfo()
+		if err != nil {
+			return err
+		}
+		dev := d.ToOCI()
+		if dev.UID == nil && spec.Process != nil {
+			if uid := spec.Process.User.UID; uid > 0 {
+				dev.UID = &uid
+			}
+		}
+		if dev.GID == nil && spec.Process != nil {
+			if gid := spec.Process.User.GID; gid > 0 {
+				dev.GID = &gid
+			}
+		}
+
+		specgen.RemoveDevice(dev.Path)
+		specgen.AddDevice(dev)
+
+		if dev.Type == "b" || dev.Type == "c" {
+			access := d.Permissions
+			if access == "" {
+				access = "rwm"
+			}
+			specgen.AddLinuxResourcesDevice(true, dev.Type, &dev.Major, &dev.Minor, access)
+		}
+	}
+
+	if len(e.Mounts) > 0 {
+		for _, m := range e.Mounts {
+			specgen.RemoveMount(m.ContainerPath)
+			specgen.AddMount(m.ToOCI())
+		}
+		sortMounts(&specgen)
+	}
+
+	for _, h := range e.Hooks {
+		switch h.HookName {
+		case PrestartHook:
+			specgen.AddPreStartHook(h.ToOCI())
+		case PoststartHook:
+			specgen.AddPostStartHook(h.ToOCI())
+		case PoststopHook:
+			specgen.AddPostStopHook(h.ToOCI())
+			// TODO: Maybe runtime-tools/generate should be updated with these...
+		case CreateRuntimeHook:
+			ensureOCIHooks(spec)
+			spec.Hooks.CreateRuntime = append(spec.Hooks.CreateRuntime, h.ToOCI())
+		case CreateContainerHook:
+			ensureOCIHooks(spec)
+			spec.Hooks.CreateContainer = append(spec.Hooks.CreateContainer, h.ToOCI())
+		case StartContainerHook:
+			ensureOCIHooks(spec)
+			spec.Hooks.StartContainer = append(spec.Hooks.StartContainer, h.ToOCI())
+		default:
+			return fmt.Errorf("unknown hook name %q", h.HookName)
+		}
+	}
+
+	return nil
+}
+
+// Validate container edits.
+func (e *ContainerEdits) Validate() error {
+	if e == nil || e.ContainerEdits == nil {
+		return nil
+	}
+
+	if err := ValidateEnv(e.Env); err != nil {
+		return fmt.Errorf("invalid container edits: %w", err)
+	}
+	for _, d := range e.DeviceNodes {
+		if err := (&DeviceNode{d}).Validate(); err != nil {
+			return err
+		}
+	}
+	for _, h := range e.Hooks {
+		if err := (&Hook{h}).Validate(); err != nil {
+			return err
+		}
+	}
+	for _, m := range e.Mounts {
+		if err := (&Mount{m}).Validate(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Append other edits into this one. If called with a nil receiver,
+// allocates and returns newly allocated edits.
+func (e *ContainerEdits) Append(o *ContainerEdits) *ContainerEdits {
+	if o == nil || o.ContainerEdits == nil {
+		return e
+	}
+	if e == nil {
+		e = &ContainerEdits{}
+	}
+	if e.ContainerEdits == nil {
+		e.ContainerEdits = &specs.ContainerEdits{}
+	}
+
+	e.Env = append(e.Env, o.Env...)
+	e.DeviceNodes = append(e.DeviceNodes, o.DeviceNodes...)
+	e.Hooks = append(e.Hooks, o.Hooks...)
+	e.Mounts = append(e.Mounts, o.Mounts...)
+
+	return e
+}
+
+// isEmpty returns true if these edits are empty. This is valid in a
+// global Spec context but invalid in a Device context.
+func (e *ContainerEdits) isEmpty() bool {
+	if e == nil {
+		return false
+	}
+	return len(e.Env)+len(e.DeviceNodes)+len(e.Hooks)+len(e.Mounts) == 0
+}
+
+// ValidateEnv validates the given environment variables.
+func ValidateEnv(env []string) error {
+	for _, v := range env {
+		if strings.IndexByte(v, byte('=')) <= 0 {
+			return fmt.Errorf("invalid environment variable %q", v)
+		}
+	}
+	return nil
+}
+
+// DeviceNode is a CDI Spec DeviceNode wrapper, used for validating DeviceNodes.
+type DeviceNode struct {
+	*specs.DeviceNode
+}
+
+// Validate a CDI Spec DeviceNode.
+func (d *DeviceNode) Validate() error {
+	validTypes := map[string]struct{}{
+		"":  {},
+		"b": {},
+		"c": {},
+		"u": {},
+		"p": {},
+	}
+
+	if d.Path == "" {
+		return errors.New("invalid (empty) device path")
+	}
+	if _, ok := validTypes[d.Type]; !ok {
+		return fmt.Errorf("device %q: invalid type %q", d.Path, d.Type)
+	}
+	for _, bit := range d.Permissions {
+		if bit != 'r' && bit != 'w' && bit != 'm' {
+			return fmt.Errorf("device %q: invalid permissions %q",
+				d.Path, d.Permissions)
+		}
+	}
+	return nil
+}
+
+// Hook is a CDI Spec Hook wrapper, used for validating hooks.
+type Hook struct {
+	*specs.Hook
+}
+
+// Validate a hook.
+func (h *Hook) Validate() error {
+	if _, ok := validHookNames[h.HookName]; !ok {
+		return fmt.Errorf("invalid hook name %q", h.HookName)
+	}
+	if h.Path == "" {
+		return fmt.Errorf("invalid hook %q with empty path", h.HookName)
+	}
+	if err := ValidateEnv(h.Env); err != nil {
+		return fmt.Errorf("invalid hook %q: %w", h.HookName, err)
+	}
+	return nil
+}
+
+// Mount is a CDI Mount wrapper, used for validating mounts.
+type Mount struct {
+	*specs.Mount
+}
+
+// Validate a mount.
+func (m *Mount) Validate() error {
+	if m.HostPath == "" {
+		return errors.New("invalid mount, empty host path")
+	}
+	if m.ContainerPath == "" {
+		return errors.New("invalid mount, empty container path")
+	}
+	return nil
+}
+
+// Ensure OCI Spec hooks are not nil so we can add hooks.
+func ensureOCIHooks(spec *oci.Spec) {
+	if spec.Hooks == nil {
+		spec.Hooks = &oci.Hooks{}
+	}
+}
+
+// sortMounts sorts the mounts in the given OCI Spec.
+func sortMounts(specgen *ocigen.Generator) {
+	mounts := specgen.Mounts()
+	specgen.ClearMounts()
+	sort.Sort(orderedMounts(mounts))
+	specgen.Config.Mounts = mounts
+}
+
+// orderedMounts defines how to sort an OCI Spec Mount slice.
+// This is the almost the same implementation sa used by CRI-O and Docker,
+// with a minor tweak for stable sorting order (easier to test):
+//
+//	https://github.com/moby/moby/blob/17.05.x/daemon/volumes.go#L26
+type orderedMounts []oci.Mount
+
+// Len returns the number of mounts. Used in sorting.
+func (m orderedMounts) Len() int {
+	return len(m)
+}
+
+// Less returns true if the number of parts (a/b/c would be 3 parts) in the
+// mount indexed by parameter 1 is less than that of the mount indexed by
+// parameter 2. Used in sorting.
+func (m orderedMounts) Less(i, j int) bool {
+	ip, jp := m.parts(i), m.parts(j)
+	if ip < jp {
+		return true
+	}
+	if jp < ip {
+		return false
+	}
+	return m[i].Destination < m[j].Destination
+}
+
+// Swap swaps two items in an array of mounts. Used in sorting
+func (m orderedMounts) Swap(i, j int) {
+	m[i], m[j] = m[j], m[i]
+}
+
+// parts returns the number of parts in the destination of a mount. Used in sorting.
+func (m orderedMounts) parts(i int) int {
+	return strings.Count(filepath.Clean(m[i].Destination), string(os.PathSeparator))
+}

src/runtime/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/container-edits_unix.go

@@ -0,0 +1,57 @@
+//go:build !windows
+// +build !windows
+
+/*
+   Copyright © 2021 The CDI Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package cdi
+
+import (
+	"fmt"
+
+	runc "github.com/opencontainers/runc/libcontainer/devices"
+)
+
+// fillMissingInfo fills in missing mandatory attributes from the host device.
+func (d *DeviceNode) fillMissingInfo() error {
+	if d.HostPath == "" {
+		d.HostPath = d.Path
+	}
+
+	if d.Type != "" && (d.Major != 0 || d.Type == "p") {
+		return nil
+	}
+
+	hostDev, err := runc.DeviceFromPath(d.HostPath, "rwm")
+	if err != nil {
+		return fmt.Errorf("failed to stat CDI host device %q: %w", d.HostPath, err)
+	}
+
+	if d.Type == "" {
+		d.Type = string(hostDev.Type)
+	} else {
+		if d.Type != string(hostDev.Type) {
+			return fmt.Errorf("CDI device (%q, %q), host type mismatch (%s, %s)",
+				d.Path, d.HostPath, d.Type, string(hostDev.Type))
+		}
+	}
+	if d.Major == 0 && d.Type != "p" {
+		d.Major = hostDev.Major
+		d.Minor = hostDev.Minor
+	}
+
+	return nil
+}

src/runtime/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/container-edits_windows.go

@@ -0,0 +1,27 @@
+//go:build windows
+// +build windows
+
+/*
+   Copyright © 2021 The CDI Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package cdi
+
+import "fmt"
+
+// fillMissingInfo fills in missing mandatory attributes from the host device.
+func (d *DeviceNode) fillMissingInfo() error {
+	return fmt.Errorf("unimplemented")
+}

src/runtime/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/device.go

@@ -0,0 +1,88 @@
+/*
+   Copyright © 2021 The CDI Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package cdi
+
+import (
+	"fmt"
+
+	"github.com/container-orchestrated-devices/container-device-interface/internal/validation"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/parser"
+	cdi "github.com/container-orchestrated-devices/container-device-interface/specs-go"
+	oci "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+// Device represents a CDI device of a Spec.
+type Device struct {
+	*cdi.Device
+	spec *Spec
+}
+
+// Create a new Device, associate it with the given Spec.
+func newDevice(spec *Spec, d cdi.Device) (*Device, error) {
+	dev := &Device{
+		Device: &d,
+		spec:   spec,
+	}
+
+	if err := dev.validate(); err != nil {
+		return nil, err
+	}
+
+	return dev, nil
+}
+
+// GetSpec returns the Spec this device is defined in.
+func (d *Device) GetSpec() *Spec {
+	return d.spec
+}
+
+// GetQualifiedName returns the qualified name for this device.
+func (d *Device) GetQualifiedName() string {
+	return parser.QualifiedName(d.spec.GetVendor(), d.spec.GetClass(), d.Name)
+}
+
+// ApplyEdits applies the device-speific container edits to an OCI Spec.
+func (d *Device) ApplyEdits(ociSpec *oci.Spec) error {
+	return d.edits().Apply(ociSpec)
+}
+
+// edits returns the applicable container edits for this spec.
+func (d *Device) edits() *ContainerEdits {
+	return &ContainerEdits{&d.ContainerEdits}
+}
+
+// Validate the device.
+func (d *Device) validate() error {
+	if err := ValidateDeviceName(d.Name); err != nil {
+		return err
+	}
+	name := d.Name
+	if d.spec != nil {
+		name = d.GetQualifiedName()
+	}
+	if err := validation.ValidateSpecAnnotations(name, d.Annotations); err != nil {
+		return err
+	}
+	edits := d.edits()
+	if edits.isEmpty() {
+		return fmt.Errorf("invalid device, empty device edits")
+	}
+	if err := edits.Validate(); err != nil {
+		return fmt.Errorf("invalid device %q: %w", d.Name, err)
+	}
+	return nil
+}

src/runtime/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/doc.go

@@ -0,0 +1,272 @@
+// Package cdi has the primary purpose of providing an API for
+// interacting with CDI and consuming CDI devices.
+//
+// For more information about Container Device Interface, please refer to
+// https://github.com/container-orchestrated-devices/container-device-interface
+//
+// Container Device Interface
+//
+// Container Device Interface, or CDI for short, provides comprehensive
+// third party device support for container runtimes. CDI uses vendor
+// provided specification files, CDI Specs for short, to describe how a
+// container's runtime environment should be modified when one or more
+// of the vendor-specific devices is injected into the container. Beyond
+// describing the low level platform-specific details of how to gain
+// basic access to a device, CDI Specs allow more fine-grained device
+// initialization, and the automatic injection of any necessary vendor-
+// or device-specific software that might be required for a container
+// to use a device or take full advantage of it.
+//
+// In the CDI device model containers request access to a device using
+// fully qualified device names, qualified names for short, consisting of
+// a vendor identifier, a device class and a device name or identifier.
+// These pieces of information together uniquely identify a device among
+// all device vendors, classes and device instances.
+//
+// This package implements an API for easy consumption of CDI. The API
+// implements discovery, loading and caching of CDI Specs and injection
+// of CDI devices into containers. This is the most common functionality
+// the vast majority of CDI consumers need. The API should be usable both
+// by OCI runtime clients and runtime implementations.
+//
+// CDI Registry
+//
+// The primary interface to interact with CDI devices is the Registry. It
+// is essentially a cache of all Specs and devices discovered in standard
+// CDI directories on the host. The registry has two main functionality,
+// injecting devices into an OCI Spec and refreshing the cache of CDI
+// Specs and devices.
+//
+// Device Injection
+//
+// Using the Registry one can inject CDI devices into a container with code
+// similar to the following snippet:
+//
+//  import (
+//      "fmt"
+//      "strings"
+//
+//      log "github.com/sirupsen/logrus"
+//
+//      "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+//      oci "github.com/opencontainers/runtime-spec/specs-go"
+//  )
+//
+//  func injectCDIDevices(spec *oci.Spec, devices []string) error {
+//      log.Debug("pristine OCI Spec: %s", dumpSpec(spec))
+//
+//      unresolved, err := cdi.GetRegistry().InjectDevices(spec, devices)
+//      if err != nil {
+//          return fmt.Errorf("CDI device injection failed: %w", err)
+//      }
+//
+//      log.Debug("CDI-updated OCI Spec: %s", dumpSpec(spec))
+//      return nil
+//  }
+//
+// Cache Refresh
+//
+// By default the CDI Spec cache monitors the configured Spec directories
+// and automatically refreshes itself when necessary. This behavior can be
+// disabled using the WithAutoRefresh(false) option.
+//
+// Failure to set up monitoring for a Spec directory causes the directory to
+// get ignored and an error to be recorded among the Spec directory errors.
+// These errors can be queried using the GetSpecDirErrors() function. If the
+// error condition is transient, for instance a missing directory which later
+// gets created, the corresponding error will be removed once the condition
+// is over.
+//
+// With auto-refresh enabled injecting any CDI devices can be done without
+// an explicit call to Refresh(), using a code snippet similar to the
+// following:
+//
+// In a runtime implementation one typically wants to make sure the
+// CDI Spec cache is up to date before performing device injection.
+// A code snippet similar to the following accmplishes that:
+//
+//  import (
+//      "fmt"
+//      "strings"
+//
+//      log "github.com/sirupsen/logrus"
+//
+//      "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+//      oci "github.com/opencontainers/runtime-spec/specs-go"
+//  )
+//
+//  func injectCDIDevices(spec *oci.Spec, devices []string) error {
+//      registry := cdi.GetRegistry()
+//
+//      if err := registry.Refresh(); err != nil {
+//          // Note:
+//          //   It is up to the implementation to decide whether
+//          //   to abort injection on errors. A failed Refresh()
+//          //   does not necessarily render the registry unusable.
+//          //   For instance, a parse error in a Spec file for
+//          //   vendor A does not have any effect on devices of
+//          //   vendor B...
+//          log.Warnf("pre-injection Refresh() failed: %v", err)
+//      }
+//
+//      log.Debug("pristine OCI Spec: %s", dumpSpec(spec))
+//
+//      unresolved, err := registry.InjectDevices(spec, devices)
+//      if err != nil {
+//          return fmt.Errorf("CDI device injection failed: %w", err)
+//      }
+//
+//      log.Debug("CDI-updated OCI Spec: %s", dumpSpec(spec))
+//      return nil
+//  }
+//
+// Generated Spec Files, Multiple Directories, Device Precedence
+//
+// It is often necessary to generate Spec files dynamically. On some
+// systems the available or usable set of CDI devices might change
+// dynamically which then needs to be reflected in CDI Specs. For
+// some device classes it makes sense to enumerate the available
+// devices at every boot and generate Spec file entries for each
+// device found. Some CDI devices might need special client- or
+// request-specific configuration which can only be fulfilled by
+// dynamically generated client-specific entries in transient Spec
+// files.
+//
+// CDI can collect Spec files from multiple directories. Spec files are
+// automatically assigned priorities according to which directory they
+// were loaded from. The later a directory occurs in the list of CDI
+// directories to scan, the higher priority Spec files loaded from that
+// directory are assigned to. When two or more Spec files define the
+// same device, conflict is resolved by choosing the definition from the
+// Spec file with the highest priority.
+//
+// The default CDI directory configuration is chosen to encourage
+// separating dynamically generated CDI Spec files from static ones.
+// The default directories are '/etc/cdi' and '/var/run/cdi'. By putting
+// dynamically generated Spec files under '/var/run/cdi', those take
+// precedence over static ones in '/etc/cdi'. With this scheme, static
+// Spec files, typically installed by distro-specific packages, go into
+// '/etc/cdi' while all the dynamically generated Spec files, transient
+// or other, go into '/var/run/cdi'.
+//
+// Spec File Generation
+//
+// CDI offers two functions for writing and removing dynamically generated
+// Specs from CDI Spec directories. These functions, WriteSpec() and
+// RemoveSpec() implicitly follow the principle of separating dynamic Specs
+// from the rest and therefore always write to and remove Specs from the
+// last configured directory.
+//
+// Corresponding functions are also provided for generating names for Spec
+// files. These functions follow a simple naming convention to ensure that
+// multiple entities generating Spec files simultaneously on the same host
+// do not end up using conflicting Spec file names. GenerateSpecName(),
+// GenerateNameForSpec(), GenerateTransientSpecName(), and
+// GenerateTransientNameForSpec() all generate names which can be passed
+// as such to WriteSpec() and subsequently to RemoveSpec().
+//
+// Generating a Spec file for a vendor/device class can be done with a
+// code snippet similar to the following:
+//
+// import (
+//     "fmt"
+//     ...
+//     "github.com/container-orchestrated-devices/container-device-interface/specs-go"
+//     "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+// )
+//
+// func generateDeviceSpecs() error {
+//     registry := cdi.GetRegistry()
+//     spec := &specs.Spec{
+//         Version: specs.CurrentVersion,
+//         Kind:    vendor+"/"+class,
+//     }
+//
+//     for _, dev := range enumerateDevices() {
+//         spec.Devices = append(spec.Devices, specs.Device{
+//             Name: dev.Name,
+//             ContainerEdits: getContainerEditsForDevice(dev),
+//         })
+//     }
+//
+//     specName, err := cdi.GenerateNameForSpec(spec)
+//     if err != nil {
+//         return fmt.Errorf("failed to generate Spec name: %w", err)
+//     }
+//
+//     return registry.SpecDB().WriteSpec(spec, specName)
+// }
+//
+// Similarly, generating and later cleaning up transient Spec files can be
+// done with code fragments similar to the following. These transient Spec
+// files are temporary Spec files with container-specific parametrization.
+// They are typically created before the associated container is created
+// and removed once that container is removed.
+//
+// import (
+//     "fmt"
+//     ...
+//     "github.com/container-orchestrated-devices/container-device-interface/specs-go"
+//     "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+// )
+//
+// func generateTransientSpec(ctr Container) error {
+//     registry := cdi.GetRegistry()
+//     devices := getContainerDevs(ctr, vendor, class)
+//     spec := &specs.Spec{
+//         Version: specs.CurrentVersion,
+//         Kind:    vendor+"/"+class,
+//     }
+//
+//     for _, dev := range devices {
+//         spec.Devices = append(spec.Devices, specs.Device{
+//             // the generated name needs to be unique within the
+//             // vendor/class domain on the host/node.
+//             Name: generateUniqueDevName(dev, ctr),
+//             ContainerEdits: getEditsForContainer(dev),
+//         })
+//     }
+//
+//     // transientID is expected to guarantee that the Spec file name
+//     // generated using <vendor, class, transientID> is unique within
+//     // the host/node. If more than one device is allocated with the
+//     // same vendor/class domain, either all generated Spec entries
+//     // should go to a single Spec file (like in this sample snippet),
+//     // or transientID should be unique for each generated Spec file.
+//     transientID := getSomeSufficientlyUniqueIDForContainer(ctr)
+//     specName, err := cdi.GenerateNameForTransientSpec(vendor, class, transientID)
+//     if err != nil {
+//         return fmt.Errorf("failed to generate Spec name: %w", err)
+//     }
+//
+//     return registry.SpecDB().WriteSpec(spec, specName)
+// }
+//
+// func removeTransientSpec(ctr Container) error {
+//     registry := cdi.GetRegistry()
+//     transientID := getSomeSufficientlyUniqueIDForContainer(ctr)
+//     specName := cdi.GenerateNameForTransientSpec(vendor, class, transientID)
+//
+//     return registry.SpecDB().RemoveSpec(specName)
+// }
+//
+// CDI Spec Validation
+//
+// This package performs both syntactic and semantic validation of CDI
+// Spec file data when a Spec file is loaded via the registry or using
+// the ReadSpec API function. As part of the semantic verification, the
+// Spec file is verified against the CDI Spec JSON validation schema.
+//
+// If a valid externally provided JSON validation schema is found in
+// the filesystem at /etc/cdi/schema/schema.json it is loaded and used
+// as the default validation schema. If such a file is not found or
+// fails to load, an embedded no-op schema is used.
+//
+// The used validation schema can also be changed programmatically using
+// the SetSchema API convenience function. This function also accepts
+// the special "builtin" (BuiltinSchemaName) and "none" (NoneSchemaName)
+// schema names which switch the used schema to the in-repo validation
+// schema embedded into the binary or the now default no-op schema
+// correspondingly. Other names are interpreted as the path to the actual
+// validation schema to load and use.
+package cdi

src/runtime/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/qualified-device.go

@@ -0,0 +1,113 @@
+/*
+   Copyright © 2021 The CDI Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package cdi
+
+import (
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/parser"
+)
+
+// QualifiedName returns the qualified name for a device.
+// The syntax for a qualified device names is
+//
+//	"<vendor>/<class>=<name>".
+//
+// A valid vendor and class name may contain the following runes:
+//
+//	'A'-'Z', 'a'-'z', '0'-'9', '.', '-', '_'.
+//
+// A valid device name may contain the following runes:
+//
+//	'A'-'Z', 'a'-'z', '0'-'9', '-', '_', '.', ':'
+//
+// Deprecated: use parser.QualifiedName instead
+func QualifiedName(vendor, class, name string) string {
+	return parser.QualifiedName(vendor, class, name)
+}
+
+// IsQualifiedName tests if a device name is qualified.
+//
+// Deprecated: use parser.IsQualifiedName instead
+func IsQualifiedName(device string) bool {
+	return parser.IsQualifiedName(device)
+}
+
+// ParseQualifiedName splits a qualified name into device vendor, class,
+// and name. If the device fails to parse as a qualified name, or if any
+// of the split components fail to pass syntax validation, vendor and
+// class are returned as empty, together with the verbatim input as the
+// name and an error describing the reason for failure.
+//
+// Deprecated: use parser.ParseQualifiedName instead
+func ParseQualifiedName(device string) (string, string, string, error) {
+	return parser.ParseQualifiedName(device)
+}
+
+// ParseDevice tries to split a device name into vendor, class, and name.
+// If this fails, for instance in the case of unqualified device names,
+// ParseDevice returns an empty vendor and class together with name set
+// to the verbatim input.
+//
+// Deprecated: use parser.ParseDevice instead
+func ParseDevice(device string) (string, string, string) {
+	return parser.ParseDevice(device)
+}
+
+// ParseQualifier splits a device qualifier into vendor and class.
+// The syntax for a device qualifier is
+//
+//	"<vendor>/<class>"
+//
+// If parsing fails, an empty vendor and the class set to the
+// verbatim input is returned.
+//
+// Deprecated: use parser.ParseQualifier instead
+func ParseQualifier(kind string) (string, string) {
+	return parser.ParseQualifier(kind)
+}
+
+// ValidateVendorName checks the validity of a vendor name.
+// A vendor name may contain the following ASCII characters:
+//   - upper- and lowercase letters ('A'-'Z', 'a'-'z')
+//   - digits ('0'-'9')
+//   - underscore, dash, and dot ('_', '-', and '.')
+//
+// Deprecated: use parser.ValidateVendorName instead
+func ValidateVendorName(vendor string) error {
+	return parser.ValidateVendorName(vendor)
+}
+
+// ValidateClassName checks the validity of class name.
+// A class name may contain the following ASCII characters:
+//   - upper- and lowercase letters ('A'-'Z', 'a'-'z')
+//   - digits ('0'-'9')
+//   - underscore, dash, and dot ('_', '-', and '.')
+//
+// Deprecated: use parser.ValidateClassName instead
+func ValidateClassName(class string) error {
+	return parser.ValidateClassName(class)
+}
+
+// ValidateDeviceName checks the validity of a device name.
+// A device name may contain the following ASCII characters:
+//   - upper- and lowercase letters ('A'-'Z', 'a'-'z')
+//   - digits ('0'-'9')
+//   - underscore, dash, dot, colon ('_', '-', '.', ':')
+//
+// Deprecated: use parser.ValidateDeviceName instead
+func ValidateDeviceName(name string) error {
+	return parser.ValidateDeviceName(name)
+}

src/runtime/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/registry.go

@@ -0,0 +1,150 @@
+/*
+   Copyright © 2021 The CDI Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package cdi
+
+import (
+	"sync"
+
+	cdi "github.com/container-orchestrated-devices/container-device-interface/specs-go"
+	oci "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+// Registry keeps a cache of all CDI Specs installed or generated on
+// the host. Registry is the primary interface clients should use to
+// interact with CDI.
+//
+// The most commonly used Registry functions are for refreshing the
+// registry and injecting CDI devices into an OCI Spec.
+type Registry interface {
+	RegistryResolver
+	RegistryRefresher
+	DeviceDB() RegistryDeviceDB
+	SpecDB() RegistrySpecDB
+}
+
+// RegistryRefresher is the registry interface for refreshing the
+// cache of CDI Specs and devices.
+//
+// Configure reconfigures the registry with the given options.
+//
+// Refresh rescans all CDI Spec directories and updates the
+// state of the cache to reflect any changes. It returns any
+// errors encountered during the refresh.
+//
+// GetErrors returns all errors encountered for any of the scanned
+// Spec files during the last cache refresh.
+//
+// GetSpecDirectories returns the set up CDI Spec directories
+// currently in use. The directories are returned in the scan
+// order of Refresh().
+//
+// GetSpecDirErrors returns any errors related to the configured
+// Spec directories.
+type RegistryRefresher interface {
+	Configure(...Option) error
+	Refresh() error
+	GetErrors() map[string][]error
+	GetSpecDirectories() []string
+	GetSpecDirErrors() map[string]error
+}
+
+// RegistryResolver is the registry interface for injecting CDI
+// devices into an OCI Spec.
+//
+// InjectDevices takes an OCI Spec and injects into it a set of
+// CDI devices given by qualified name. It returns the names of
+// any unresolved devices and an error if injection fails.
+type RegistryResolver interface {
+	InjectDevices(spec *oci.Spec, device ...string) (unresolved []string, err error)
+}
+
+// RegistryDeviceDB is the registry interface for querying devices.
+//
+// GetDevice returns the CDI device for the given qualified name. If
+// the device is not GetDevice returns nil.
+//
+// ListDevices returns a slice with the names of qualified device
+// known. The returned slice is sorted.
+type RegistryDeviceDB interface {
+	GetDevice(device string) *Device
+	ListDevices() []string
+}
+
+// RegistrySpecDB is the registry interface for querying CDI Specs.
+//
+// ListVendors returns a slice with all vendors known. The returned
+// slice is sorted.
+//
+// ListClasses returns a slice with all classes known. The returned
+// slice is sorted.
+//
+// GetVendorSpecs returns a slice of all Specs for the vendor.
+//
+// GetSpecErrors returns any errors for the Spec encountered during
+// the last cache refresh.
+//
+// WriteSpec writes the Spec with the given content and name to the
+// last Spec directory.
+type RegistrySpecDB interface {
+	ListVendors() []string
+	ListClasses() []string
+	GetVendorSpecs(vendor string) []*Spec
+	GetSpecErrors(*Spec) []error
+	WriteSpec(raw *cdi.Spec, name string) error
+	RemoveSpec(name string) error
+}
+
+type registry struct {
+	*Cache
+}
+
+var _ Registry = &registry{}
+
+var (
+	reg      *registry
+	initOnce sync.Once
+)
+
+// GetRegistry returns the CDI registry. If any options are given, those
+// are applied to the registry.
+func GetRegistry(options ...Option) Registry {
+	var new bool
+	initOnce.Do(func() {
+		reg, _ = getRegistry(options...)
+		new = true
+	})
+	if !new && len(options) > 0 {
+		reg.Configure(options...)
+		reg.Refresh()
+	}
+	return reg
+}
+
+// DeviceDB returns the registry interface for querying devices.
+func (r *registry) DeviceDB() RegistryDeviceDB {
+	return r
+}
+
+// SpecDB returns the registry interface for querying Specs.
+func (r *registry) SpecDB() RegistrySpecDB {
+	return r
+}
+
+func getRegistry(options ...Option) (*registry, error) {
+	c, err := NewCache(options...)
+	return &registry{c}, err
+}

src/runtime/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/spec-dirs.go

@@ -0,0 +1,114 @@
+/*
+   Copyright © 2021 The CDI Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package cdi
+
+import (
+	"errors"
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+const (
+	// DefaultStaticDir is the default directory for static CDI Specs.
+	DefaultStaticDir = "/etc/cdi"
+	// DefaultDynamicDir is the default directory for generated CDI Specs
+	DefaultDynamicDir = "/var/run/cdi"
+)
+
+var (
+	// DefaultSpecDirs is the default Spec directory configuration.
+	// While altering this variable changes the package defaults,
+	// the preferred way of overriding the default directories is
+	// to use a WithSpecDirs options. Otherwise the change is only
+	// effective if it takes place before creating the Registry or
+	// other Cache instances.
+	DefaultSpecDirs = []string{DefaultStaticDir, DefaultDynamicDir}
+	// ErrStopScan can be returned from a ScanSpecFunc to stop the scan.
+	ErrStopScan = errors.New("stop Spec scan")
+)
+
+// WithSpecDirs returns an option to override the CDI Spec directories.
+func WithSpecDirs(dirs ...string) Option {
+	return func(c *Cache) error {
+		specDirs := make([]string, len(dirs))
+		for i, dir := range dirs {
+			specDirs[i] = filepath.Clean(dir)
+		}
+		c.specDirs = specDirs
+		return nil
+	}
+}
+
+// scanSpecFunc is a function for processing CDI Spec files.
+type scanSpecFunc func(string, int, *Spec, error) error
+
+// ScanSpecDirs scans the given directories looking for CDI Spec files,
+// which are all files with a '.json' or '.yaml' suffix. For every Spec
+// file discovered, ScanSpecDirs loads a Spec from the file then calls
+// the scan function passing it the path to the file, the priority (the
+// index of the directory in the slice of directories given), the Spec
+// itself, and any error encountered while loading the Spec.
+//
+// Scanning stops once all files have been processed or when the scan
+// function returns an error. The result of ScanSpecDirs is the error
+// returned by the scan function, if any. The special error ErrStopScan
+// can be used to terminate the scan gracefully without ScanSpecDirs
+// returning an error. ScanSpecDirs silently skips any subdirectories.
+func scanSpecDirs(dirs []string, scanFn scanSpecFunc) error {
+	var (
+		spec *Spec
+		err  error
+	)
+
+	for priority, dir := range dirs {
+		err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+			// for initial stat failure Walk calls us with nil info
+			if info == nil {
+				if errors.Is(err, fs.ErrNotExist) {
+					return nil
+				}
+				return err
+			}
+			// first call from Walk is for dir itself, others we skip
+			if info.IsDir() {
+				if path == dir {
+					return nil
+				}
+				return filepath.SkipDir
+			}
+
+			// ignore obviously non-Spec files
+			if ext := filepath.Ext(path); ext != ".json" && ext != ".yaml" {
+				return nil
+			}
+
+			if err != nil {
+				return scanFn(path, priority, nil, err)
+			}
+
+			spec, err = ReadSpec(path, priority)
+			return scanFn(path, priority, spec, err)
+		})
+
+		if err != nil && err != ErrStopScan {
+			return err
+		}
+	}
+
+	return nil
+}

src/runtime/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/spec.go

@@ -0,0 +1,352 @@
+/*
+   Copyright © 2021 The CDI Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package cdi
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	oci "github.com/opencontainers/runtime-spec/specs-go"
+	"sigs.k8s.io/yaml"
+
+	"github.com/container-orchestrated-devices/container-device-interface/internal/validation"
+	cdi "github.com/container-orchestrated-devices/container-device-interface/specs-go"
+)
+
+const (
+	// defaultSpecExt is the file extension for the default encoding.
+	defaultSpecExt = ".yaml"
+)
+
+var (
+	// Externally set CDI Spec validation function.
+	specValidator func(*cdi.Spec) error
+	validatorLock sync.RWMutex
+)
+
+// Spec represents a single CDI Spec. It is usually loaded from a
+// file and stored in a cache. The Spec has an associated priority.
+// This priority is inherited from the associated priority of the
+// CDI Spec directory that contains the CDI Spec file and is used
+// to resolve conflicts if multiple CDI Spec files contain entries
+// for the same fully qualified device.
+type Spec struct {
+	*cdi.Spec
+	vendor   string
+	class    string
+	path     string
+	priority int
+	devices  map[string]*Device
+}
+
+// ReadSpec reads the given CDI Spec file. The resulting Spec is
+// assigned the given priority. If reading or parsing the Spec
+// data fails ReadSpec returns a nil Spec and an error.
+func ReadSpec(path string, priority int) (*Spec, error) {
+	data, err := ioutil.ReadFile(path)
+	switch {
+	case os.IsNotExist(err):
+		return nil, err
+	case err != nil:
+		return nil, fmt.Errorf("failed to read CDI Spec %q: %w", path, err)
+	}
+
+	raw, err := ParseSpec(data)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse CDI Spec %q: %w", path, err)
+	}
+	if raw == nil {
+		return nil, fmt.Errorf("failed to parse CDI Spec %q, no Spec data", path)
+	}
+
+	spec, err := newSpec(raw, path, priority)
+	if err != nil {
+		return nil, err
+	}
+
+	return spec, nil
+}
+
+// newSpec creates a new Spec from the given CDI Spec data. The
+// Spec is marked as loaded from the given path with the given
+// priority. If Spec data validation fails newSpec returns a nil
+// Spec and an error.
+func newSpec(raw *cdi.Spec, path string, priority int) (*Spec, error) {
+	err := validateSpec(raw)
+	if err != nil {
+		return nil, err
+	}
+
+	spec := &Spec{
+		Spec:     raw,
+		path:     filepath.Clean(path),
+		priority: priority,
+	}
+
+	if ext := filepath.Ext(spec.path); ext != ".yaml" && ext != ".json" {
+		spec.path += defaultSpecExt
+	}
+
+	spec.vendor, spec.class = ParseQualifier(spec.Kind)
+
+	if spec.devices, err = spec.validate(); err != nil {
+		return nil, fmt.Errorf("invalid CDI Spec: %w", err)
+	}
+
+	return spec, nil
+}
+
+// Write the CDI Spec to the file associated with it during instantiation
+// by newSpec() or ReadSpec().
+func (s *Spec) write(overwrite bool) error {
+	var (
+		data []byte
+		dir  string
+		tmp  *os.File
+		err  error
+	)
+
+	err = validateSpec(s.Spec)
+	if err != nil {
+		return err
+	}
+
+	if filepath.Ext(s.path) == ".yaml" {
+		data, err = yaml.Marshal(s.Spec)
+		data = append([]byte("---\n"), data...)
+	} else {
+		data, err = json.Marshal(s.Spec)
+	}
+	if err != nil {
+		return fmt.Errorf("failed to marshal Spec file: %w", err)
+	}
+
+	dir = filepath.Dir(s.path)
+	err = os.MkdirAll(dir, 0o755)
+	if err != nil {
+		return fmt.Errorf("failed to create Spec dir: %w", err)
+	}
+
+	tmp, err = os.CreateTemp(dir, "spec.*.tmp")
+	if err != nil {
+		return fmt.Errorf("failed to create Spec file: %w", err)
+	}
+	_, err = tmp.Write(data)
+	tmp.Close()
+	if err != nil {
+		return fmt.Errorf("failed to write Spec file: %w", err)
+	}
+
+	err = renameIn(dir, filepath.Base(tmp.Name()), filepath.Base(s.path), overwrite)
+
+	if err != nil {
+		os.Remove(tmp.Name())
+		err = fmt.Errorf("failed to write Spec file: %w", err)
+	}
+
+	return err
+}
+
+// GetVendor returns the vendor of this Spec.
+func (s *Spec) GetVendor() string {
+	return s.vendor
+}
+
+// GetClass returns the device class of this Spec.
+func (s *Spec) GetClass() string {
+	return s.class
+}
+
+// GetDevice returns the device for the given unqualified name.
+func (s *Spec) GetDevice(name string) *Device {
+	return s.devices[name]
+}
+
+// GetPath returns the filesystem path of this Spec.
+func (s *Spec) GetPath() string {
+	return s.path
+}
+
+// GetPriority returns the priority of this Spec.
+func (s *Spec) GetPriority() int {
+	return s.priority
+}
+
+// ApplyEdits applies the Spec's global-scope container edits to an OCI Spec.
+func (s *Spec) ApplyEdits(ociSpec *oci.Spec) error {
+	return s.edits().Apply(ociSpec)
+}
+
+// edits returns the applicable global container edits for this spec.
+func (s *Spec) edits() *ContainerEdits {
+	return &ContainerEdits{&s.ContainerEdits}
+}
+
+// Validate the Spec.
+func (s *Spec) validate() (map[string]*Device, error) {
+	if err := validateVersion(s.Version); err != nil {
+		return nil, err
+	}
+
+	minVersion, err := MinimumRequiredVersion(s.Spec)
+	if err != nil {
+		return nil, fmt.Errorf("could not determine minimum required version: %v", err)
+	}
+	if newVersion(minVersion).IsGreaterThan(newVersion(s.Version)) {
+		return nil, fmt.Errorf("the spec version must be at least v%v", minVersion)
+	}
+
+	if err := ValidateVendorName(s.vendor); err != nil {
+		return nil, err
+	}
+	if err := ValidateClassName(s.class); err != nil {
+		return nil, err
+	}
+	if err := validation.ValidateSpecAnnotations(s.Kind, s.Annotations); err != nil {
+		return nil, err
+	}
+	if err := s.edits().Validate(); err != nil {
+		return nil, err
+	}
+
+	devices := make(map[string]*Device)
+	for _, d := range s.Devices {
+		dev, err := newDevice(s, d)
+		if err != nil {
+			return nil, fmt.Errorf("failed add device %q: %w", d.Name, err)
+		}
+		if _, conflict := devices[d.Name]; conflict {
+			return nil, fmt.Errorf("invalid spec, multiple device %q", d.Name)
+		}
+		devices[d.Name] = dev
+	}
+
+	return devices, nil
+}
+
+// validateVersion checks whether the specified spec version is supported.
+func validateVersion(version string) error {
+	if !validSpecVersions.isValidVersion(version) {
+		return fmt.Errorf("invalid version %q", version)
+	}
+
+	return nil
+}
+
+// ParseSpec parses CDI Spec data into a raw CDI Spec.
+func ParseSpec(data []byte) (*cdi.Spec, error) {
+	var raw *cdi.Spec
+	err := yaml.UnmarshalStrict(data, &raw)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal CDI Spec: %w", err)
+	}
+	return raw, nil
+}
+
+// SetSpecValidator sets a CDI Spec validator function. This function
+// is used for extra CDI Spec content validation whenever a Spec file
+// loaded (using ReadSpec() or written (using WriteSpec()).
+func SetSpecValidator(fn func(*cdi.Spec) error) {
+	validatorLock.Lock()
+	defer validatorLock.Unlock()
+	specValidator = fn
+}
+
+// validateSpec validates the Spec using the extneral validator.
+func validateSpec(raw *cdi.Spec) error {
+	validatorLock.RLock()
+	defer validatorLock.RUnlock()
+
+	if specValidator == nil {
+		return nil
+	}
+	err := specValidator(raw)
+	if err != nil {
+		return fmt.Errorf("Spec validation failed: %w", err)
+	}
+	return nil
+}
+
+// GenerateSpecName generates a vendor+class scoped Spec file name. The
+// name can be passed to WriteSpec() to write a Spec file to the file
+// system.
+//
+// vendor and class should match the vendor and class of the CDI Spec.
+// The file name is generated without a ".json" or ".yaml" extension.
+// The caller can append the desired extension to choose a particular
+// encoding. Otherwise WriteSpec() will use its default encoding.
+//
+// This function always returns the same name for the same vendor/class
+// combination. Therefore it cannot be used as such to generate multiple
+// Spec file names for a single vendor and class.
+func GenerateSpecName(vendor, class string) string {
+	return vendor + "-" + class
+}
+
+// GenerateTransientSpecName generates a vendor+class scoped transient
+// Spec file name. The name can be passed to WriteSpec() to write a Spec
+// file to the file system.
+//
+// Transient Specs are those whose lifecycle is tied to that of some
+// external entity, for instance a container. vendor and class should
+// match the vendor and class of the CDI Spec. transientID should be
+// unique among all CDI users on the same host that might generate
+// transient Spec files using the same vendor/class combination. If
+// the external entity to which the lifecycle of the transient Spec
+// is tied to has a unique ID of its own, then this is usually a
+// good choice for transientID.
+//
+// The file name is generated without a ".json" or ".yaml" extension.
+// The caller can append the desired extension to choose a particular
+// encoding. Otherwise WriteSpec() will use its default encoding.
+func GenerateTransientSpecName(vendor, class, transientID string) string {
+	transientID = strings.ReplaceAll(transientID, "/", "_")
+	return GenerateSpecName(vendor, class) + "_" + transientID
+}
+
+// GenerateNameForSpec generates a name for the given Spec using
+// GenerateSpecName with the vendor and class taken from the Spec.
+// On success it returns the generated name and a nil error. If
+// the Spec does not contain a valid vendor or class, it returns
+// an empty name and a non-nil error.
+func GenerateNameForSpec(raw *cdi.Spec) (string, error) {
+	vendor, class := ParseQualifier(raw.Kind)
+	if vendor == "" {
+		return "", fmt.Errorf("invalid vendor/class %q in Spec", raw.Kind)
+	}
+
+	return GenerateSpecName(vendor, class), nil
+}
+
+// GenerateNameForTransientSpec generates a name for the given transient
+// Spec using GenerateTransientSpecName with the vendor and class taken
+// from the Spec. On success it returns the generated name and a nil error.
+// If the Spec does not contain a valid vendor or class, it returns an
+// an empty name and a non-nil error.
+func GenerateNameForTransientSpec(raw *cdi.Spec, transientID string) (string, error) {
+	vendor, class := ParseQualifier(raw.Kind)
+	if vendor == "" {
+		return "", fmt.Errorf("invalid vendor/class %q in Spec", raw.Kind)
+	}
+
+	return GenerateTransientSpecName(vendor, class, transientID), nil
+}

src/runtime/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/spec_linux.go

@@ -0,0 +1,48 @@
+/*
+   Copyright © 2022 The CDI Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package cdi
+
+import (
+	"fmt"
+	"os"
+
+	"golang.org/x/sys/unix"
+)
+
+// Rename src to dst, both relative to the directory dir. If dst already exists
+// refuse renaming with an error unless overwrite is explicitly asked for.
+func renameIn(dir, src, dst string, overwrite bool) error {
+	var flags uint
+
+	dirf, err := os.Open(dir)
+	if err != nil {
+		return fmt.Errorf("rename failed: %w", err)
+	}
+	defer dirf.Close()
+
+	if !overwrite {
+		flags = unix.RENAME_NOREPLACE
+	}
+
+	dirFd := int(dirf.Fd())
+	err = unix.Renameat2(dirFd, src, dirFd, dst, flags)
+	if err != nil {
+		return fmt.Errorf("rename failed: %w", err)
+	}
+
+	return nil
+}

src/runtime/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/spec_other.go

@@ -0,0 +1,39 @@
+//go:build !linux
+// +build !linux
+
+/*
+   Copyright © 2022 The CDI Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package cdi
+
+import (
+	"os"
+	"path/filepath"
+)
+
+// Rename src to dst, both relative to the directory dir. If dst already exists
+// refuse renaming with an error unless overwrite is explicitly asked for.
+func renameIn(dir, src, dst string, overwrite bool) error {
+	src = filepath.Join(dir, src)
+	dst = filepath.Join(dir, dst)
+
+	_, err := os.Stat(dst)
+	if err == nil && !overwrite {
+		return os.ErrExist
+	}
+
+	return os.Rename(src, dst)
+}

src/runtime/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/version.go

@@ -0,0 +1,188 @@
+/*
+   Copyright © The CDI Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package cdi
+
+import (
+	"strings"
+
+	"golang.org/x/mod/semver"
+
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/parser"
+	cdi "github.com/container-orchestrated-devices/container-device-interface/specs-go"
+)
+
+const (
+	// CurrentVersion is the current version of the CDI Spec.
+	CurrentVersion = cdi.CurrentVersion
+
+	// vCurrent is the current version as a semver-comparable type
+	vCurrent version = "v" + CurrentVersion
+
+	// These represent the released versions of the CDI specification
+	v010 version = "v0.1.0"
+	v020 version = "v0.2.0"
+	v030 version = "v0.3.0"
+	v040 version = "v0.4.0"
+	v050 version = "v0.5.0"
+	v060 version = "v0.6.0"
+
+	// vEarliest is the earliest supported version of the CDI specification
+	vEarliest version = v030
+)
+
+// validSpecVersions stores a map of spec versions to functions to check the required versions.
+// Adding new fields / spec versions requires that a `requiredFunc` be implemented and
+// this map be updated.
+var validSpecVersions = requiredVersionMap{
+	v010: nil,
+	v020: nil,
+	v030: nil,
+	v040: requiresV040,
+	v050: requiresV050,
+	v060: requiresV060,
+}
+
+// MinimumRequiredVersion determines the minimum spec version for the input spec.
+func MinimumRequiredVersion(spec *cdi.Spec) (string, error) {
+	minVersion := validSpecVersions.requiredVersion(spec)
+	return minVersion.String(), nil
+}
+
+// version represents a semantic version string
+type version string
+
+// newVersion creates a version that can be used for semantic version comparisons.
+func newVersion(v string) version {
+	return version("v" + strings.TrimPrefix(v, "v"))
+}
+
+// String returns the string representation of the version.
+// This trims a leading v if present.
+func (v version) String() string {
+	return strings.TrimPrefix(string(v), "v")
+}
+
+// IsGreaterThan checks with a version is greater than the specified version.
+func (v version) IsGreaterThan(o version) bool {
+	return semver.Compare(string(v), string(o)) > 0
+}
+
+// IsLatest checks whether the version is the latest supported version
+func (v version) IsLatest() bool {
+	return v == vCurrent
+}
+
+type requiredFunc func(*cdi.Spec) bool
+
+type requiredVersionMap map[version]requiredFunc
+
+// isValidVersion checks whether the specified version is valid.
+// A version is valid if it is contained in the required version map.
+func (r requiredVersionMap) isValidVersion(specVersion string) bool {
+	_, ok := validSpecVersions[newVersion(specVersion)]
+
+	return ok
+}
+
+// requiredVersion returns the minimum version required for the given spec
+func (r requiredVersionMap) requiredVersion(spec *cdi.Spec) version {
+	minVersion := vEarliest
+
+	for v, isRequired := range validSpecVersions {
+		if isRequired == nil {
+			continue
+		}
+		if isRequired(spec) && v.IsGreaterThan(minVersion) {
+			minVersion = v
+		}
+		// If we have already detected the latest version then no later version could be detected
+		if minVersion.IsLatest() {
+			break
+		}
+	}
+
+	return minVersion
+}
+
+// requiresV060 returns true if the spec uses v0.6.0 features
+func requiresV060(spec *cdi.Spec) bool {
+	// The v0.6.0 spec allows annotations to be specified at a spec level
+	for range spec.Annotations {
+		return true
+	}
+
+	// The v0.6.0 spec allows annotations to be specified at a device level
+	for _, d := range spec.Devices {
+		for range d.Annotations {
+			return true
+		}
+	}
+
+	// The v0.6.0 spec allows dots "." in Kind name label (class)
+	vendor, class := parser.ParseQualifier(spec.Kind)
+	if vendor != "" {
+		if strings.ContainsRune(class, '.') {
+			return true
+		}
+	}
+
+	return false
+}
+
+// requiresV050 returns true if the spec uses v0.5.0 features
+func requiresV050(spec *cdi.Spec) bool {
+	var edits []*cdi.ContainerEdits
+
+	for _, d := range spec.Devices {
+		// The v0.5.0 spec allowed device names to start with a digit instead of requiring a letter
+		if len(d.Name) > 0 && !parser.IsLetter(rune(d.Name[0])) {
+			return true
+		}
+		edits = append(edits, &d.ContainerEdits)
+	}
+
+	edits = append(edits, &spec.ContainerEdits)
+	for _, e := range edits {
+		for _, dn := range e.DeviceNodes {
+			// The HostPath field was added in v0.5.0
+			if dn.HostPath != "" {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// requiresV040 returns true if the spec uses v0.4.0 features
+func requiresV040(spec *cdi.Spec) bool {
+	var edits []*cdi.ContainerEdits
+
+	for _, d := range spec.Devices {
+		edits = append(edits, &d.ContainerEdits)
+	}
+
+	edits = append(edits, &spec.ContainerEdits)
+	for _, e := range edits {
+		for _, m := range e.Mounts {
+			// The Type field was added in v0.4.0
+			if m.Type != "" {
+				return true
+			}
+		}
+	}
+	return false
+}

src/runtime/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/parser/parser.go

@@ -0,0 +1,212 @@
+/*
+   Copyright © The CDI Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package parser
+
+import (
+	"fmt"
+	"strings"
+)
+
+// QualifiedName returns the qualified name for a device.
+// The syntax for a qualified device names is
+//
+//	"<vendor>/<class>=<name>".
+//
+// A valid vendor and class name may contain the following runes:
+//
+//	'A'-'Z', 'a'-'z', '0'-'9', '.', '-', '_'.
+//
+// A valid device name may contain the following runes:
+//
+//	'A'-'Z', 'a'-'z', '0'-'9', '-', '_', '.', ':'
+func QualifiedName(vendor, class, name string) string {
+	return vendor + "/" + class + "=" + name
+}
+
+// IsQualifiedName tests if a device name is qualified.
+func IsQualifiedName(device string) bool {
+	_, _, _, err := ParseQualifiedName(device)
+	return err == nil
+}
+
+// ParseQualifiedName splits a qualified name into device vendor, class,
+// and name. If the device fails to parse as a qualified name, or if any
+// of the split components fail to pass syntax validation, vendor and
+// class are returned as empty, together with the verbatim input as the
+// name and an error describing the reason for failure.
+func ParseQualifiedName(device string) (string, string, string, error) {
+	vendor, class, name := ParseDevice(device)
+
+	if vendor == "" {
+		return "", "", device, fmt.Errorf("unqualified device %q, missing vendor", device)
+	}
+	if class == "" {
+		return "", "", device, fmt.Errorf("unqualified device %q, missing class", device)
+	}
+	if name == "" {
+		return "", "", device, fmt.Errorf("unqualified device %q, missing device name", device)
+	}
+
+	if err := ValidateVendorName(vendor); err != nil {
+		return "", "", device, fmt.Errorf("invalid device %q: %w", device, err)
+	}
+	if err := ValidateClassName(class); err != nil {
+		return "", "", device, fmt.Errorf("invalid device %q: %w", device, err)
+	}
+	if err := ValidateDeviceName(name); err != nil {
+		return "", "", device, fmt.Errorf("invalid device %q: %w", device, err)
+	}
+
+	return vendor, class, name, nil
+}
+
+// ParseDevice tries to split a device name into vendor, class, and name.
+// If this fails, for instance in the case of unqualified device names,
+// ParseDevice returns an empty vendor and class together with name set
+// to the verbatim input.
+func ParseDevice(device string) (string, string, string) {
+	if device == "" || device[0] == '/' {
+		return "", "", device
+	}
+
+	parts := strings.SplitN(device, "=", 2)
+	if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
+		return "", "", device
+	}
+
+	name := parts[1]
+	vendor, class := ParseQualifier(parts[0])
+	if vendor == "" {
+		return "", "", device
+	}
+
+	return vendor, class, name
+}
+
+// ParseQualifier splits a device qualifier into vendor and class.
+// The syntax for a device qualifier is
+//
+//	"<vendor>/<class>"
+//
+// If parsing fails, an empty vendor and the class set to the
+// verbatim input is returned.
+func ParseQualifier(kind string) (string, string) {
+	parts := strings.SplitN(kind, "/", 2)
+	if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
+		return "", kind
+	}
+	return parts[0], parts[1]
+}
+
+// ValidateVendorName checks the validity of a vendor name.
+// A vendor name may contain the following ASCII characters:
+//   - upper- and lowercase letters ('A'-'Z', 'a'-'z')
+//   - digits ('0'-'9')
+//   - underscore, dash, and dot ('_', '-', and '.')
+func ValidateVendorName(vendor string) error {
+	err := validateVendorOrClassName(vendor)
+	if err != nil {
+		err = fmt.Errorf("invalid vendor. %w", err)
+	}
+	return err
+}
+
+// ValidateClassName checks the validity of class name.
+// A class name may contain the following ASCII characters:
+//   - upper- and lowercase letters ('A'-'Z', 'a'-'z')
+//   - digits ('0'-'9')
+//   - underscore, dash, and dot ('_', '-', and '.')
+func ValidateClassName(class string) error {
+	err := validateVendorOrClassName(class)
+	if err != nil {
+		err = fmt.Errorf("invalid class. %w", err)
+	}
+	return err
+}
+
+// validateVendorOrClassName checks the validity of vendor or class name.
+// A name may contain the following ASCII characters:
+//   - upper- and lowercase letters ('A'-'Z', 'a'-'z')
+//   - digits ('0'-'9')
+//   - underscore, dash, and dot ('_', '-', and '.')
+func validateVendorOrClassName(name string) error {
+	if name == "" {
+		return fmt.Errorf("empty name")
+	}
+	if !IsLetter(rune(name[0])) {
+		return fmt.Errorf("%q, should start with letter", name)
+	}
+	for _, c := range string(name[1 : len(name)-1]) {
+		switch {
+		case IsAlphaNumeric(c):
+		case c == '_' || c == '-' || c == '.':
+		default:
+			return fmt.Errorf("invalid character '%c' in name %q",
+				c, name)
+		}
+	}
+	if !IsAlphaNumeric(rune(name[len(name)-1])) {
+		return fmt.Errorf("%q, should end with a letter or digit", name)
+	}
+
+	return nil
+}
+
+// ValidateDeviceName checks the validity of a device name.
+// A device name may contain the following ASCII characters:
+//   - upper- and lowercase letters ('A'-'Z', 'a'-'z')
+//   - digits ('0'-'9')
+//   - underscore, dash, dot, colon ('_', '-', '.', ':')
+func ValidateDeviceName(name string) error {
+	if name == "" {
+		return fmt.Errorf("invalid (empty) device name")
+	}
+	if !IsAlphaNumeric(rune(name[0])) {
+		return fmt.Errorf("invalid class %q, should start with a letter or digit", name)
+	}
+	if len(name) == 1 {
+		return nil
+	}
+	for _, c := range string(name[1 : len(name)-1]) {
+		switch {
+		case IsAlphaNumeric(c):
+		case c == '_' || c == '-' || c == '.' || c == ':':
+		default:
+			return fmt.Errorf("invalid character '%c' in device name %q",
+				c, name)
+		}
+	}
+	if !IsAlphaNumeric(rune(name[len(name)-1])) {
+		return fmt.Errorf("invalid name %q, should end with a letter or digit", name)
+	}
+	return nil
+}
+
+// IsLetter reports whether the rune is a letter.
+func IsLetter(c rune) bool {
+	return ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z')
+}
+
+// IsDigit reports whether the rune is a digit.
+func IsDigit(c rune) bool {
+	return '0' <= c && c <= '9'
+}
+
+// IsAlphaNumeric reports whether the rune is a letter or digit.
+func IsAlphaNumeric(c rune) bool {
+	return IsLetter(c) || IsDigit(c)
+}

src/runtime/vendor/github.com/container-orchestrated-devices/container-device-interface/specs-go/config.go

@@ -0,0 +1,62 @@
+package specs
+
+import "os"
+
+// CurrentVersion is the current version of the Spec.
+const CurrentVersion = "0.6.0"
+
+// Spec is the base configuration for CDI
+type Spec struct {
+	Version string `json:"cdiVersion"`
+	Kind    string `json:"kind"`
+	// Annotations add meta information per CDI spec. Note these are CDI-specific and do not affect container metadata.
+	Annotations    map[string]string `json:"annotations,omitempty"`
+	Devices        []Device          `json:"devices"`
+	ContainerEdits ContainerEdits    `json:"containerEdits,omitempty"`
+}
+
+// Device is a "Device" a container runtime can add to a container
+type Device struct {
+	Name string `json:"name"`
+	// Annotations add meta information per device. Note these are CDI-specific and do not affect container metadata.
+	Annotations    map[string]string `json:"annotations,omitempty"`
+	ContainerEdits ContainerEdits    `json:"containerEdits"`
+}
+
+// ContainerEdits are edits a container runtime must make to the OCI spec to expose the device.
+type ContainerEdits struct {
+	Env         []string      `json:"env,omitempty"`
+	DeviceNodes []*DeviceNode `json:"deviceNodes,omitempty"`
+	Hooks       []*Hook       `json:"hooks,omitempty"`
+	Mounts      []*Mount      `json:"mounts,omitempty"`
+}
+
+// DeviceNode represents a device node that needs to be added to the OCI spec.
+type DeviceNode struct {
+	Path        string       `json:"path"`
+	HostPath    string       `json:"hostPath,omitempty"`
+	Type        string       `json:"type,omitempty"`
+	Major       int64        `json:"major,omitempty"`
+	Minor       int64        `json:"minor,omitempty"`
+	FileMode    *os.FileMode `json:"fileMode,omitempty"`
+	Permissions string       `json:"permissions,omitempty"`
+	UID         *uint32      `json:"uid,omitempty"`
+	GID         *uint32      `json:"gid,omitempty"`
+}
+
+// Mount represents a mount that needs to be added to the OCI spec.
+type Mount struct {
+	HostPath      string   `json:"hostPath"`
+	ContainerPath string   `json:"containerPath"`
+	Options       []string `json:"options,omitempty"`
+	Type          string   `json:"type,omitempty"`
+}
+
+// Hook represents a hook that needs to be added to the OCI spec.
+type Hook struct {
+	HookName string   `json:"hookName"`
+	Path     string   `json:"path"`
+	Args     []string `json:"args,omitempty"`
+	Env      []string `json:"env,omitempty"`
+	Timeout  *int     `json:"timeout,omitempty"`
+}

src/runtime/vendor/github.com/container-orchestrated-devices/container-device-interface/specs-go/oci.go

@@ -0,0 +1,113 @@
+package specs
+
+import (
+	"errors"
+	"fmt"
+
+	spec "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+// ApplyOCIEditsForDevice applies devices OCI edits, in other words
+// it finds the device in the CDI spec and applies the OCI patches that device
+// requires to the OCI specification.
+func ApplyOCIEditsForDevice(config *spec.Spec, cdi *Spec, dev string) error {
+	for _, d := range cdi.Devices {
+		if d.Name != dev {
+			continue
+		}
+
+		return ApplyEditsToOCISpec(config, &d.ContainerEdits)
+	}
+
+	return fmt.Errorf("CDI: device %q not found for spec %q", dev, cdi.Kind)
+}
+
+// ApplyOCIEdits applies the OCI edits the CDI spec declares globally
+func ApplyOCIEdits(config *spec.Spec, cdi *Spec) error {
+	return ApplyEditsToOCISpec(config, &cdi.ContainerEdits)
+}
+
+// ApplyEditsToOCISpec applies the specified edits to the OCI spec.
+func ApplyEditsToOCISpec(config *spec.Spec, edits *ContainerEdits) error {
+	if config == nil {
+		return errors.New("spec is nil")
+	}
+	if edits == nil {
+		return nil
+	}
+
+	if len(edits.Env) > 0 {
+		if config.Process == nil {
+			config.Process = &spec.Process{}
+		}
+		config.Process.Env = append(config.Process.Env, edits.Env...)
+	}
+
+	for _, d := range edits.DeviceNodes {
+		if config.Linux == nil {
+			config.Linux = &spec.Linux{}
+		}
+		config.Linux.Devices = append(config.Linux.Devices, d.ToOCI())
+	}
+
+	for _, m := range edits.Mounts {
+		config.Mounts = append(config.Mounts, m.ToOCI())
+	}
+
+	for _, h := range edits.Hooks {
+		if config.Hooks == nil {
+			config.Hooks = &spec.Hooks{}
+		}
+		switch h.HookName {
+		case "prestart":
+			config.Hooks.Prestart = append(config.Hooks.Prestart, h.ToOCI())
+		case "createRuntime":
+			config.Hooks.CreateRuntime = append(config.Hooks.CreateRuntime, h.ToOCI())
+		case "createContainer":
+			config.Hooks.CreateContainer = append(config.Hooks.CreateContainer, h.ToOCI())
+		case "startContainer":
+			config.Hooks.StartContainer = append(config.Hooks.StartContainer, h.ToOCI())
+		case "poststart":
+			config.Hooks.Poststart = append(config.Hooks.Poststart, h.ToOCI())
+		case "poststop":
+			config.Hooks.Poststop = append(config.Hooks.Poststop, h.ToOCI())
+		default:
+			fmt.Printf("CDI: Unknown hook %q\n", h.HookName)
+		}
+	}
+
+	return nil
+}
+
+// ToOCI returns the opencontainers runtime Spec Hook for this Hook.
+func (h *Hook) ToOCI() spec.Hook {
+	return spec.Hook{
+		Path:    h.Path,
+		Args:    h.Args,
+		Env:     h.Env,
+		Timeout: h.Timeout,
+	}
+}
+
+// ToOCI returns the opencontainers runtime Spec Mount for this Mount.
+func (m *Mount) ToOCI() spec.Mount {
+	return spec.Mount{
+		Source:      m.HostPath,
+		Destination: m.ContainerPath,
+		Options:     m.Options,
+		Type:        m.Type,
+	}
+}
+
+// ToOCI returns the opencontainers runtime Spec LinuxDevice for this DeviceNode.
+func (d *DeviceNode) ToOCI() spec.LinuxDevice {
+	return spec.LinuxDevice{
+		Path:     d.Path,
+		Type:     d.Type,
+		Major:    d.Major,
+		Minor:    d.Minor,
+		FileMode: d.FileMode,
+		UID:      d.UID,
+		GID:      d.GID,
+	}
+}

src/runtime/vendor/github.com/containerd/containerd/plugin/plugin.go

@@ -78,6 +78,10 @@ const (
 	EventPlugin Type = "io.containerd.event.v1"
 	// TracingProcessorPlugin implements a open telemetry span processor
 	TracingProcessorPlugin Type = "io.containerd.tracing.processor.v1"
+	// CRIServicePlugin implements cri needed service
+	CRIServicePlugin Type = "io.containerd.cri.service.v1"
+	// CRIPlugin implements different cri implementations
+	CRIPlugin Type = "io.containerd.cri.v1"
 )
 
 const (

src/runtime/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go

@@ -105,11 +105,11 @@ type ttrpcService interface {
 	RegisterTTRPC(*ttrpc.Server) error
 }
 
-type taskService struct {
+type TaskService struct {
 	shimapi.TaskService
 }
 
-func (t taskService) RegisterTTRPC(server *ttrpc.Server) error {
+func (t TaskService) RegisterTTRPC(server *ttrpc.Server) error {
 	shimapi.RegisterTaskService(server, t.TaskService)
 	return nil
 }
@@ -289,7 +289,7 @@ func run(ctx context.Context, manager Manager, initFunc Init, name string, confi
 				plugin.EventPlugin,
 			},
 			InitFn: func(ic *plugin.InitContext) (interface{}, error) {
-				return taskService{service}, nil
+				return TaskService{service}, nil
 			},
 		})
 		manager = shimToManager{

src/runtime/vendor/github.com/containerd/containerd/runtime/v2/task/image.pb.go

@@ -0,0 +1,565 @@
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: image.proto
+
+package task
+
+import (
+	context "context"
+	fmt "fmt"
+	github_com_containerd_ttrpc "github.com/containerd/ttrpc"
+	proto "github.com/gogo/protobuf/proto"
+	io "io"
+	math "math"
+	math_bits "math/bits"
+	reflect "reflect"
+	strings "strings"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
+
+type PullImageRequest struct {
+	// Name of the image.
+	Image                string   `protobuf:"bytes,1,opt,name=image,proto3" json:"image,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *PullImageRequest) Reset()      { *m = PullImageRequest{} }
+func (*PullImageRequest) ProtoMessage() {}
+func (*PullImageRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_9624c68e2b547544, []int{0}
+}
+func (m *PullImageRequest) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *PullImageRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_PullImageRequest.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *PullImageRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_PullImageRequest.Merge(m, src)
+}
+func (m *PullImageRequest) XXX_Size() int {
+	return m.Size()
+}
+func (m *PullImageRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_PullImageRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_PullImageRequest proto.InternalMessageInfo
+
+type PullImageResponse struct {
+	// Reference to the image in use. For most runtimes, this should be an
+	// image ID or digest.
+	ImageRef             string   `protobuf:"bytes,1,opt,name=image_ref,json=imageRef,proto3" json:"image_ref,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *PullImageResponse) Reset()      { *m = PullImageResponse{} }
+func (*PullImageResponse) ProtoMessage() {}
+func (*PullImageResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_9624c68e2b547544, []int{1}
+}
+func (m *PullImageResponse) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *PullImageResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_PullImageResponse.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *PullImageResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_PullImageResponse.Merge(m, src)
+}
+func (m *PullImageResponse) XXX_Size() int {
+	return m.Size()
+}
+func (m *PullImageResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_PullImageResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_PullImageResponse proto.InternalMessageInfo
+
+func init() {
+	proto.RegisterType((*PullImageRequest)(nil), "containerd.task.v2.PullImageRequest")
+	proto.RegisterType((*PullImageResponse)(nil), "containerd.task.v2.PullImageResponse")
+}
+
+func init() { proto.RegisterFile("image.proto", fileDescriptor_9624c68e2b547544) }
+
+var fileDescriptor_9624c68e2b547544 = []byte{
+	// 213 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0xce, 0xcc, 0x4d, 0x4c,
+	0x4f, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x12, 0x4a, 0xce, 0xcf, 0x2b, 0x49, 0xcc, 0xcc,
+	0x4b, 0x2d, 0x4a, 0xd1, 0x2b, 0x49, 0x2c, 0xce, 0xd6, 0x2b, 0x33, 0x52, 0xd2, 0xe0, 0x12, 0x08,
+	0x28, 0xcd, 0xc9, 0xf1, 0x04, 0x29, 0x0b, 0x4a, 0x2d, 0x2c, 0x4d, 0x2d, 0x2e, 0x11, 0x12, 0xe1,
+	0x62, 0x05, 0x6b, 0x93, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x0c, 0x82, 0x70, 0x94, 0x0c, 0xb8, 0x04,
+	0x91, 0x54, 0x16, 0x17, 0xe4, 0xe7, 0x15, 0xa7, 0x0a, 0x49, 0x73, 0x71, 0x82, 0x65, 0xe3, 0x8b,
+	0x52, 0xd3, 0xa0, 0xca, 0x39, 0x32, 0x21, 0x2a, 0xd2, 0x8c, 0x92, 0xb9, 0x58, 0xc1, 0xaa, 0x85,
+	0xa2, 0xb8, 0x38, 0xe1, 0x5a, 0x85, 0x54, 0xf4, 0x30, 0x9d, 0xa1, 0x87, 0xee, 0x06, 0x29, 0x55,
+	0x02, 0xaa, 0x20, 0xf6, 0x2b, 0x31, 0x38, 0xf9, 0x9f, 0x78, 0x28, 0xc7, 0x70, 0xe3, 0xa1, 0x1c,
+	0x43, 0xc3, 0x23, 0x39, 0xc6, 0x13, 0x8f, 0xe4, 0x18, 0x2f, 0x3c, 0x92, 0x63, 0x7c, 0xf0, 0x48,
+	0x8e, 0x31, 0xca, 0x34, 0x3d, 0xb3, 0x24, 0xa3, 0x34, 0x49, 0x2f, 0x39, 0x3f, 0x57, 0x1f, 0x61,
+	0x18, 0x32, 0xb3, 0xa8, 0x34, 0xaf, 0x24, 0x33, 0x37, 0x55, 0xbf, 0xcc, 0x48, 0x1f, 0x64, 0x85,
+	0x35, 0x88, 0x48, 0x62, 0x03, 0x07, 0x96, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff, 0xf1, 0x9d, 0xd6,
+	0xbf, 0x3b, 0x01, 0x00, 0x00,
+}
+
+func (m *PullImageRequest) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *PullImageRequest) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *PullImageRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.XXX_unrecognized != nil {
+		i -= len(m.XXX_unrecognized)
+		copy(dAtA[i:], m.XXX_unrecognized)
+	}
+	if len(m.Image) > 0 {
+		i -= len(m.Image)
+		copy(dAtA[i:], m.Image)
+		i = encodeVarintImage(dAtA, i, uint64(len(m.Image)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+func (m *PullImageResponse) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *PullImageResponse) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *PullImageResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.XXX_unrecognized != nil {
+		i -= len(m.XXX_unrecognized)
+		copy(dAtA[i:], m.XXX_unrecognized)
+	}
+	if len(m.ImageRef) > 0 {
+		i -= len(m.ImageRef)
+		copy(dAtA[i:], m.ImageRef)
+		i = encodeVarintImage(dAtA, i, uint64(len(m.ImageRef)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+func encodeVarintImage(dAtA []byte, offset int, v uint64) int {
+	offset -= sovImage(v)
+	base := offset
+	for v >= 1<<7 {
+		dAtA[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	dAtA[offset] = uint8(v)
+	return base
+}
+func (m *PullImageRequest) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.Image)
+	if l > 0 {
+		n += 1 + l + sovImage(uint64(l))
+	}
+	if m.XXX_unrecognized != nil {
+		n += len(m.XXX_unrecognized)
+	}
+	return n
+}
+
+func (m *PullImageResponse) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.ImageRef)
+	if l > 0 {
+		n += 1 + l + sovImage(uint64(l))
+	}
+	if m.XXX_unrecognized != nil {
+		n += len(m.XXX_unrecognized)
+	}
+	return n
+}
+
+func sovImage(x uint64) (n int) {
+	return (math_bits.Len64(x|1) + 6) / 7
+}
+func sozImage(x uint64) (n int) {
+	return sovImage(uint64((x << 1) ^ uint64((int64(x) >> 63))))
+}
+func (this *PullImageRequest) String() string {
+	if this == nil {
+		return "nil"
+	}
+	s := strings.Join([]string{`&PullImageRequest{`,
+		`Image:` + fmt.Sprintf("%v", this.Image) + `,`,
+		`XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`,
+		`}`,
+	}, "")
+	return s
+}
+func (this *PullImageResponse) String() string {
+	if this == nil {
+		return "nil"
+	}
+	s := strings.Join([]string{`&PullImageResponse{`,
+		`ImageRef:` + fmt.Sprintf("%v", this.ImageRef) + `,`,
+		`XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`,
+		`}`,
+	}, "")
+	return s
+}
+func valueToStringImage(v interface{}) string {
+	rv := reflect.ValueOf(v)
+	if rv.IsNil() {
+		return "nil"
+	}
+	pv := reflect.Indirect(rv).Interface()
+	return fmt.Sprintf("*%v", pv)
+}
+
+type ImageService interface {
+	PullImage(ctx context.Context, req *PullImageRequest) (*PullImageResponse, error)
+}
+
+func RegisterImageService(srv *github_com_containerd_ttrpc.Server, svc ImageService) {
+	srv.Register("containerd.task.v2.Image", map[string]github_com_containerd_ttrpc.Method{
+		"PullImage": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) {
+			var req PullImageRequest
+			if err := unmarshal(&req); err != nil {
+				return nil, err
+			}
+			return svc.PullImage(ctx, &req)
+		},
+	})
+}
+
+type imageClient struct {
+	client *github_com_containerd_ttrpc.Client
+}
+
+func NewImageClient(client *github_com_containerd_ttrpc.Client) ImageService {
+	return &imageClient{
+		client: client,
+	}
+}
+
+func (c *imageClient) PullImage(ctx context.Context, req *PullImageRequest) (*PullImageResponse, error) {
+	var resp PullImageResponse
+	if err := c.client.Call(ctx, "containerd.task.v2.Image", "PullImage", req, &resp); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+func (m *PullImageRequest) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowImage
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: PullImageRequest: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: PullImageRequest: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Image", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowImage
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthImage
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthImage
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Image = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipImage(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthImage
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *PullImageResponse) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowImage
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: PullImageResponse: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: PullImageResponse: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field ImageRef", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowImage
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthImage
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthImage
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.ImageRef = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipImage(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthImage
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func skipImage(dAtA []byte) (n int, err error) {
+	l := len(dAtA)
+	iNdEx := 0
+	depth := 0
+	for iNdEx < l {
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return 0, ErrIntOverflowImage
+			}
+			if iNdEx >= l {
+				return 0, io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		wireType := int(wire & 0x7)
+		switch wireType {
+		case 0:
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowImage
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				iNdEx++
+				if dAtA[iNdEx-1] < 0x80 {
+					break
+				}
+			}
+		case 1:
+			iNdEx += 8
+		case 2:
+			var length int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowImage
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				length |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if length < 0 {
+				return 0, ErrInvalidLengthImage
+			}
+			iNdEx += length
+		case 3:
+			depth++
+		case 4:
+			if depth == 0 {
+				return 0, ErrUnexpectedEndOfGroupImage
+			}
+			depth--
+		case 5:
+			iNdEx += 4
+		default:
+			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
+		}
+		if iNdEx < 0 {
+			return 0, ErrInvalidLengthImage
+		}
+		if depth == 0 {
+			return iNdEx, nil
+		}
+	}
+	return 0, io.ErrUnexpectedEOF
+}
+
+var (
+	ErrInvalidLengthImage        = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowImage          = fmt.Errorf("proto: integer overflow")
+	ErrUnexpectedEndOfGroupImage = fmt.Errorf("proto: unexpected end of group")
+)

src/runtime/vendor/github.com/containerd/containerd/runtime/v2/task/image.proto

@@ -0,0 +1,37 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+syntax = "proto3";
+
+package containerd.task.v2;
+
+option go_package = "github.com/containerd/containerd/runtime/v2/task;task";
+
+// Image defines the public APIs for managing images.
+service Image {
+    // PullImage pulls an image with authentication config.
+    rpc PullImage(PullImageRequest) returns (PullImageResponse) {}
+}
+
+message PullImageRequest {
+    // Name of the image.
+    string image = 1;
+}
+
+message PullImageResponse {
+    // Reference to the image in use. For most runtimes, this should be an
+    // image ID or digest.
+    string image_ref = 1;
+}

src/runtime/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go

@@ -12,6 +12,8 @@ type Spec struct {
 	Root *Root `json:"root,omitempty"`
 	// Hostname configures the container's hostname.
 	Hostname string `json:"hostname,omitempty"`
+	// Domainname configures the container's domainname.
+	Domainname string `json:"domainname,omitempty"`
 	// Mounts configures additional mounts (on top of Root).
 	Mounts []Mount `json:"mounts,omitempty"`
 	// Hooks configures callbacks for container lifecycle events.
@@ -117,6 +119,11 @@ type Mount struct {
 	Source string `json:"source,omitempty"`
 	// Options are fstab style mount options.
 	Options []string `json:"options,omitempty"`
+
+	// UID/GID mappings used for changing file owners w/o calling chown, fs should support it.
+	// Every mount point could have its own mapping.
+	UIDMappings []LinuxIDMapping `json:"uidMappings,omitempty" platform:"linux"`
+	GIDMappings []LinuxIDMapping `json:"gidMappings,omitempty" platform:"linux"`
 }
 
 // Hook specifies a command that is run at a particular event in the lifecycle of a container
@@ -252,8 +259,8 @@ type LinuxInterfacePriority struct {
 	Priority uint32 `json:"priority"`
 }
 
-// linuxBlockIODevice holds major:minor format supported in blkio cgroup
-type linuxBlockIODevice struct {
+// LinuxBlockIODevice holds major:minor format supported in blkio cgroup
+type LinuxBlockIODevice struct {
 	// Major is the device's major number.
 	Major int64 `json:"major"`
 	// Minor is the device's minor number.
@@ -262,7 +269,7 @@ type linuxBlockIODevice struct {
 
 // LinuxWeightDevice struct holds a `major:minor weight` pair for weightDevice
 type LinuxWeightDevice struct {
-	linuxBlockIODevice
+	LinuxBlockIODevice
 	// Weight is the bandwidth rate for the device.
 	Weight *uint16 `json:"weight,omitempty"`
 	// LeafWeight is the bandwidth rate for the device while competing with the cgroup's child cgroups, CFQ scheduler only
@@ -271,7 +278,7 @@ type LinuxWeightDevice struct {
 
 // LinuxThrottleDevice struct holds a `major:minor rate_per_second` pair
 type LinuxThrottleDevice struct {
-	linuxBlockIODevice
+	LinuxBlockIODevice
 	// Rate is the IO rate limit per cgroup per device
 	Rate uint64 `json:"rate"`
 }
@@ -312,6 +319,10 @@ type LinuxMemory struct {
 	DisableOOMKiller *bool `json:"disableOOMKiller,omitempty"`
 	// Enables hierarchical memory accounting
 	UseHierarchy *bool `json:"useHierarchy,omitempty"`
+	// CheckBeforeUpdate enables checking if a new memory limit is lower
+	// than the current usage during update, and if so, rejecting the new
+	// limit.
+	CheckBeforeUpdate *bool `json:"checkBeforeUpdate,omitempty"`
 }
 
 // LinuxCPU for Linux cgroup 'cpu' resource management
@@ -320,6 +331,9 @@ type LinuxCPU struct {
 	Shares *uint64 `json:"shares,omitempty"`
 	// CPU hardcap limit (in usecs). Allowed cpu time in a given period.
 	Quota *int64 `json:"quota,omitempty"`
+	// CPU hardcap burst limit (in usecs). Allowed accumulated cpu time additionally for burst in a
+	// given period.
+	Burst *uint64 `json:"burst,omitempty"`
 	// CPU period to be used for hardcapping (in usecs).
 	Period *uint64 `json:"period,omitempty"`
 	// How much time realtime scheduling may use (in usecs).
@@ -330,6 +344,8 @@ type LinuxCPU struct {
 	Cpus string `json:"cpus,omitempty"`
 	// List of memory nodes in the cpuset. Default is to use any available memory node.
 	Mems string `json:"mems,omitempty"`
+	// cgroups are configured with minimum weight, 0: default behavior, 1: SCHED_IDLE.
+	Idle *int64 `json:"idle,omitempty"`
 }
 
 // LinuxPids for Linux cgroup 'pids' resource management (Linux 4.3)
@@ -524,11 +540,21 @@ type WindowsMemoryResources struct {
 
 // WindowsCPUResources contains CPU resource management settings.
 type WindowsCPUResources struct {
-	// Number of CPUs available to the container.
+	// Count is the number of CPUs available to the container. It represents the
+	// fraction of the configured processor `count` in a container in relation
+	// to the processors available in the host. The fraction ultimately
+	// determines the portion of processor cycles that the threads in a
+	// container can use during each scheduling interval, as the number of
+	// cycles per 10,000 cycles.
 	Count *uint64 `json:"count,omitempty"`
-	// CPU shares (relative weight to other containers with cpu shares).
+	// Shares limits the share of processor time given to the container relative
+	// to other workloads on the processor. The processor `shares` (`weight` at
+	// the platform level) is a value between 0 and 10000.
 	Shares *uint16 `json:"shares,omitempty"`
-	// Specifies the portion of processor cycles that this container can use as a percentage times 100.
+	// Maximum determines the portion of processor cycles that the threads in a
+	// container can use during each scheduling interval, as the number of
+	// cycles per 10,000 cycles. Set processor `maximum` to a percentage times
+	// 100.
 	Maximum *uint16 `json:"maximum,omitempty"`
 }
 
@@ -615,6 +641,23 @@ type Arch string
 // LinuxSeccompFlag is a flag to pass to seccomp(2).
 type LinuxSeccompFlag string
 
+const (
+	// LinuxSeccompFlagLog is a seccomp flag to request all returned
+	// actions except SECCOMP_RET_ALLOW to be logged. An administrator may
+	// override this filter flag by preventing specific actions from being
+	// logged via the /proc/sys/kernel/seccomp/actions_logged file. (since
+	// Linux 4.14)
+	LinuxSeccompFlagLog LinuxSeccompFlag = "SECCOMP_FILTER_FLAG_LOG"
+
+	// LinuxSeccompFlagSpecAllow can be used to disable Speculative Store
+	// Bypass mitigation. (since Linux 4.17)
+	LinuxSeccompFlagSpecAllow LinuxSeccompFlag = "SECCOMP_FILTER_FLAG_SPEC_ALLOW"
+
+	// LinuxSeccompFlagWaitKillableRecv can be used to switch to the wait
+	// killable semantics. (since Linux 5.19)
+	LinuxSeccompFlagWaitKillableRecv LinuxSeccompFlag = "SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV"
+)
+
 // Additional architectures permitted to be used for system calls
 // By default only the native architecture of the kernel is permitted
 const (