Merge pull request #6817 from fidencio/3.1.1-branch-bump

# Kata Containers 3.1.1

.github/workflows/add-backport-label.yaml

@@ -62,15 +62,15 @@ jobs:
           has_backport_needed_label=${{ contains(github.event.pull_request.labels.*.name, 'needs-backport') }}
           has_no_backport_needed_label=${{ contains(github.event.pull_request.labels.*.name, 'no-backport-needed') }}
 
-          echo "add_backport_label=false" >> $GITHUB_OUTPUT
+          echo "::set-output name=add_backport_label::false"
           if [ $has_backport_needed_label  = true ] || [ $has_bug  = true ]; then
             if [[ $has_no_backport_needed_label = false ]]; then
-              echo "add_backport_label=true" >> $GITHUB_OUTPUT
+              echo "::set-output name=add_backport_label::true"
             fi
           fi
 
           # Do not spam comment, only if auto-backport label is going to be newly added.
-          echo "auto_backport_added=$CONTAINS_AUTO_BACKPORT" >> $GITHUB_OUTPUT
+          echo "::set-output name=auto_backport_added::$CONTAINS_AUTO_BACKPORT"
 
       - name: Add comment
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && steps.add_label.outputs.add_backport_label == 'true' && steps.add_label.outputs.auto_backport_added == 'false' }}
@@ -97,4 +97,4 @@ jobs:
         uses: andymckay/labeler@e6c4322d0397f3240f0e7e30a33b5c5df2d39e90
         with:
           add-labels: "auto-backport"
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/build-kata-static-tarball-amd64.yaml

@@ -1,94 +0,0 @@
-name: CI | Build kata-static tarball for amd64
-on:
-  workflow_call:
-    inputs:
-      tarball-suffix:
-        required: false
-        type: string
-      push-to-registry:
-        required: false
-        type: string
-        default: no
-
-jobs:
-  build-asset:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        asset:
-          - cloud-hypervisor
-          - firecracker
-          - kernel
-          - kernel-sev
-          - kernel-dragonball-experimental
-          - kernel-tdx-experimental
-          - kernel-nvidia-gpu
-          - kernel-nvidia-gpu-snp
-          - kernel-nvidia-gpu-tdx-experimental
-          - nydus
-          - ovmf
-          - ovmf-sev
-          - qemu
-          - qemu-snp-experimental
-          - qemu-tdx-experimental
-          - rootfs-image
-          - rootfs-initrd
-          - rootfs-initrd-sev
-          - shim-v2
-          - tdvf
-          - virtiofsd
-    steps:
-      - name: Login to Kata Containers quay.io
-        if: ${{ inputs.push-to-registry == 'yes' }}
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0 # This is needed in order to keep the commit ids history
-
-      - name: Build ${{ matrix.asset }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
-
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts-amd64${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: ubuntu-latest
-    needs: build-asset
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-      - name: get-artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-artifacts-amd64${{ inputs.tarball-suffix }}
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
-          path: kata-static.tar.xz
-          retention-days: 1
-          if-no-files-found: error

.github/workflows/build-kata-static-tarball-arm64.yaml

@@ -1,90 +0,0 @@
-name: CI | Build kata-static tarball for arm64
-on:
-  workflow_call:
-    inputs:
-      tarball-suffix:
-        required: false
-        type: string
-      push-to-registry:
-        required: false
-        type: string
-        default: no
-
-jobs:
-  build-asset:
-    runs-on: arm64
-    strategy:
-      matrix:
-        asset:
-          - cloud-hypervisor
-          - firecracker
-          - kernel
-          - kernel-dragonball-experimental
-          - nydus
-          - qemu
-          - rootfs-image
-          - rootfs-initrd
-          - shim-v2
-          - virtiofsd
-    steps:
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - name: Login to Kata Containers quay.io
-        if: ${{ inputs.push-to-registry == 'yes' }}
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0 # This is needed in order to keep the commit ids history
-      - name: Build ${{ matrix.asset }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
-
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts-arm64${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: arm64
-    needs: build-asset
-    steps:
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-      - name: get-artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-artifacts-arm64${{ inputs.tarball-suffix }}
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-static-tarball-arm64${{ inputs.tarball-suffix }}
-          path: kata-static.tar.xz
-          retention-days: 1
-          if-no-files-found: error

.github/workflows/build-kata-static-tarball-s390x.yaml

@@ -1,87 +0,0 @@
-name: CI | Build kata-static tarball for s390x
-on:
-  workflow_call:
-    inputs:
-      tarball-suffix:
-        required: false
-        type: string
-      push-to-registry:
-        required: false
-        type: string
-        default: no
-
-jobs:
-  build-asset:
-    runs-on: s390x
-    strategy:
-      matrix:
-        asset:
-          - kernel
-          - qemu
-          - rootfs-image
-          - rootfs-initrd
-          - shim-v2
-          - virtiofsd
-    steps:
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - name: Login to Kata Containers quay.io
-        if: ${{ inputs.push-to-registry == 'yes' }}
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0 # This is needed in order to keep the commit ids history
-      - name: Build ${{ matrix.asset }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-          sudo chown -R $(id -u):$(id -g) "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
-
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts-s390x${{ inputs.tarball-suffix }}
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: s390x
-    needs: build-asset
-    steps:
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-      - name: get-artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-artifacts-s390x${{ inputs.tarball-suffix }}
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-static-tarball-s390x${{ inputs.tarball-suffix }}
-          path: kata-static.tar.xz
-          retention-days: 1
-          if-no-files-found: error

.github/workflows/cc-payload-after-push-amd64.yaml

@@ -1,161 +0,0 @@
-name: CI | Publish CC runtime payload for amd64
-on:
-  workflow_call:
-    inputs:
-      target-arch:
-        required: true
-        type: string
-
-jobs:
-  build-asset:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        asset:
-          - cc-cloud-hypervisor
-          - cc-kernel
-          - cc-qemu
-          - cc-rootfs-image
-          - cc-virtiofsd
-          - cc-sev-kernel
-          - cc-sev-ovmf
-          - cc-x86_64-ovmf
-          - cc-snp-qemu
-          - cc-sev-rootfs-initrd
-          - cc-tdx-kernel
-          - cc-tdx-rootfs-image
-          - cc-tdx-qemu
-          - cc-tdx-td-shim
-          - cc-tdx-tdvf
-    steps:
-      - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # This is needed in order to keep the commit ids history
-      - name: Build ${{ matrix.asset }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-          PUSH_TO_REGISTRY: yes
-
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-      - name: store-artifact root_hash_tdx.txt 
-        uses: actions/upload-artifact@v3
-        with:
-          name: root_hash_tdx.txt
-          path: tools/osbuilder/root_hash_tdx.txt
-          retention-days: 1
-          if-no-files-found: ignore
-
-      - name: store-artifact root_hash_vanilla.txt 
-        uses: actions/upload-artifact@v3
-        with:
-          name: root_hash_vanilla.txt
-          path: tools/osbuilder/root_hash_vanilla.txt
-          retention-days: 1
-          if-no-files-found: ignore
-
-  build-asset-cc-shim-v2:
-    runs-on: ubuntu-latest
-    needs: build-asset
-    steps:
-      - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - uses: actions/checkout@v3
-
-      - name: Get root_hash_tdx.txt
-        uses: actions/download-artifact@v3
-        with:
-          name: root_hash_tdx.txt
-          path: tools/osbuilder/
-
-      - name: Get root_hash_vanilla.txt
-        uses: actions/download-artifact@v3
-        with:
-          name: root_hash_vanilla.txt
-          path: tools/osbuilder/
-
-      - name: Build cc-shim-v2
-        run: |
-          make cc-shim-v2-tarball
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          PUSH_TO_REGISTRY: yes
-
-      - name: store-artifact cc-shim-v2
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-cc-shim-v2.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: ubuntu-latest
-    needs: [build-asset, build-asset-cc-shim-v2]
-    steps:
-      - uses: actions/checkout@v3
-      - name: get-artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-artifacts
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-static-tarball
-          path: kata-static.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  kata-payload:
-    needs: create-kata-tarball
-    runs-on: ubuntu-latest
-    steps:
-      - name: Login to Confidential Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
-
-      - uses: actions/checkout@v3
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball
-
-      - name: build-and-push-kata-payload
-        id: build-and-push-kata-payload
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz "quay.io/confidential-containers/runtime-payload-ci" \
-          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/cc-payload-after-push-s390x.yaml

@@ -1,164 +0,0 @@
-name: CI | Publish CC runtime payload for s390x
-on:
-  workflow_call:
-    inputs:
-      target-arch:
-        required: true
-        type: string
-
-jobs:
-  build-asset:
-    runs-on: s390x
-    strategy:
-      matrix:
-        asset:
-          - cc-kernel
-          - cc-qemu
-          - cc-rootfs-image
-          - cc-rootfs-initrd
-          - cc-se-image
-          - cc-virtiofsd
-    steps:
-      - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # This is needed in order to keep the commit ids history
-
-      - name: Place a host key document
-        run: |
-          mkdir -p "host-key-document"
-          cp "${CI_HKD_PATH}" "host-key-document"
-        env:
-          CI_HKD_PATH: ${{ secrets.CI_HKD_PATH }}
-
-      - name: Build ${{ matrix.asset }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-          sudo chown -R $(id -u):$(id -g) "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-          PUSH_TO_REGISTRY: yes
-          HKD_PATH: "host-key-document"
-
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts-s390x
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-      - name: store-artifact root_hash_vanilla.txt 
-        uses: actions/upload-artifact@v3
-        with:
-          name: root_hash_vanilla.txt-s390x
-          path: tools/osbuilder/root_hash_vanilla.txt
-          retention-days: 1
-          if-no-files-found: ignore
-
-  build-asset-cc-shim-v2:
-    runs-on: s390x
-    needs: build-asset
-    steps:
-      - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-
-      - name: Get root_hash_vanilla.txt
-        uses: actions/download-artifact@v3
-        with:
-          name: root_hash_vanilla.txt-s390x
-          path: tools/osbuilder/
-
-      - name: Build cc-shim-v2
-        run: |
-          make cc-shim-v2-tarball
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          PUSH_TO_REGISTRY: yes
-
-      - name: store-artifact cc-shim-v2
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts-s390x
-          path: kata-build/kata-static-cc-shim-v2.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: s390x
-    needs: [build-asset, build-asset-cc-shim-v2]
-    steps:
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-      - name: get-artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-artifacts-s390x
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-static-tarball-s390x
-          path: kata-static.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  kata-payload:
-    needs: create-kata-tarball
-    runs-on: s390x
-    steps:
-      - name: Login to Confidential Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
-
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-s390x
-
-      - name: build-and-push-kata-payload
-        id: build-and-push-kata-payload
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz "quay.io/confidential-containers/runtime-payload-ci" \
-          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/cc-payload-after-push.yaml

@@ -1,46 +0,0 @@
-name: CI | Publish Kata Containers payload for Confidential Containers
-on:
-  push:
-    branches:
-      - CCv0
-
-jobs:
-  build-assets-amd64:
-    uses: ./.github/workflows/cc-payload-after-push-amd64.yaml
-    with:
-      target-arch: amd64
-    secrets: inherit
-
-  build-assets-s390x:
-    uses: ./.github/workflows/cc-payload-after-push-s390x.yaml
-    with:
-      target-arch: s390x
-    secrets: inherit
-
-  publish:
-    runs-on: ubuntu-latest
-    needs: [build-assets-amd64, build-assets-s390x]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Login to Confidential Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
-
-      - name: Push commit multi-arch manifest
-        run: |
-          docker manifest create quay.io/confidential-containers/runtime-payload-ci:kata-containers-${GITHUB_SHA} \
-          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-${GITHUB_SHA}-amd64 \
-          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-${GITHUB_SHA}-s390x
-          docker manifest push quay.io/confidential-containers/runtime-payload-ci:kata-containers-${GITHUB_SHA}
-
-      - name: Push latest multi-arch manifest
-        run: |
-          docker manifest create quay.io/confidential-containers/runtime-payload-ci:kata-containers-latest \
-          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-amd64 \
-          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-s390x
-          docker manifest push quay.io/confidential-containers/runtime-payload-ci:kata-containers-latest

.github/workflows/cc-payload-amd64.yaml

@@ -1,144 +0,0 @@
-name: Publish Kata Containers payload for Confidential Containers (amd64)
-on:
-  workflow_call:
-    inputs:
-      target-arch:
-        required: true
-        type: string
-
-jobs:
-  build-asset:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        asset:
-          - cc-cloud-hypervisor
-          - cc-kernel
-          - cc-qemu
-          - cc-rootfs-image
-          - cc-virtiofsd
-          - cc-sev-kernel
-          - cc-sev-ovmf
-          - cc-x86_64-ovmf
-          - cc-snp-qemu
-          - cc-sev-rootfs-initrd
-          - cc-tdx-kernel
-          - cc-tdx-rootfs-image
-          - cc-tdx-qemu
-          - cc-tdx-td-shim
-          - cc-tdx-tdvf
-    steps:
-      - uses: actions/checkout@v3
-      - name: Build ${{ matrix.asset }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-      - name: store-artifact root_hash_tdx.txt 
-        uses: actions/upload-artifact@v3
-        with:
-          name: root_hash_tdx.txt
-          path: tools/osbuilder/root_hash_tdx.txt
-          retention-days: 1
-          if-no-files-found: ignore
-
-      - name: store-artifact root_hash_vanilla.txt 
-        uses: actions/upload-artifact@v3
-        with:
-          name: root_hash_vanilla.txt
-          path: tools/osbuilder/root_hash_vanilla.txt
-          retention-days: 1
-          if-no-files-found: ignore
-
-  build-asset-cc-shim-v2:
-    runs-on: ubuntu-latest
-    needs: build-asset
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Get root_hash_tdx.txt
-        uses: actions/download-artifact@v3
-        with:
-          name: root_hash_tdx.txt
-          path: tools/osbuilder/
-
-      - name: Get root_hash_vanilla.txt
-        uses: actions/download-artifact@v3
-        with:
-          name: root_hash_vanilla.txt
-          path: tools/osbuilder/
-
-      - name: Build cc-shim-v2
-        run: |
-          make cc-shim-v2-tarball
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-
-      - name: store-artifact cc-shim-v2
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-cc-shim-v2.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: ubuntu-latest
-    needs: [build-asset, build-asset-cc-shim-v2]
-    steps:
-      - uses: actions/checkout@v3
-      - name: get-artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-artifacts
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-static-tarball
-          path: kata-static.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  kata-payload:
-    needs: create-kata-tarball
-    runs-on: ubuntu-latest
-    steps:
-      - name: Login to quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
-
-      - uses: actions/checkout@v3
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball
-
-      - name: build-and-push-kata-payload
-        id: build-and-push-kata-payload
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz \
-          "quay.io/confidential-containers/runtime-payload" \
-          "kata-containers-${{ inputs.target-arch }}"
-

.github/workflows/cc-payload-s390x.yaml

@@ -1,134 +0,0 @@
-name: Publish Kata Containers payload for Confidential Containers (s390x)
-on:
-  workflow_call:
-    inputs:
-      target-arch:
-        required: true
-        type: string
-
-jobs:
-  build-asset:
-    runs-on: s390x
-    strategy:
-      matrix:
-        asset:
-          - cc-kernel
-          - cc-qemu
-          - cc-rootfs-image
-          - cc-virtiofsd
-    steps:
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-      - name: Build ${{ matrix.asset }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts-s390x
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-      - name: store-artifact root_hash_vanilla.txt 
-        uses: actions/upload-artifact@v3
-        with:
-          name: root_hash_vanilla.txt-s390x
-          path: tools/osbuilder/root_hash_vanilla.txt
-          retention-days: 1
-          if-no-files-found: ignore
-
-  build-asset-cc-shim-v2:
-    runs-on: s390x
-    needs: build-asset
-    steps:
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-
-      - name: Get root_hash_vanilla.txt
-        uses: actions/download-artifact@v3
-        with:
-          name: root_hash_vanilla.txt-s390x
-          path: tools/osbuilder/
-
-      - name: Build cc-shim-v2
-        run: |
-          make cc-shim-v2-tarball
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-
-      - name: store-artifact cc-shim-v2
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts-s390x
-          path: kata-build/kata-static-cc-shim-v2.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: s390x
-    needs: [build-asset, build-asset-cc-shim-v2]
-    steps:
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-      - name: get-artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-artifacts-s390x
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-static-tarball-s390x
-          path: kata-static.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  kata-payload:
-    needs: create-kata-tarball
-    runs-on: s390x
-    steps:
-      - name: Login to quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
-
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-s390x
-
-      - name: build-and-push-kata-payload
-        id: build-and-push-kata-payload
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz \
-          "quay.io/confidential-containers/runtime-payload" \
-          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/cc-payload.yaml

@@ -1,46 +0,0 @@
-name: Publish Kata Containers payload for Confidential Containers
-on:
-  push:
-    tags:
-      - 'CC\-[0-9]+.[0-9]+.[0-9]+'
-
-jobs:
-  build-assets-amd64:
-    uses: ./.github/workflows/cc-payload-amd64.yaml
-    with:
-      target-arch: amd64
-    secrets: inherit
-
-  build-assets-s390x:
-    uses: ./.github/workflows/cc-payload-s390x.yaml
-    with:
-      target-arch: s390x
-    secrets: inherit
-
-  publish:
-    runs-on: ubuntu-latest
-    needs: [build-assets-amd64, build-assets-s390x]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Login to Confidential Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
-
-      - name: Push commit multi-arch manifest
-        run: |
-          docker manifest create quay.io/confidential-containers/runtime-payload:kata-containers-${GITHUB_SHA} \
-          --amend quay.io/confidential-containers/runtime-payload:kata-containers-${GITHUB_SHA}-amd64 \
-          --amend quay.io/confidential-containers/runtime-payload:kata-containers-${GITHUB_SHA}-s390x
-          docker manifest push quay.io/confidential-containers/runtime-payload:kata-containers-${GITHUB_SHA}
-
-      - name: Push latest multi-arch manifest
-        run: |
-          docker manifest create quay.io/confidential-containers/runtime-payload:kata-containers-latest \
-          --amend quay.io/confidential-containers/runtime-payload:kata-containers-amd64 \
-          --amend quay.io/confidential-containers/runtime-payload:kata-containers-s390x
-          docker manifest push quay.io/confidential-containers/runtime-payload:kata-containers-latest

.github/workflows/ci-on-push.yaml

@@ -1,54 +0,0 @@
-name: Kata Containers CI
-on:
-  pull_request_target:
-    branches:
-      - 'main'
-
-jobs:
-  build-kata-static-tarball-amd64:
-    uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
-    with:
-      tarball-suffix: -${{ github.event.pull_request.number}}-${{ github.event.pull_request.head.sha }}
-
-  publish-kata-deploy-payload-amd64:
-    needs: build-kata-static-tarball-amd64
-    uses: ./.github/workflows/publish-kata-deploy-payload-amd64.yaml
-    with:
-      tarball-suffix: -${{ github.event.pull_request.number}}-${{ github.event.pull_request.head.sha }}
-      registry: ghcr.io
-      repo: ${{ github.repository_owner }}/kata-deploy-ci
-      tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64
-    secrets: inherit
-
-  run-k8s-tests-on-aks:
-    needs: publish-kata-deploy-payload-amd64
-    uses: ./.github/workflows/run-k8s-tests-on-aks.yaml
-    with:
-      registry: ghcr.io
-      repo: ${{ github.repository_owner }}/kata-deploy-ci
-      tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64
-    secrets: inherit
-
-  run-k8s-tests-on-sev:
-    needs: publish-kata-deploy-payload-amd64
-    uses: ./.github/workflows/run-k8s-tests-on-sev.yaml
-    with:
-      registry: ghcr.io
-      repo: ${{ github.repository_owner }}/kata-deploy-ci
-      tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64
-
-  run-k8s-tests-on-snp:
-    needs: publish-kata-deploy-payload-amd64
-    uses: ./.github/workflows/run-k8s-tests-on-snp.yaml
-    with:
-      registry: ghcr.io
-      repo: ${{ github.repository_owner }}/kata-deploy-ci
-      tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64
-
-  run-k8s-tests-on-tdx:
-    needs: publish-kata-deploy-payload-amd64
-    uses: ./.github/workflows/run-k8s-tests-on-tdx.yaml
-    with:
-      registry: ghcr.io
-      repo: ${{ github.repository_owner }}/kata-deploy-ci
-      tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64

.github/workflows/commit-message-check.yaml

@@ -47,7 +47,7 @@ jobs:
       uses: tim-actions/commit-message-checker-with-regex@v0.3.1
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
-        pattern: '^.{0,75}(\n.*)*$|^Merge pull request (?:kata-containers)?#[\d]+ from.*'
+        pattern: '^.{0,75}(\n.*)*$'
         error: 'Subject too long (max 75)'
         post_error: ${{ env.error_msg }}
 
@@ -62,9 +62,6 @@ jobs:
         #   to be specified at the start of the regex as the action is passed
         #   the entire commit message.
         #
-        # - This check will pass if the commit message only contains a subject
-        #   line, as other body message properties are enforced elsewhere.
-        #
         # - Body lines *can* be longer than the maximum if they start
         #   with a non-alphabetic character or if there is no whitespace in
         #   the line.
@@ -78,7 +75,7 @@ jobs:
         #
         # - A SoB comment can be any length (as it is unreasonable to penalise
         #   people with long names/email addresses :)
-        pattern: '(^[^\n]+$|^.+(\n([a-zA-Z].{0,150}|[^a-zA-Z\n].*|[^\s\n]*|Signed-off-by:.*|))+$)'
+        pattern: '^.+(\n([a-zA-Z].{0,150}|[^a-zA-Z\n].*|[^\s\n]*|Signed-off-by:.*|))+$'
         error: 'Body line too long (max 150)'
         post_error: ${{ env.error_msg }}
 
@@ -98,6 +95,6 @@ jobs:
       uses: tim-actions/commit-message-checker-with-regex@v0.3.1
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
-        pattern: '^[\s\t]*[^:\s\t]+[\s\t]*:|^Merge pull request (?:kata-containers)?#[\d]+ from.*'
+        pattern: '^[\s\t]*[^:\s\t]+[\s\t]*:'
         error: 'Failed to find subsystem in subject'
         post_error: ${{ env.error_msg }}

.github/workflows/kata-deploy-push.yaml

@@ -0,0 +1,80 @@
+name: kata deploy build
+
+on: 
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+    paths:
+      - tools/**
+      - versions.yaml
+
+jobs:
+  build-asset:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        asset:
+          - kernel
+          - kernel-dragonball-experimental
+          - shim-v2
+          - qemu
+          - cloud-hypervisor
+          - firecracker
+          - rootfs-image
+          - rootfs-initrd
+          - virtiofsd
+          - nydus
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build ${{ matrix.asset }}
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r --preserve=all "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: build-asset
+    steps:
+      - uses: actions/checkout@v2
+      - name: get-artifacts
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        uses: actions/download-artifact@v2
+        with:
+          name: kata-artifacts
+          path: build
+      - name: merge-artifacts
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        run: |
+          make merge-builds
+      - name: store-artifacts
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: kata-static-tarball
+          path: kata-static.tar.xz
+
+  make-kata-tarball:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: make kata-tarball
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        run: |
+          make kata-tarball
+          sudo make install-tarball

.github/workflows/kata-deploy-test.yaml

@@ -1,8 +1,14 @@
 on:
+  workflow_dispatch: # this is used to trigger the workflow on non-main branches
+    inputs:
+      pr:
+        description: 'PR number from the selected branch to test'
+        type: string
+        required: true
   issue_comment:
     types: [created, edited]
 
-name: deploy-ccv0-demo
+name: test-kata-deploy
 
 jobs:
   check-comment-and-membership:
@@ -11,20 +17,21 @@ jobs:
       github.event.issue.pull_request
       && github.event_name == 'issue_comment'
       && github.event.action == 'created'
-      && startsWith(github.event.comment.body, '/deploy-ccv0-demo')
+      && startsWith(github.event.comment.body, '/test_kata_deploy')
+      || github.event_name == 'workflow_dispatch'
     steps:
-      - name: Check membership
+      - name: Check membership on comment or dispatch
         uses: kata-containers/is-organization-member@1.0.1
         id: is_organization_member
         with:
           organization: kata-containers
-          username: ${{ github.event.comment.user.login }}
+          username: ${{ github.event.comment.user.login || github.event.sender.login }}
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Fail if not member
         run: |
           result=${{ steps.is_organization_member.outputs.result }}
           if [ $result == false ]; then
-              user=${{ github.event.comment.user.login }}
+              user=${{ github.event.comment.user.login || github.event.sender.login }}
               echo Either ${user} is not part of the kata-containers organization
               echo or ${user} has its Organization Visibility set to Private at
               echo https://github.com/orgs/kata-containers/people?query=${user}
@@ -43,22 +50,27 @@ jobs:
           - cloud-hypervisor
           - firecracker
           - kernel
+          - kernel-dragonball-experimental
+          - nydus
           - qemu
           - rootfs-image
           - rootfs-initrd
           - shim-v2
+          - virtiofsd
     steps:
-      - uses: actions/checkout@v2
-      - name: Prepare confidential container rootfs
-        if: ${{ matrix.asset == 'rootfs-initrd' }}
+      - name: get-PR-ref
+        id: get-PR-ref
         run: |
-          pushd include_rootfs/etc
-          curl -LO https://raw.githubusercontent.com/confidential-containers/documentation/main/demos/ssh-demo/aa-offline_fs_kbc-keys.json
-          mkdir kata-containers
-          envsubst < docs/how-to/data/confidential-agent-config.toml.in > kata-containers/agent.toml
-          popd
-        env:
-          AA_KBC_PARAMS: offline_fs_kbc::null
+            if [ ${{ github.event_name }} == 'issue_comment' ]; then
+                ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
+            else # workflow_dispatch
+                ref="refs/pull/${{ github.event.inputs.pr }}/merge"
+            fi
+            echo "reference for PR: " ${ref} "event:" ${{ github.event_name }}
+            echo "##[set-output name=pr-ref;]${ref}"
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
 
       - name: Build ${{ matrix.asset }}
         run: |
@@ -67,8 +79,6 @@ jobs:
           # store-artifact does not work with symlink
           sudo cp -r "${build_dir}" "kata-build"
         env:
-          AA_KBC: offline_fs_kbc
-          INCLUDE_ROOTFS: include_rootfs
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
 
@@ -83,7 +93,19 @@ jobs:
     runs-on: ubuntu-latest
     needs: build-asset
     steps:
+      - name: get-PR-ref
+        id: get-PR-ref
+        run: |
+            if [ ${{ github.event_name }} == 'issue_comment' ]; then
+                ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
+            else # workflow_dispatch
+                ref="refs/pull/${{ github.event.inputs.pr }}/merge"
+            fi
+            echo "reference for PR: " ${ref} "event:" ${{ github.event_name }}
+            echo "##[set-output name=pr-ref;]${ref}"
       - uses: actions/checkout@v2
+        with:
+          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
       - name: get-artifacts
         uses: actions/download-artifact@v2
         with:
@@ -102,7 +124,19 @@ jobs:
     needs: create-kata-tarball
     runs-on: ubuntu-latest
     steps:
+      - name: get-PR-ref
+        id: get-PR-ref
+        run: |
+            if [ ${{ github.event_name }} == 'issue_comment' ]; then
+                ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
+            else # workflow_dispatch
+                ref="refs/pull/${{ github.event.inputs.pr }}/merge"
+            fi
+            echo "reference for PR: " ${ref} "event:" ${{ github.event_name }}
+            echo "##[set-output name=pr-ref;]${ref}"
       - uses: actions/checkout@v2
+        with:
+          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
       - name: get-kata-tarball
         uses: actions/download-artifact@v2
         with:
@@ -110,15 +144,21 @@ jobs:
       - name: build-and-push-kata-deploy-ci
         id: build-and-push-kata-deploy-ci
         run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          pushd $GITHUB_WORKSPACE
-          git checkout $tag
-          pkg_sha=$(git rev-parse HEAD)
-          popd
+          PR_SHA=$(git log --format=format:%H -n1)
           mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
-          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t quay.io/confidential-containers/runtime-payload:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
+          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t quay.io/kata-containers/kata-deploy-ci:$PR_SHA $GITHUB_WORKSPACE/tools/packaging/kata-deploy
           docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
-          docker push quay.io/confidential-containers/runtime-payload:$pkg_sha
+          docker push quay.io/kata-containers/kata-deploy-ci:$PR_SHA
           mkdir -p packaging/kata-deploy
           ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
-          echo "::set-output name=PKG_SHA::${pkg_sha}"
+          echo "::set-output name=PKG_SHA::${PR_SHA}"
+      - name: test-kata-deploy-ci-in-aks
+        uses: ./packaging/kata-deploy/action
+        with:
+          packaging-sha: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
+        env:
+          PKG_SHA: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
+          AZ_APPID: ${{ secrets.AZ_APPID }}
+          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
+          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}

.github/workflows/payload-after-push.yaml

@@ -1,74 +0,0 @@
-name: CI | Publish Kata Containers payload
-on:
-  push:
-    branches:
-      - main
-      - stable-*
-
-jobs:
-  build-assets-amd64:
-    uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
-    with:
-      push-to-registry: yes
-    secrets: inherit
-
-  build-assets-arm64:
-    uses: ./.github/workflows/build-kata-static-tarball-arm64.yaml
-    with:
-      push-to-registry: yes
-    secrets: inherit
-
-  build-assets-s390x:
-    uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml
-    with:
-      push-to-registry: yes
-    secrets: inherit
-
-  publish-kata-deploy-payload-amd64:
-    needs: build-assets-amd64
-    uses: ./.github/workflows/publish-kata-deploy-payload-amd64.yaml
-    with:
-      registry: quay.io
-      repo: kata-containers/kata-deploy-ci
-      tag: kata-containers-amd64
-    secrets: inherit
-
-  publish-kata-deploy-payload-arm64:
-    needs: build-assets-arm64
-    uses: ./.github/workflows/publish-kata-deploy-payload-arm64.yaml
-    with:
-      registry: quay.io
-      repo: kata-containers/kata-deploy-ci
-      tag: kata-containers-arm64
-    secrets: inherit
-
-  publish-kata-deploy-payload-s390x:
-    needs: build-assets-s390x
-    uses: ./.github/workflows/publish-kata-deploy-payload-s390x.yaml
-    with:
-      registry: quay.io
-      repo: kata-containers/kata-deploy-ci
-      tag: kata-containers-s390x
-    secrets: inherit
-
-  publish-manifest:
-    runs-on: ubuntu-latest
-    needs: [publish-kata-deploy-payload-amd64, publish-kata-deploy-payload-arm64, publish-kata-deploy-payload-s390x]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - name: Push multi-arch manifest
-        run: |
-          docker manifest create quay.io/kata-containers/kata-deploy-ci:kata-containers-latest \
-          --amend quay.io/kata-containers/kata-deploy-ci:kata-containers-amd64 \
-          --amend quay.io/kata-containers/kata-deploy-ci:kata-containers-arm64 \
-          --amend quay.io/kata-containers/kata-deploy-ci:kata-containers-s390x
-          docker manifest push quay.io/kata-containers/kata-deploy-ci:kata-containers-latest

.github/workflows/publish-kata-deploy-payload-amd64.yaml

@@ -1,52 +0,0 @@
-name: CI | Publish kata-deploy payload for amd64
-on:
-  workflow_call:
-    inputs:
-      tarball-suffix:
-        required: false
-        type: string
-      registry:
-        required: true
-        type: string
-      repo:
-        required: true
-        type: string
-      tag:
-        required: true
-        type: string
-
-jobs:
-  kata-payload:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
-
-      - name: Login to Kata Containers quay.io
-        if: ${{ inputs.registry == 'quay.io' }}
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - name: Login to Kata Containers ghcr.io
-        if: ${{ inputs.registry == 'ghcr.io' }}
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: build-and-push-kata-payload
-        id: build-and-push-kata-payload
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz \
-          ${{ inputs.registry }}/${{ inputs.repo }} ${{ inputs.tag }}

.github/workflows/publish-kata-deploy-payload-arm64.yaml

@@ -1,57 +0,0 @@
-name: CI | Publish kata-deploy payload for arm64
-on:
-  workflow_call:
-    inputs:
-      tarball-suffix:
-        required: false
-        type: string
-      registry:
-        required: true
-        type: string
-      repo:
-        required: true
-        type: string
-      tag:
-        required: true
-        type: string
-
-jobs:
-  kata-payload:
-    runs-on: arm64
-    steps:
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-arm64${{ inputs.tarball-suffix }}
-
-      - name: Login to Kata Containers quay.io
-        if: ${{ inputs.registry == 'quay.io' }}
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - name: Login to Kata Containers ghcr.io
-        if: ${{ inputs.registry == 'ghcr.io' }}
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: build-and-push-kata-payload
-        id: build-and-push-kata-payload
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz \
-          ${{ inputs.registry }}/${{ inputs.repo }} ${{ inputs.tag }}
-

.github/workflows/publish-kata-deploy-payload-s390x.yaml

@@ -1,56 +0,0 @@
-name: CI | Publish kata-deploy payload for s390x
-on:
-  workflow_call:
-    inputs:
-      tarball-suffix:
-        required: false
-        type: string
-      registry:
-        required: true
-        type: string
-      repo:
-        required: true
-        type: string
-      tag:
-        required: true
-        type: string
-
-jobs:
-  kata-payload:
-    runs-on: s390x
-    steps:
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-s390x${{ inputs.tarball-suffix }}
-
-      - name: Login to Kata Containers quay.io
-        if: ${{ inputs.registry == 'quay.io' }}
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - name: Login to Kata Containers ghcr.io
-        if: ${{ inputs.registry == 'ghcr.io' }}
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: build-and-push-kata-payload
-        id: build-and-push-kata-payload
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz \
-          ${{ inputs.registry }}/${{ inputs.repo }} ${{ inputs.tag }}

.github/workflows/release-amd64.yaml

@@ -1,51 +0,0 @@
-name: Publish Kata release artifacts for amd64
-on:
-  workflow_call:
-    inputs:
-      target-arch:
-        required: true
-        type: string
-
-jobs:
-  build-kata-static-tarball-amd64:
-    uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
-
-  kata-deploy:
-    needs: build-kata-static-tarball-amd64
-    runs-on: ubuntu-latest
-    steps:
-      - name: Login to Kata Containers docker.io
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - uses: actions/checkout@v3
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-amd64
-
-      - name: build-and-push-kata-deploy-ci-amd64
-        id: build-and-push-kata-deploy-ci-amd64
-        run: |
-          # We need to do such trick here as the format of the $GITHUB_REF 
-          # is "refs/tags/<tag>"
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          tags=($tag)
-          tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
-          for tag in ${tags[@]}; do
-              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
-                  "${tag}-${{ inputs.target-arch }}"
-              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
-                  "${tag}-${{ inputs.target-arch }}"
-          done

.github/workflows/release-arm64.yaml

@@ -1,51 +0,0 @@
-name: Publish Kata release artifacts for arm64
-on:
-  workflow_call:
-    inputs:
-      target-arch:
-        required: true
-        type: string
-
-jobs:
-  build-kata-static-tarball-arm64:
-    uses: ./.github/workflows/build-kata-static-tarball-arm64.yaml
-
-  kata-deploy:
-    needs: build-kata-static-tarball-arm64
-    runs-on: arm64
-    steps:
-      - name: Login to Kata Containers docker.io
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - uses: actions/checkout@v3
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-arm64
-
-      - name: build-and-push-kata-deploy-ci-arm64
-        id: build-and-push-kata-deploy-ci-arm64
-        run: |
-          # We need to do such trick here as the format of the $GITHUB_REF 
-          # is "refs/tags/<tag>"
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          tags=($tag)
-          tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
-          for tag in ${tags[@]}; do
-              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
-                  "${tag}-${{ inputs.target-arch }}"
-              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
-                  "${tag}-${{ inputs.target-arch }}"
-          done

.github/workflows/release-s390x.yaml

@@ -1,51 +0,0 @@
-name: Publish Kata release artifacts for s390x
-on:
-  workflow_call:
-    inputs:
-      target-arch:
-        required: true
-        type: string
-
-jobs:
-  build-kata-static-tarball-s390x:
-    uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml
-
-  kata-deploy:
-    needs: build-kata-static-tarball-s390x
-    runs-on: s390x
-    steps:
-      - name: Login to Kata Containers docker.io
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - uses: actions/checkout@v3
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-s390x
-
-      - name: build-and-push-kata-deploy-ci-s390x
-        id: build-and-push-kata-deploy-ci-s390x
-        run: |
-          # We need to do such trick here as the format of the $GITHUB_REF 
-          # is "refs/tags/<tag>"
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          tags=($tag)
-          tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
-          for tag in ${tags[@]}; do
-              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
-                  "${tag}-${{ inputs.target-arch }}"
-              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-                  $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
-                  "${tag}-${{ inputs.target-arch }}"
-          done

.github/workflows/release.yaml

@@ -5,82 +5,124 @@ on:
       - '[0-9]+.[0-9]+.[0-9]+*'
 
 jobs:
-  build-and-push-assets-amd64:
-    uses: ./.github/workflows/release-amd64.yaml
-    with:
-      target-arch: amd64
-    secrets: inherit
-
-  build-and-push-assets-arm64:
-    uses: ./.github/workflows/release-arm64.yaml
-    with:
-      target-arch: arm64
-    secrets: inherit
-
-  build-and-push-assets-s390x:
-    uses: ./.github/workflows/release-s390x.yaml
-    with:
-      target-arch: s390x
-    secrets: inherit
-
-  publish-multi-arch-images:
+  build-asset:
     runs-on: ubuntu-latest
-    needs: [build-and-push-assets-amd64, build-and-push-assets-arm64, build-and-push-assets-s390x]
+    strategy:
+      matrix:
+        asset:
+          - cloud-hypervisor
+          - firecracker
+          - kernel
+          - kernel-dragonball-experimental
+          - nydus
+          - qemu
+          - rootfs-image
+          - rootfs-initrd
+          - shim-v2
+          - virtiofsd
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v2
+      - name: Build ${{ matrix.asset }}
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh --build="${KATA_ASSET}"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
 
-      - name: Login to Kata Containers docker.io
-        uses: docker/login-action@v2
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v2
         with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
+          name: kata-artifacts
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          if-no-files-found: error
 
-      - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v2
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: build-asset
+    steps:
+      - uses: actions/checkout@v2
+      - name: get-artifacts
+        uses: actions/download-artifact@v2
         with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+          name: kata-artifacts
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: kata-static-tarball
+          path: kata-static.tar.xz
 
-      - name: Push multi-arch manifest
+  kata-deploy:
+    needs: create-kata-tarball
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v2
+        with:
+          name: kata-static-tarball
+      - name: build-and-push-kata-deploy-ci
+        id: build-and-push-kata-deploy-ci
+        run: |
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          pushd $GITHUB_WORKSPACE
+          git checkout $tag
+          pkg_sha=$(git rev-parse HEAD)
+          popd
+          mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
+          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:$pkg_sha -t quay.io/kata-containers/kata-deploy-ci:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
+          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
+          docker push katadocker/kata-deploy-ci:$pkg_sha
+          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
+          docker push quay.io/kata-containers/kata-deploy-ci:$pkg_sha
+          mkdir -p packaging/kata-deploy
+          ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
+          echo "::set-output name=PKG_SHA::${pkg_sha}"
+      - name: test-kata-deploy-ci-in-aks
+        uses: ./packaging/kata-deploy/action
+        with:
+          packaging-sha: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
+        env:
+          PKG_SHA: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
+          AZ_APPID: ${{ secrets.AZ_APPID }}
+          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
+          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
+      - name: push-tarball
         run: |
           # tag the container image we created and push to DockerHub
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
           tags=($tag)
           tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
-          # push to quay.io and docker.io
-          for tag in ${tags[@]}; do
-            docker manifest create quay.io/kata-containers/kata-deploy:${tag} \
-              --amend quay.io/kata-containers/kata-deploy:${tag}-amd64 \
-              --amend quay.io/kata-containers/kata-deploy:${tag}-arm64 \
-              --amend quay.io/kata-containers/kata-deploy:${tag}-s390x
-
-            docker manifest create docker.io/katadocker/kata-deploy:${tag} \
-              --amend docker.io/katadocker/kata-deploy:${tag}-amd64 \
-              --amend docker.io/katadocker/kata-deploy:${tag}-arm64 \
-              --amend docker.io/katadocker/kata-deploy:${tag}-s390x
-
-            docker manifest push quay.io/kata-containers/kata-deploy:${tag}
-            docker manifest push docker.io/katadocker/kata-deploy:${tag}
+          for tag in ${tags[@]}; do \
+            docker tag katadocker/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} katadocker/kata-deploy:${tag} && \
+            docker tag quay.io/kata-containers/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} quay.io/kata-containers/kata-deploy:${tag} && \
+            docker push katadocker/kata-deploy:${tag} && \
+            docker push quay.io/kata-containers/kata-deploy:${tag}; \
           done
 
-  upload-multi-arch-static-tarball:
-    needs: publish-multi-arch-images
+  upload-static-tarball:
+    needs: kata-deploy
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
+      - name: download-artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: kata-static-tarball
       - name: install hub
         run: |
           HUB_VER=$(curl -s "https://api.github.com/repos/github/hub/releases/latest" | jq -r .tag_name | sed 's/^v//')
           wget -q -O- https://github.com/github/hub/releases/download/v$HUB_VER/hub-linux-amd64-$HUB_VER.tgz | \
           tar xz --strip-components=2 --wildcards '*/bin/hub' && sudo mv hub /usr/local/bin/hub
-
-      - name: download-artifacts-amd64
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-amd64
-      - name: push amd64 static tarball to github
+      - name: push static tarball to github
         run: |
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
           tarball="kata-static-$tag-x86_64.tar.xz"
@@ -90,39 +132,11 @@ jobs:
           GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}"
           popd
 
-      - name: download-artifacts-arm64
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-arm64
-      - name: push arm64 static tarball to github
-        run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          tarball="kata-static-$tag-aarch64.tar.xz"
-          mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
-          pushd $GITHUB_WORKSPACE
-          echo "uploading asset '${tarball}' for tag: ${tag}"
-          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}"
-          popd
-
-      - name: download-artifacts-s390x
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-s390x
-      - name: push s390x static tarball to github
-        run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          tarball="kata-static-$tag-s390x.tar.xz"
-          mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
-          pushd $GITHUB_WORKSPACE
-          echo "uploading asset '${tarball}' for tag: ${tag}"
-          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}"
-          popd
-
   upload-cargo-vendored-tarball:
-    needs: upload-multi-arch-static-tarball
+    needs: upload-static-tarball
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
       - name: generate-and-upload-tarball
         run: |
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
@@ -136,7 +150,7 @@ jobs:
     needs: upload-cargo-vendored-tarball
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
       - name: download-and-upload-tarball
         env:
           GITHUB_TOKEN: ${{ secrets.GIT_UPLOAD_TOKEN }}

.github/workflows/run-k8s-tests-on-aks.yaml

@@ -1,95 +0,0 @@
-name: CI | Run kubernetes tests on AKS
-on:
-  workflow_call:
-    inputs:
-      registry:
-        required: true
-        type: string
-      repo:
-        required: true
-        type: string
-      tag:
-        required: true
-        type: string
-
-jobs:
-  run-k8s-tests:
-    strategy:
-      fail-fast: false
-      matrix:
-        vmm:
-          - clh
-          - dragonball
-          - qemu
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-
-      - name: Download Azure CLI
-        run: |
-          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
-
-      - name: Log into the Azure account
-        run: |
-          az login \
-            --service-principal \
-            -u "${{ secrets.AZ_APPID }}" \
-            -p "${{ secrets.AZ_PASSWORD }}" \
-            --tenant "${{ secrets.AZ_TENANT_ID }}"
-
-      - name: Create AKS cluster
-        run: |
-          az aks create \
-            -g "kataCI" \
-            -n "${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-amd64" \
-            -s "Standard_D4s_v5" \
-            --node-count 1 \
-            --generate-ssh-keys
-
-      - name: Install `bats`
-        run: |
-          sudo apt-get update
-          sudo apt-get -y install bats
-
-      - name: Install `kubectl`
-        run: |
-          sudo az aks install-cli
-
-      - name: Download credentials for the Kubernetes CLI to use them
-        run: |
-          az aks get-credentials -g "kataCI" -n ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-amd64
-
-      - name: Run tests
-        timeout-minutes: 35
-        run: |
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-
-          kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
-          kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
-
-          # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
-          # which may cause issues like not having the node properly labeled or the artefacts
-          # properly deployed when the tests actually start running.
-          sleep 150s
-
-          pushd tests/integration/kubernetes
-          sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
-          bash run_kubernetes_tests.sh
-          popd
-        env:
-          KATA_HYPERVISOR: ${{ matrix.vmm }}
-
-      - name: Delete AKS cluster
-        if: always()
-        run: |
-          az aks delete \
-            -g "kataCI" \
-            -n "${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-amd64" \
-            --yes \
-            --no-wait

.github/workflows/run-k8s-tests-on-sev.yaml

@@ -1,68 +0,0 @@
-name: CI | Run kubernetes tests on SEV
-on:
-  workflow_call:
-    inputs:
-      registry:
-        required: true
-        type: string
-      repo:
-        required: true
-        type: string
-      tag:
-        required: true
-        type: string
-
-jobs:
-  run-k8s-tests:
-    strategy:
-      fail-fast: false
-      matrix:
-        vmm:
-          - qemu-sev
-    runs-on: sev
-    env:
-      KUBECONFIG: /home/kata/.kube/config
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-
-      - name: Run tests
-        timeout-minutes: 30
-        run: |
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-
-          kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
-          kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
-
-          # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
-          # which may cause issues like not having the node properly labeled or the artefacts
-          # properly deployed when the tests actually start running.
-          sleep 60s
-
-          pushd tests/integration/kubernetes
-          sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
-          bash run_kubernetes_tests.sh
-          popd
-        env:
-          KATA_HYPERVISOR: ${{ matrix.vmm }}
-
-      - name: Delete kata-deploy
-        if: always()
-        run: |
-          kubectl delete -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
-
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-          kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          sleep 180s
-
-          kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml

.github/workflows/run-k8s-tests-on-snp.yaml

@@ -1,68 +0,0 @@
-name: CI | Run kubernetes tests on SEV-SNP
-on:
-  workflow_call:
-    inputs:
-      registry:
-        required: true
-        type: string
-      repo:
-        required: true
-        type: string
-      tag:
-        required: true
-        type: string
-
-jobs:
-  run-k8s-tests:
-    strategy:
-      fail-fast: false
-      matrix:
-        vmm:
-          - qemu-snp
-    runs-on: sev-snp
-    env:
-      KUBECONFIG: /home/kata/.kube/config
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-
-      - name: Run tests
-        timeout-minutes: 30
-        run: |
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-
-          kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
-          kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
-
-          # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
-          # which may cause issues like not having the node properly labeled or the artefacts
-          # properly deployed when the tests actually start running.
-          sleep 60s
-
-          pushd tests/integration/kubernetes
-          sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
-          bash run_kubernetes_tests.sh
-          popd
-        env:
-          KATA_HYPERVISOR: ${{ matrix.vmm }}
-
-      - name: Delete kata-deploy
-        if: always()
-        run: |
-          kubectl delete -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
-
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-          kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          sleep 180s
-
-          kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml

.github/workflows/run-k8s-tests-on-tdx.yaml

@@ -1,68 +0,0 @@
-name: CI | Run kubernetes tests on TDX
-on:
-  workflow_call:
-    inputs:
-      registry:
-        required: true
-        type: string
-      repo:
-        required: true
-        type: string
-      tag:
-        required: true
-        type: string
-
-jobs:
-  run-k8s-tests:
-    strategy:
-      fail-fast: false
-      matrix:
-        vmm:
-          - qemu-tdx
-    runs-on: tdx
-    env:
-      KUBECONFIG: /etc/rancher/k3s/k3s.yaml
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-
-      - name: Run tests
-        timeout-minutes: 30
-        run: |
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-
-          kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl apply -k tools/packaging/kata-deploy/kata-deploy/overlays/k3s
-          kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
-          kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
-
-          # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
-          # which may cause issues like not having the node properly labeled or the artefacts
-          # properly deployed when the tests actually start running.
-          sleep 60s
-
-          pushd tests/integration/kubernetes
-          sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
-          bash run_kubernetes_tests.sh
-          popd
-        env:
-          KATA_HYPERVISOR: ${{ matrix.vmm }}
-
-      - name: Delete kata-deploy
-        if: always()
-        run: |
-          kubectl delete -k tools/packaging/kata-deploy/kata-deploy/overlays/k3s
-          kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
-
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-          kubectl apply -k tools/packaging/kata-deploy/kata-cleanup/overlays/k3s
-          sleep 180s
-
-          kubectl delete -k tools/packaging/kata-deploy/kata-cleanup/overlays/k3s
-          kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml

.github/workflows/snap-release.yaml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Check out Git repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
         with:
           fetch-depth: 0
 
@@ -32,7 +32,6 @@ jobs:
         run: |
           # Removing man-db, workflow kept failing, fixes: #4480
           sudo apt -y remove --purge man-db
-          sudo apt-get update
           sudo apt-get install -y git git-extras
           kata_url="https://github.com/kata-containers/kata-containers"
           latest_version=$(git ls-remote --tags ${kata_url}  | egrep -o "refs.*" | egrep -v "\-alpha|\-rc|{}" | egrep -o "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+" | sort -V -r | head -1)

.github/workflows/snap.yaml

@@ -14,7 +14,7 @@ jobs:
     steps:
       - name: Check out
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
         with:
           fetch-depth: 0
 

.github/workflows/static-checks.yaml

@@ -19,14 +19,13 @@ jobs:
           - "make test"
           - "sudo -E PATH=\"$PATH\" make test"
     env:
+      TRAVIS: "true"
+      TRAVIS_BRANCH: ${{ github.base_ref }}
+      TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }}
+      TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }}
       RUST_BACKTRACE: "1"
       target_branch: ${{ github.base_ref }}
-      GOPATH: ${{ github.workspace }}
     steps:
-    - name: Free disk space
-      run: |
-        sudo rm -rf /usr/share/dotnet
-        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
     - name: Checkout code
       uses: actions/checkout@v3
       with:
@@ -36,13 +35,16 @@ jobs:
       uses: actions/setup-go@v3
       with:
         go-version: 1.19.3
+      env:
+        GOPATH: ${{ runner.workspace }}/kata-containers
     - name: Check kernel config version
       run: |
         cd "${{ github.workspace }}/src/github.com/${{ github.repository }}"
         kernel_dir="tools/packaging/kernel/"
         kernel_version_file="${kernel_dir}kata_config_version"
-        modified_files=$(git diff --name-only origin/CCv0..HEAD)
-        if git diff --name-only origin/CCv0..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then
+        modified_files=$(git diff --name-only origin/main..HEAD)
+        result=$(git whatchanged origin/main..HEAD "${kernel_dir}" >>"/dev/null")
+        if git whatchanged origin/main..HEAD "${kernel_dir}" >>"/dev/null"; then
           echo "Kernel directory has changed, checking if $kernel_version_file has been updated"
           if echo "$modified_files" | grep -v "README.md" | grep "${kernel_dir}" >>"/dev/null"; then
             echo "$modified_files" | grep "$kernel_version_file" >>/dev/null || ( echo "Please bump version in $kernel_version_file" && exit 1)
@@ -51,14 +53,29 @@ jobs:
           fi
           echo "Check passed"
         fi
-    - name: Set PATH
+    - name: Setup GOPATH
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
+        echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}"
+        echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}"
+        echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}"
+        echo "TRAVIS: ${TRAVIS}"
+    - name: Set env
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
         echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+    - name: Setup travis references
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}"
+        target_branch=${TRAVIS_BRANCH}
     - name: Setup
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
         cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
+      env:
+        GOPATH: ${{ runner.workspace }}/kata-containers
     - name: Installing rust
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |

.gitignore

@@ -6,8 +6,6 @@
 **/.vscode
 **/.idea
 **/.fleet
-**/*.swp
-**/*.swo
 pkg/logging/Cargo.lock
 src/agent/src/version.rs
 src/agent/kata-agent.service

Makefile

@@ -18,7 +18,6 @@ TOOLS =
 TOOLS += agent-ctl
 TOOLS += kata-ctl
 TOOLS += log-parser
-TOOLS += log-parser-rs
 TOOLS += runk
 TOOLS += trace-forwarder
 

README.md

@@ -134,7 +134,6 @@ The table below lists the remaining parts of the project:
 | [osbuilder](tools/osbuilder) | infrastructure | Tool to create "mini O/S" rootfs and initrd images and kernel for the hypervisor. |
 | [`agent-ctl`](src/tools/agent-ctl) | utility | Tool that provides low-level access for testing the agent. |
 | [`kata-ctl`](src/tools/kata-ctl) | utility | Tool that provides advanced commands and debug facilities. |
-| [`log-parser-rs`](src/tools/log-parser-rs) | utility | Tool that aid in analyzing logs from the kata runtime. |
 | [`trace-forwarder`](src/tools/trace-forwarder) | utility | Agent tracing helper. |
 | [`runk`](src/tools/runk) | utility | Standard OCI container runtime based on the agent. |
 | [`ci`](https://github.com/kata-containers/ci) | CI | Continuous Integration configuration files and scripts. |

VERSION

@@ -1 +1 @@
-3.2.0-alpha3
+3.1.1

ci/install_libseccomp.sh

@@ -72,8 +72,7 @@ build_and_install_gperf() {
     curl -sLO "${gperf_tarball_url}"
     tar -xf "${gperf_tarball}"
     pushd "gperf-${gperf_version}"
-    # gperf is a build time dependency of libseccomp and not to be used in the target.
-    # Unset $CC since that might point to a cross compiler.
+    # Unset $CC for configure, we will always use native for gperf
     CC= ./configure --prefix="${gperf_install_dir}"
     make
     make install

ci/lib.sh

@@ -64,86 +64,3 @@ run_get_pr_changed_file_details()
 	source "$tests_repo_dir/.ci/lib.sh"
 	get_pr_changed_file_details
 }
-
-# Check if the 1st argument version is greater than and equal to 2nd one
-# Version format: [0-9]+ separated by period (e.g. 2.4.6, 1.11.3 and etc.)
-#
-# Parameters:
-#	$1	- a version to be tested
-#	$2	- a target version
-#
-# Return:
-# 	0 if $1 is greater than and equal to $2
-#	1 otherwise
-version_greater_than_equal() {
-	local current_version=$1
-	local target_version=$2
-	smaller_version=$(echo -e "$current_version\n$target_version" | sort -V | head -1)
-	if [ "${smaller_version}" = "${target_version}" ]; then
-		return 0
-	else
-		return 1
-	fi
-}
-
-# Build a IBM zSystem secure execution (SE) image
-#
-# Parameters:
-#	$1	- kernel_parameters
-#	$2	- a source directory where kernel and initrd are located
-#	$3	- a destination directory where a SE image is built
-#
-# Return:
-# 	0 if the image is successfully built
-#	1 otherwise
-build_secure_image() {
-	kernel_params="${1:-}"
-	install_src_dir="${2:-}"
-	install_dest_dir="${3:-}"
-
-	if [ ! -f "${install_src_dir}/vmlinuz.container" ] ||
-		[ ! -f "${install_src_dir}/kata-containers-initrd.img" ]; then
-		cat << EOF >&2
-Either kernel or initrd does not exist or is mistakenly named
-A file name for kernel must be vmlinuz.container (raw binary)
-A file name for initrd must be kata-containers-initrd.img
-EOF
-		return 1
-	fi
-
-	cmdline="${kernel_params} panic=1 scsi_mod.scan=none swiotlb=262144"
-	parmfile="$(mktemp --suffix=-cmdline)"
-	echo "${cmdline}" > "${parmfile}"
-	chmod 600 "${parmfile}"
-
-	[ -n "${HKD_PATH:-}" ] || (echo >&2 "No host key document specified." && return 1)
-	cert_list=($(ls -1 $HKD_PATH))
-	declare hkd_options
-	eval "for cert in ${cert_list[*]}; do
-		hkd_options+=\"--host-key-document=\\\"\$HKD_PATH/\$cert\\\" \"
-	done"
-
-	command -v genprotimg > /dev/null 2>&1 || { apt update; apt install -y s390-tools; }
-	extra_arguments=""
-	genprotimg_version=$(genprotimg --version | grep -Po '(?<=version )[^-]+')
-	if ! version_greater_than_equal "${genprotimg_version}" "2.17.0"; then
-		extra_arguments="--x-pcf '0xe0'"
-	fi
-
-	eval genprotimg \
-		"${extra_arguments}" \
-		"${hkd_options}" \
-		--output="${install_dest_dir}/kata-containers-secure.img" \
-		--image="${install_src_dir}/vmlinuz.container" \
-		--ramdisk="${install_src_dir}/kata-containers-initrd.img" \
-		--parmfile="${parmfile}" \
-		--no-verify # no verification for CI testing purposes
-
-	build_result=$?
-	rm -f "${parmfile}"
-	if [ $build_result -eq 0 ]; then
-		return 0
-	else
-		return 1
-	fi
-}

docs/Developer-Guide.md

@@ -2,8 +2,6 @@
 
 This document is written **specifically for developers**: it is not intended for end users.
 
-If you want to contribute changes that you have made, please read the [community guidelines](https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md) for information about our processes.
-
 # Assumptions
 
 - You are working on a non-critical test or development system.
@@ -656,7 +654,7 @@ section when using rootfs, or when using initrd, complete the steps in the [Buil
 
 Install the image:
 
->**Note**: When using an initrd image, replace the below rootfs image name `kata-containers.img`
+>**Note**: When using an initrd image, replace the below rootfs image name `kata-containers.img` 
 >with the initrd image name `kata-containers-initrd.img`.
 
 ```bash
@@ -690,25 +688,25 @@ $ sudo crictl run -r kata container.yaml pod.yaml
 
 The steps required to enable debug console for QEMU slightly differ with
 those for firecracker / cloud-hypervisor.
-
+ 
 ##### Enabling debug console for QEMU
 
-Add `agent.debug_console` to the guest kernel command line to allow the agent process to start a debug console.
+Add `agent.debug_console` to the guest kernel command line to allow the agent process to start a debug console. 
 
 ```bash
 $ sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.debug_console"/g' "${kata_configuration_file}"
 ```
 
-Here `kata_configuration_file` could point to `/etc/kata-containers/configuration.toml`
+Here `kata_configuration_file` could point to `/etc/kata-containers/configuration.toml` 
 or `/usr/share/defaults/kata-containers/configuration.toml`
 or `/opt/kata/share/defaults/kata-containers/configuration-{hypervisor}.toml`, if
 you installed Kata Containers using `kata-deploy`.
 
 ##### Enabling debug console for cloud-hypervisor / firecracker
 
-Slightly different configuration is required in case of firecracker and cloud hypervisor.
-Firecracker and cloud-hypervisor don't have a UNIX socket connected to `/dev/console`.
-Hence, the kernel command line option `agent.debug_console` will not work for them.
+Slightly different configuration is required in case of firecracker and cloud hypervisor. 
+Firecracker and cloud-hypervisor don't have a UNIX socket connected to `/dev/console`. 
+Hence, the kernel command line option `agent.debug_console` will not work for them. 
 These hypervisors support `hybrid vsocks`,  which can be used for communication
 between the host and the guest. The kernel command line option `agent.debug_console_vport`
  was added to allow developers specify on which `vsock` port the debugging console should be connected.
@@ -721,7 +719,7 @@ sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.debug_cons
 ```
 
 > **Note** Ports 1024 and 1025 are reserved for communication with the agent
-> and gathering of agent logs respectively.
+> and gathering of agent logs respectively. 
 
 ##### Connecting to the debug console
 

docs/Release-Process.md

@@ -28,6 +28,23 @@
   $ ./update-repository-version.sh -p "$NEW_VERSION" "$BRANCH"
   ```
 
+### Point tests repository to stable branch
+
+  If you create a new stable branch, i.e. if your release changes a major or minor version number (not a patch release), then
+  you should modify the `tests` repository to point to that newly created stable branch and not the `main` branch.
+  The objective is that changes in the CI on the main branch will not impact the stable branch.
+
+  In the test directory, change references the main branch in:
+  * `README.md`
+  * `versions.yaml`
+  * `cmd/github-labels/labels.yaml.in`
+  * `cmd/pmemctl/pmemctl.sh`
+  * `.ci/lib.sh`
+  * `.ci/static-checks.sh`
+
+  See the commits in [the corresponding PR for stable-2.1](https://github.com/kata-containers/tests/pull/3504) for an example of the changes.
+
+
 ### Merge all bump version Pull requests
 
   - The above step will create a GitHub pull request in the Kata projects. Trigger the CI using `/test` command on each bump Pull request.
@@ -46,24 +63,6 @@
   $ ./tag_repos.sh -p -b "$BRANCH" tag
   ```
 
-### Point tests repository to stable branch
-
-  If your release changes a major or minor version number(not a patch release), then the above 
-  `./tag_repos.sh` script will create a new stable branch in all the repositories in addition to tagging them.
-  This happens when you are making the first `rc` release for a new major or minor version in Kata.
-  In this case, you should modify the `tests` repository to point to the newly created stable branch and not the `main` branch.
-  The objective is that changes in the CI on the main branch will not impact the stable branch.
-
-  In the test directory, change references of the `main` branch to the new stable branch in:
-  * `README.md`
-  * `versions.yaml`
-  * `cmd/github-labels/labels.yaml.in`
-  * `cmd/pmemctl/pmemctl.sh`
-  * `.ci/lib.sh`
-  * `.ci/static-checks.sh`
-
-  See the commits in [the corresponding PR for stable-2.1](https://github.com/kata-containers/tests/pull/3504) for an example of the changes.
-
 ### Check Git-hub Actions
 
   We make use of [GitHub actions](https://github.com/features/actions) in this [file](../.github/workflows/release.yaml) in the `kata-containers/kata-containers` repository to build and upload release artifacts. This action is auto triggered with the above step when a new tag is pushed to the `kata-containers/kata-containers` repository.

docs/design/README.md

@@ -11,7 +11,6 @@ Kata Containers design documents:
 - [Host cgroups](host-cgroups.md)
 - [Agent systemd cgroup](agent-systemd-cgroup.md)
 - [`Inotify` support](inotify.md)
-- [`Hooks` support](hooks-handling.md)
 - [Metrics(Kata 2.0)](kata-2-0-metrics.md)
 - [Design for Kata Containers `Lazyload` ability with `nydus`](kata-nydus-design.md)
 - [Design for direct-assigned volume](direct-blk-device-assignment.md)

docs/design/architecture/networking.md

@@ -36,7 +36,7 @@ compatibility, and performance on par with MACVTAP.
 Kata Containers has deprecated support for bridge due to lacking performance relative to TC-filter and MACVTAP.
 
 Kata Containers supports both
-[CNM](https://github.com/moby/libnetwork/blob/master/docs/design.md#the-container-network-model)
+[CNM](https://github.com/docker/libnetwork/blob/master/docs/design.md#the-container-network-model)
 and [CNI](https://github.com/containernetworking/cni) for networking management.
 
 ## Network Hotplug

docs/design/hooks-handling.md

@@ -1,63 +0,0 @@
-# Kata Containers support for `Hooks`
-
-## Introduction
-
-During container's lifecycle, different Hooks can be executed to do custom actions. In Kata Containers, we support two types of Hooks, `OCI Hooks` and `Kata Hooks`.
-
-### OCI Hooks
-
-The OCI Spec stipulates six hooks that can be executed at different time points and namespaces, including `Prestart Hooks`, `CreateRuntime Hooks`, `CreateContainer Hooks`, `StartContainer Hooks`, `Poststart Hooks` and `Poststop Hooks`. We support these types of Hooks as compatible as possible in Kata Containers.
-
-The path and arguments of these hooks will be passed to Kata for execution via `bundle/config.json`. For example:
-```
-...
-"hooks": {
-  "prestart": [
-    {
-      "path": "/usr/bin/prestart-hook",
-      "args": ["prestart-hook", "arg1", "arg2"],
-      "env":  [ "key1=value1"]
-    }
-  ],
-  "createRuntime": [
-    {
-      "path": "/usr/bin/createRuntime-hook",
-      "args": ["createRuntime-hook", "arg1", "arg2"],
-      "env":  [ "key1=value1"]
-    }
-  ]
-}
-...
-```
-
-### Kata Hooks
-
-In Kata, we support another three kinds of hooks executed in guest VM, including `Guest Prestart Hook`, `Guest Poststart Hook`, `Guest Poststop Hook`.
-
-The executable files for Kata Hooks must be packaged in the *guest rootfs*. The file path to those guest hooks should be specified in the configuration file, and guest hooks must be stored in a subdirectory of `guest_hook_path` according to their hook type. For example:
-
-+ In configuration file:
-```
-guest_hook_path="/usr/share/hooks"
-```
-+ In guest rootfs, prestart-hook is stored in `/usr/share/hooks/prestart/prestart-hook`.
-
-## Execution
-The table below summarized when and where those different hooks will be executed in Kata Containers:
-
-| Hook Name | Hook Type | Hook Path | Exec Place | Exec Time |
-|---|---|---|---|---|
-| `Prestart(deprecated)` | OCI hook | host runtime namespace | host runtime namespace | After VM is started, before container is created. |
-| `CreateRuntime` | OCI hook | host runtime namespace | host runtime namespace | After VM is started, before container is created, after `Prestart` hooks. |
-| `CreateContainer` | OCI hook | host runtime namespace | host vmm namespace* | After VM is started, before container is created, after `CreateRuntime` hooks. |
-| `StartContainer` | OCI hook | guest container namespace | guest container namespace | After container is created, before container is started. |
-| `Poststart` | OCI hook | host runtime namespace | host runtime namespace | After container is started, before start operation returns. |
-| `Poststop` | OCI hook | host runtime namespace | host runtime namespace | After container is deleted, before delete operation returns. |
-| `Guest Prestart` | Kata hook | guest agent namespace | guest agent namespace | During start operation, before container command is executed. |
-| `Guest Poststart` | Kata hook | guest agent namespace | guest agent namespace | During start operation, after container command is executed, before start operation returns. |
-| `Guest Poststop` | Kata hook | guest agent namespace | guest agent namespace | During delete operation, after container is deleted, before delete operation returns. |
-
-+ `Hook Path` specifies where hook's path be resolved.
-+ `Exec Place` specifies in which namespace those hooks can be executed.
-  + For `CreateContainer` Hooks, OCI requires to run them inside the container namespace while the hook executable path is in the host runtime, which is a non-starter for VM-based containers. So we design to keep them running in the *host vmm namespace.* 
-+ `Exec Time` specifies at which time point those hooks can be executed.

docs/how-to/README.md

@@ -45,6 +45,3 @@
 - [How to run Kata Containers with `nydus`](how-to-use-virtio-fs-nydus-with-kata.md)
 - [How to run Kata Containers with AMD SEV-SNP](how-to-run-kata-containers-with-SNP-VMs.md)
 - [How to use EROFS to build rootfs in Kata Containers](how-to-use-erofs-build-rootfs.md)
-## Confidential Containers
-- [How to use build and test the Confidential Containers `CCv0` proof of concept](how-to-build-and-test-ccv0.md)
-- [How to generate a Kata Containers payload for the Confidential Containers Operator](how-to-generate-a-kata-containers-payload-for-the-confidential-containers-operator.md)

docs/how-to/ccv0.sh

@@ -1,635 +0,0 @@
-#!/bin/bash -e
-#
-# Copyright (c) 2021, 2023 IBM Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-# Disclaimer: This script is work in progress for supporting the CCv0 prototype
-# It shouldn't be considered supported by the Kata Containers community, or anyone else
-
-# Based on https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md,
-# but with elements of the tests/.ci scripts used
-
-readonly script_name="$(basename "${BASH_SOURCE[0]}")"
-
-# By default in Golang >= 1.16 GO111MODULE is set to "on", but not all modules support it, so overwrite to "auto"
-export GO111MODULE="auto"
-
-# Setup kata containers environments if not set - we default to use containerd
-export CRI_CONTAINERD=${CRI_CONTAINERD:-"yes"}
-export CRI_RUNTIME=${CRI_RUNTIME:-"containerd"}
-export CRIO=${CRIO:-"no"}
-export KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}"
-export KUBERNETES=${KUBERNETES:-"no"}
-export AGENT_INIT="${AGENT_INIT:-${TEST_INITRD:-no}}"
-export AA_KBC="${AA_KBC:-offline_fs_kbc}"
-export KATA_BUILD_CC=${KATA_BUILD_CC:-"yes"}
-export TEE_TYPE=${TEE_TYPE:-}
-export PREFIX="${PREFIX:-/opt/confidential-containers}"
-export RUNTIME_CONFIG_PATH="${RUNTIME_CONFIG_PATH:-${PREFIX}/share/defaults/kata-containers/configuration.toml}"
-
-# Allow the user to overwrite the default repo and branch names if they want to build from a fork
-export katacontainers_repo="${katacontainers_repo:-github.com/kata-containers/kata-containers}"
-export katacontainers_branch="${katacontainers_branch:-CCv0}" 
-export kata_default_branch=${katacontainers_branch}
-export tests_repo="${tests_repo:-github.com/kata-containers/tests}"
-export tests_branch="${tests_branch:-CCv0}"
-export target_branch=${tests_branch} # kata-containers/ci/lib.sh uses target branch var to check out tests repo
-
-# if .bash_profile exists then use it, otherwise fall back to .profile
-export PROFILE="${HOME}/.profile"
-if [ -r "${HOME}/.bash_profile" ]; then
-    export PROFILE="${HOME}/.bash_profile"
-fi
-# Stop PS1: unbound variable error happening
-export PS1=${PS1:-}
-
-# Create a bunch of common, derived values up front so we don't need to create them in all the different functions
-. ${PROFILE}
-if [ -z ${GOPATH} ]; then
-    export GOPATH=${HOME}/go
-fi
-export tests_repo_dir="${GOPATH}/src/${tests_repo}"
-export katacontainers_repo_dir="${GOPATH}/src/${katacontainers_repo}"
-export ROOTFS_DIR="${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder/rootfs"
-export PULL_IMAGE="${PULL_IMAGE:-quay.io/kata-containers/confidential-containers:signed}" # Doesn't need authentication
-export CONTAINER_ID="${CONTAINER_ID:-0123456789}"
-source /etc/os-release || source /usr/lib/os-release
-grep -Eq "\<fedora\>" /etc/os-release 2> /dev/null && export USE_PODMAN=true
-
-
-# If we've already checked out the test repo then source the confidential scripts
-if [ "${KUBERNETES}" == "yes" ]; then
-    export BATS_TEST_DIRNAME="${tests_repo_dir}/integration/kubernetes/confidential"
-    [ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/lib.sh"
-else
-    export BATS_TEST_DIRNAME="${tests_repo_dir}/integration/containerd/confidential"
-    [ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/lib.sh"
-fi
-
-[ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
-
-usage() {
-    exit_code="$1"
-    cat <<EOF
-Overview:
-    Build and test kata containers from source
-    Optionally set kata-containers and tests repo and branch as exported variables before running
-    e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && ~/${script_name} build_and_install_all
-Usage:
-    ${script_name} [options] <command>
-Commands:
-- agent_create_container:           Run CreateContainer command against the agent with agent-ctl
-- agent_pull_image:                 Run PullImage command against the agent with agent-ctl
-- all:                              Build and install everything, test kata with containerd and capture the logs
-- build_and_add_agent_to_rootfs:    Builds the kata-agent and adds it to the rootfs
-- build_and_install_all:            Build and install everything
-- build_and_install_rootfs:         Builds and installs the rootfs image
-- build_kata_runtime:               Build and install the kata runtime
-- build_cloud_hypervisor            Checkout, patch, build and install Cloud Hypervisor
-- build_qemu:                       Checkout, patch, build and install QEMU
-- configure:                        Configure Kata to use rootfs and enable debug
-- connect_to_ssh_demo_pod:          Ssh into the ssh demo pod, showing that the decryption succeeded
-- copy_signature_files_to_guest     Copies signature verification files to guest
-- create_rootfs:                    Create a local rootfs
-- crictl_create_cc_container        Use crictl to create a new busybox container in the kata cc pod
-- crictl_create_cc_pod              Use crictl to create a new kata cc pod
-- crictl_delete_cc                  Use crictl to delete the kata cc pod sandbox and container in it
-- help:                             Display this help
-- init_kubernetes:                  initialize a Kubernetes cluster on this system
-- initialize:                       Install dependencies and check out kata-containers source
-- install_guest_kernel:             Setup, build and install the guest kernel
-- kubernetes_create_cc_pod:         Create a Kata CC runtime busybox-based pod in Kubernetes
-- kubernetes_create_ssh_demo_pod:   Create a Kata CC runtime pod based on the ssh demo
-- kubernetes_delete_cc_pod:         Delete the Kata CC runtime busybox-based pod in Kubernetes
-- kubernetes_delete_ssh_demo_pod:   Delete the Kata CC runtime pod based on the ssh demo
-- open_kata_shell:                  Open a shell into the kata runtime
-- rebuild_and_install_kata:         Rebuild the kata runtime and agent and build and install the image
-- shim_pull_image:                  Run PullImage command against the shim with ctr
-- test_capture_logs:                Test using kata with containerd and capture the logs in the user's home directory
-- test:                             Test using kata with containerd
-
-Options:
-    -d: Enable debug
-    -h: Display this help
-EOF
-    # if script sourced don't exit as this will exit the main shell, just return instead
-    [[ $_ != $0 ]] && return "$exit_code" || exit "$exit_code"
-}
-
-build_and_install_all() {
-    initialize
-    build_and_install_kata_runtime
-    configure
-    create_a_local_rootfs
-    build_and_install_rootfs
-    install_guest_kernel_image
-    case "$KATA_HYPERVISOR" in
-        "qemu") 
-            build_qemu
-            ;;
-        "cloud-hypervisor") 
-            build_cloud_hypervisor
-            ;;
-        *)
-            echo "Invalid option: $KATA_HYPERVISOR is not supported." >&2
-            ;;
-    esac
-
-    check_kata_runtime
-    if [ "${KUBERNETES}" == "yes" ]; then
-        init_kubernetes
-    fi
-}
-
-rebuild_and_install_kata() {
-    checkout_tests_repo
-    checkout_kata_containers_repo
-    build_and_install_kata_runtime
-    build_and_add_agent_to_rootfs
-    build_and_install_rootfs
-    check_kata_runtime
-}
-
-# Based on the jenkins_job_build.sh script in kata-containers/tests/.ci - checks out source code and installs dependencies
-initialize() {
-    # We need git to checkout and bootstrap the ci scripts and some other packages used in testing
-    sudo apt-get update && sudo apt-get install -y curl git qemu-utils
-    
-    grep -qxF "export GOPATH=\${HOME}/go" "${PROFILE}" || echo "export GOPATH=\${HOME}/go" >> "${PROFILE}"
-    grep -qxF "export GOROOT=/usr/local/go" "${PROFILE}" || echo "export GOROOT=/usr/local/go" >> "${PROFILE}"
-    grep -qxF "export PATH=\${GOPATH}/bin:/usr/local/go/bin:\${PATH}" "${PROFILE}" || echo "export PATH=\${GOPATH}/bin:/usr/local/go/bin:\${PATH}" >> "${PROFILE}"
-    
-    # Load the new go and PATH parameters from the profile
-    . ${PROFILE}
-    mkdir -p "${GOPATH}"
-
-    checkout_tests_repo
-
-    pushd "${tests_repo_dir}"
-    local ci_dir_name=".ci"
-    sudo -E PATH=$PATH -s "${ci_dir_name}/install_go.sh" -p -f
-    sudo -E PATH=$PATH -s "${ci_dir_name}/install_rust.sh"
-    # Need to change ownership of rustup so later process can create temp files there
-    sudo chown -R ${USER}:${USER} "${HOME}/.rustup"
-
-    checkout_kata_containers_repo
-
-    # Run setup, but don't install kata as we will build it ourselves in locations matching the developer guide
-    export INSTALL_KATA="no"
-    sudo -E PATH=$PATH -s ${ci_dir_name}/setup.sh
-    # Reload the profile to pick up installed dependencies
-    . ${PROFILE}
-    popd
-}
-
-checkout_tests_repo() {
-    echo "Creating repo: ${tests_repo} and branch ${tests_branch} into ${tests_repo_dir}..."
-    # Due to git https://github.blog/2022-04-12-git-security-vulnerability-announced/ the tests repo needs
-    # to be owned by root as it is re-checked out in rootfs.sh
-    mkdir -p $(dirname "${tests_repo_dir}")
-    [ -d "${tests_repo_dir}" ] || sudo -E git clone "https://${tests_repo}.git" "${tests_repo_dir}"
-    sudo -E chown -R root:root "${tests_repo_dir}"
-    pushd "${tests_repo_dir}"
-    sudo -E git fetch
-    if [ -n "${tests_branch}" ]; then
-        sudo -E git checkout ${tests_branch}
-    fi
-    sudo -E git reset --hard origin/${tests_branch}
-    popd
-
-    source "${BATS_TEST_DIRNAME}/lib.sh"
-    source "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
-}
-
-# Note: clone_katacontainers_repo using go, so that needs to be installed first
-checkout_kata_containers_repo() {
-    source "${tests_repo_dir}/.ci/lib.sh"
-    echo "Creating repo: ${katacontainers_repo} and branch ${kata_default_branch} into ${katacontainers_repo_dir}..."
-    clone_katacontainers_repo
-    sudo -E chown -R ${USER}:${USER} "${katacontainers_repo_dir}"
-}
-
-build_and_install_kata_runtime() {
-    export DEFAULT_HYPERVISOR=${KATA_HYPERVISOR}
-    ${tests_repo_dir}/.ci/install_runtime.sh
-}
-
-configure() {
-    # configure kata to use rootfs, not initrd
-    sudo sed -i 's/^\(initrd =.*\)/# \1/g' ${RUNTIME_CONFIG_PATH}
-
-    enable_full_debug
-    enable_agent_console
-
-    # Switch image offload to true in kata config
-    switch_image_service_offload "on"
-
-    configure_cc_containerd
-    # From crictl v1.24.1 the default timoout leads to the pod creation failing, so update it
-    sudo crictl config --set timeout=10
-
-    # Verity checks aren't working locally, as we aren't re-genning the hash maybe? so remove it from the kernel parameters
-    remove_kernel_param "cc_rootfs_verity.scheme"
-}
-
-build_and_add_agent_to_rootfs() {
-    build_a_custom_kata_agent
-    add_custom_agent_to_rootfs
-}
-
-build_a_custom_kata_agent() {
-    # Install libseccomp for static linking
-    sudo -E PATH=$PATH GOPATH=$GOPATH ${katacontainers_repo_dir}/ci/install_libseccomp.sh /tmp/kata-libseccomp /tmp/kata-gperf
-    export LIBSECCOMP_LINK_TYPE=static
-    export LIBSECCOMP_LIB_PATH=/tmp/kata-libseccomp/lib
-
-    . "$HOME/.cargo/env"
-    pushd ${katacontainers_repo_dir}/src/agent
-    sudo -E PATH=$PATH make
-
-    ARCH=$(uname -m)
-    [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
-    [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
-
-    # Run a make install into the rootfs directory in order to create the kata-agent.service file which is required when we add to the rootfs
-    sudo -E PATH=$PATH make install DESTDIR="${ROOTFS_DIR}"
-    popd
-}
-
-create_a_local_rootfs() {
-    sudo rm -rf "${ROOTFS_DIR}"
-    pushd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder
-    export distro="ubuntu"
-    [[ -z "${USE_PODMAN:-}" ]] && use_docker="${use_docker:-1}"
-    sudo -E OS_VERSION="${OS_VERSION:-}" GOPATH=$GOPATH EXTRA_PKGS="vim iputils-ping net-tools" DEBUG="${DEBUG:-}" USE_DOCKER="${use_docker:-}" SKOPEO=${SKOPEO:-} AA_KBC=${AA_KBC:-} UMOCI=yes SECCOMP=yes ./rootfs.sh -r ${ROOTFS_DIR} ${distro}
-
-     # Install_rust.sh during rootfs.sh switches us to the main branch of the tests repo, so switch back now
-    pushd "${tests_repo_dir}"
-    sudo -E git checkout ${tests_branch}
-    popd
-    # During the ./rootfs.sh call the kata agent is built as root, so we need to update the permissions, so we can rebuild it
-    sudo chown -R ${USER}:${USER} "${katacontainers_repo_dir}/src/agent/"
-
-    popd
-}
-
-add_custom_agent_to_rootfs() {
-    pushd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder
-
-    ARCH=$(uname -m)
-    [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
-    [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
-
-    sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/usr/bin ${katacontainers_repo_dir}/src/agent/target/${ARCH}-unknown-linux-${LIBC}/release/kata-agent
-    sudo install -o root -g root -m 0440 ../../../src/agent/kata-agent.service ${ROOTFS_DIR}/usr/lib/systemd/system/
-    sudo install -o root -g root -m 0440 ../../../src/agent/kata-containers.target ${ROOTFS_DIR}/usr/lib/systemd/system/
-    popd
-}
-
-build_and_install_rootfs() {
-    build_rootfs_image
-    install_rootfs_image
-}
-
-build_rootfs_image() {
-    pushd ${katacontainers_repo_dir}/tools/osbuilder/image-builder
-    # Logic from install_kata_image.sh - if we aren't using podman (ie on a fedora like), then use docker
-    [[ -z "${USE_PODMAN:-}" ]] && use_docker="${use_docker:-1}"
-    sudo -E USE_DOCKER="${use_docker:-}" ./image_builder.sh ${ROOTFS_DIR}
-    popd
-}
-
-install_rootfs_image() {
-    pushd ${katacontainers_repo_dir}/tools/osbuilder/image-builder
-    local commit=$(git log --format=%h -1 HEAD)
-    local date=$(date +%Y-%m-%d-%T.%N%z)
-    local image="kata-containers-${date}-${commit}"
-    sudo install -o root -g root -m 0640 -D kata-containers.img "${PREFIX}/share/kata-containers/${image}"
-    (cd ${PREFIX}/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
-    echo "Built Rootfs from ${ROOTFS_DIR} to ${PREFIX}/share/kata-containers/${image}"
-    ls -al ${PREFIX}/share/kata-containers
-    popd
-}
-
-install_guest_kernel_image() {
-   ${tests_repo_dir}/.ci/install_kata_kernel.sh
-}
-
-build_qemu() {
-    ${tests_repo_dir}/.ci/install_virtiofsd.sh
-    ${tests_repo_dir}/.ci/install_qemu.sh
-}
-
-build_cloud_hypervisor() {
-    ${tests_repo_dir}/.ci/install_virtiofsd.sh
-    ${tests_repo_dir}/.ci/install_cloud_hypervisor.sh
-}
-
-check_kata_runtime() {
-    sudo kata-runtime check
-}
-
-k8s_pod_file="${HOME}/busybox-cc.yaml"
-init_kubernetes() {
-    # Check that kubeadm was installed and install it otherwise
-    if ! [ -x "$(command -v kubeadm)" ]; then
-        pushd "${tests_repo_dir}/.ci"
-        sudo -E PATH=$PATH -s install_kubernetes.sh
-        if [ "${CRI_CONTAINERD}" == "yes" ]; then
-            sudo -E PATH=$PATH -s "configure_containerd_for_kubernetes.sh"
-        fi
-        popd
-    fi
-
-    # If kubernetes init has previously run we need to clean it by removing the image and resetting k8s
-    local cid=$(sudo docker ps -a -q -f name=^/kata-registry$)
-    if [ -n "${cid}" ]; then
-        sudo docker stop ${cid} && sudo docker rm ${cid}
-    fi
-    local k8s_nodes=$(kubectl get nodes -o name 2>/dev/null || true)
-    if [ -n "${k8s_nodes}" ]; then
-        sudo kubeadm reset -f
-    fi
-
-    export CI="true" && sudo -E PATH=$PATH -s ${tests_repo_dir}/integration/kubernetes/init.sh
-    sudo chown ${USER}:$(id -g -n ${USER}) "$HOME/.kube/config"
-    cat << EOF > ${k8s_pod_file}
-apiVersion: v1
-kind: Pod
-metadata:
-  name: busybox-cc
-spec:
-  runtimeClassName: kata
-  containers:
-  - name: nginx
-    image: quay.io/kata-containers/confidential-containers:signed
-    imagePullPolicy: Always  
-EOF
-}
-
-call_kubernetes_create_cc_pod() {
-    kubernetes_create_cc_pod ${k8s_pod_file}
-}
-
-call_kubernetes_delete_cc_pod() {
-    pod_name=$(kubectl get pods -o jsonpath='{.items..metadata.name}')
-    kubernetes_delete_cc_pod $pod_name
-}
-
-call_kubernetes_create_ssh_demo_pod() {
-    setup_decryption_files_in_guest
-    kubernetes_create_ssh_demo_pod
-}
-
-call_connect_to_ssh_demo_pod() {
-    connect_to_ssh_demo_pod
-}
-
-call_kubernetes_delete_ssh_demo_pod() {
-    pod=$(kubectl get pods -o jsonpath='{.items..metadata.name}')
-    kubernetes_delete_ssh_demo_pod $pod
-}
-
-crictl_sandbox_name=kata-cc-busybox-sandbox
-call_crictl_create_cc_pod() {
-    # Update iptables to allow forwarding to the cni0 bridge avoiding issues caused by the docker0 bridge
-    sudo iptables -P FORWARD ACCEPT
-    
-    # get_pod_config in tests_common exports `pod_config` that points to the prepared pod config yaml 
-    get_pod_config
-
-    crictl_delete_cc_pod_if_exists "${crictl_sandbox_name}"
-    crictl_create_cc_pod "${pod_config}"
-    sudo crictl pods
-}
-
-call_crictl_create_cc_container() {
-    # Create container configuration yaml based on our test copy of busybox
-    # get_pod_config in tests_common exports `pod_config` that points to the prepared pod config yaml 
-    get_pod_config
-
-    local container_config="${FIXTURES_DIR}/${CONTAINER_CONFIG_FILE:-container-config.yaml}"
-    local pod_name=${crictl_sandbox_name}
-    crictl_create_cc_container ${pod_name} ${pod_config} ${container_config}
-    sudo crictl ps -a
-}
-
-crictl_delete_cc() {
-    crictl_delete_cc_pod ${crictl_sandbox_name}
-}
-
-test_kata_runtime() {
-    echo "Running ctr with the kata runtime..."
-    local test_image="quay.io/kata-containers/confidential-containers:signed"
-    if [ -z $(sudo ctr images ls -q name=="${test_image}") ]; then
-        sudo ctr image pull "${test_image}"
-    fi
-    sudo ctr run --runtime "io.containerd.kata.v2" --rm -t "${test_image}" test-kata uname -a
-}
-
-run_kata_and_capture_logs() {
-    echo "Clearing systemd journal..."
-    sudo systemctl stop systemd-journald
-    sudo rm -f /var/log/journal/*/* /run/log/journal/*/*
-    sudo systemctl start systemd-journald
-    test_kata_runtime
-    echo "Collecting logs..."
-    sudo journalctl -q -o cat -a -t kata-runtime > ${HOME}/kata-runtime.log
-    sudo journalctl -q -o cat -a -t kata > ${HOME}/shimv2.log
-    echo "Logs output to ${HOME}/kata-runtime.log and ${HOME}/shimv2.log"
-}
-
-get_ids() {
-    guest_cid=$(sudo ss -H --vsock | awk '{print $6}' | cut -d: -f1)
-    sandbox_id=$(ps -ef | grep containerd-shim-kata-v2 | egrep -o "id [^,][^,].* " | awk '{print $2}')
-}
-
-open_kata_shell() {
-    get_ids
-    sudo -E "PATH=$PATH" kata-runtime exec ${sandbox_id}
-}
-
-build_bundle_dir_if_necessary() {
-    bundle_dir="/tmp/bundle"
-    if [ ! -d "${bundle_dir}" ]; then
-        rootfs_dir="$bundle_dir/rootfs"
-        image="quay.io/kata-containers/confidential-containers:signed"
-        mkdir -p "$rootfs_dir" && (cd "$bundle_dir" && runc spec)
-        sudo docker export $(sudo docker create "$image") | tar -C "$rootfs_dir" -xvf -
-    fi
-    # There were errors in create container agent-ctl command due to /bin/ seemingly not being on the path, so hardcode it
-    sudo sed -i -e 's%^\(\t*\)"sh"$%\1"/bin/sh"%g' "${bundle_dir}/config.json"
-}
-
-build_agent_ctl() {
-    cd ${GOPATH}/src/${katacontainers_repo}/src/tools/agent-ctl/
-    if [ -e "${HOME}/.cargo/registry" ]; then
-        sudo chown -R ${USER}:${USER} "${HOME}/.cargo/registry"
-    fi
-    sudo -E PATH=$PATH -s  make
-    ARCH=$(uname -m)
-    [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
-    [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
-    cd "./target/${ARCH}-unknown-linux-${LIBC}/release/"
-}
-
-run_agent_ctl_command() {
-    get_ids
-    build_bundle_dir_if_necessary
-    command=$1
-    # If kata-agent-ctl pre-built in this directory, use it directly, otherwise build it first and switch to release
-    if [ ! -x kata-agent-ctl ]; then
-        build_agent_ctl
-    fi 
-     ./kata-agent-ctl -l debug connect --bundle-dir "${bundle_dir}" --server-address "vsock://${guest_cid}:1024" -c "${command}"
-}
-
-agent_pull_image() {
-    run_agent_ctl_command "PullImage image=${PULL_IMAGE} cid=${CONTAINER_ID} source_creds=${SOURCE_CREDS}"
-}
-
-agent_create_container() {
-    run_agent_ctl_command "CreateContainer cid=${CONTAINER_ID}"
-}
-
-shim_pull_image() {
-    get_ids
-    local ctr_shim_command="sudo ctr --namespace k8s.io shim --id ${sandbox_id} pull-image ${PULL_IMAGE} ${CONTAINER_ID}"
-    echo "Issuing command '${ctr_shim_command}'"
-    ${ctr_shim_command}
-}
-
-call_copy_signature_files_to_guest() {
-    # TODO #5173 - remove this once the kernel_params aren't ignored by the agent config
-    export DEBUG_CONSOLE="true"
-    
-    if [ "${SKOPEO:-}" = "yes" ]; then
-        add_kernel_params "agent.container_policy_file=/etc/containers/quay_verification/quay_policy.json"
-        setup_skopeo_signature_files_in_guest
-    else
-        # TODO #4888 - set config to specifically enable signature verification to be on in ImageClient
-        setup_offline_fs_kbc_signature_files_in_guest
-    fi
-}
-
-main() {
-    while getopts "dh" opt; do
-        case "$opt" in
-            d) 
-                export DEBUG="-d"
-                set -x
-                ;;
-            h) 
-                usage 0
-                ;;
-            \?)
-                echo "Invalid option: -$OPTARG" >&2
-                usage 1
-                ;;
-        esac
-    done
-
-    shift $((OPTIND - 1))
-
-    subcmd="${1:-}"
-
-    [ -z "${subcmd}" ] && usage 1
-
-    case "${subcmd}" in
-        all)
-            build_and_install_all
-            run_kata_and_capture_logs
-            ;;
-        build_and_install_all)
-            build_and_install_all
-            ;;
-        rebuild_and_install_kata)
-            rebuild_and_install_kata
-            ;;
-        initialize)
-            initialize
-            ;;
-        build_kata_runtime)
-            build_and_install_kata_runtime
-            ;;
-        configure)
-            configure
-            ;;
-        create_rootfs)
-            create_a_local_rootfs
-            ;;
-        build_and_add_agent_to_rootfs)
-            build_and_add_agent_to_rootfs
-            ;;
-        build_and_install_rootfs)
-            build_and_install_rootfs
-            ;;
-        install_guest_kernel)
-            install_guest_kernel_image
-            ;;
-        build_cloud_hypervisor)
-            build_cloud_hypervisor
-            ;;
-        build_qemu)
-            build_qemu
-            ;;
-        init_kubernetes)
-            init_kubernetes
-            ;;
-        crictl_create_cc_pod)
-            call_crictl_create_cc_pod
-            ;;
-        crictl_create_cc_container)
-            call_crictl_create_cc_container
-            ;;
-        crictl_delete_cc)
-            crictl_delete_cc
-            ;;
-        kubernetes_create_cc_pod)
-            call_kubernetes_create_cc_pod
-            ;;
-        kubernetes_delete_cc_pod)
-            call_kubernetes_delete_cc_pod
-            ;;
-        kubernetes_create_ssh_demo_pod)
-            call_kubernetes_create_ssh_demo_pod
-            ;;
-        connect_to_ssh_demo_pod)
-            call_connect_to_ssh_demo_pod
-            ;;
-        kubernetes_delete_ssh_demo_pod)
-            call_kubernetes_delete_ssh_demo_pod
-            ;;
-        test)
-            test_kata_runtime
-            ;;
-        test_capture_logs)
-            run_kata_and_capture_logs
-            ;;
-        open_kata_console)
-            open_kata_console
-            ;;
-        open_kata_shell)
-            open_kata_shell
-            ;;
-        agent_pull_image)
-            agent_pull_image
-            ;;
-        shim_pull_image)
-            shim_pull_image
-            ;;
-        agent_create_container)
-            agent_create_container
-            ;;
-        copy_signature_files_to_guest)
-            call_copy_signature_files_to_guest
-            ;;
-        *)
-            usage 1
-            ;;
-    esac
-}
-
-main $@

docs/how-to/data/confidential-agent-config.toml.in

@@ -1,45 +0,0 @@
-# Copyright (c) 2021 IBM Corp.
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-aa_kbc_params = "$AA_KBC_PARAMS"
-https_proxy = "$HTTPS_PROXY"
-[endpoints]
-allowed = [
-"AddARPNeighborsRequest",
-"AddSwapRequest",
-"CloseStdinRequest",
-"CopyFileRequest",
-"CreateContainerRequest",
-"CreateSandboxRequest",
-"DestroySandboxRequest",
-#"ExecProcessRequest",
-"GetMetricsRequest",
-"GetOOMEventRequest",
-"GuestDetailsRequest",
-"ListInterfacesRequest",
-"ListRoutesRequest",
-"MemHotplugByProbeRequest",
-"OnlineCPUMemRequest",
-"PauseContainerRequest",
-"PullImageRequest",
-"ReadStreamRequest",
-"RemoveContainerRequest",
-#"ReseedRandomDevRequest",
-"ResizeVolumeRequest",
-"ResumeContainerRequest",
-"SetGuestDateTimeRequest",
-"SignalProcessRequest",
-"StartContainerRequest",
-"StartTracingRequest",
-"StatsContainerRequest",
-"StopTracingRequest",
-"TtyWinResizeRequest",
-"UpdateContainerRequest",
-"UpdateInterfaceRequest",
-"UpdateRoutesRequest",
-"VolumeStatsRequest",
-"WaitProcessRequest",
-"WriteStreamRequest"
-]

docs/how-to/how-to-build-and-test-ccv0.md

@@ -1,475 +0,0 @@
-# How to build, run and test Kata CCv0
-
-## Introduction and Background
-
-In order to try and make building (locally) and demoing the Kata Containers `CCv0` code base as simple as possible I've
-shared a script [`ccv0.sh`](./ccv0.sh). This script was originally my attempt to automate the steps of the 
-[Developer Guide](https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md) so that I could do
-different sections of them repeatedly and reliably as I was playing around with make changes to different parts of the 
-Kata code base. I then tried to weave in some of the [`tests/.ci`](https://github.com/kata-containers/tests/tree/main/.ci) 
-scripts in order to have less duplicated code.
-As we're progress on the confidential containers journey I hope to add more features to demonstrate the functionality
-we have working.
-
-*Disclaimer: This script has mostly just been used and tested by me ([@stevenhorsman](https://github.com/stevenhorsman)),*
-*so there might be issues with it. I'm happy to try and help solve these if possible, but this shouldn't be considered a*
-*fully supported process by the Kata Containers community.*
-
-### Basic script set-up and optional environment variables
-
-In order to build, configure and demo the CCv0 functionality, these are the set-up steps I take:
-- Provision a new VM
-    - *I choose a Ubuntu 20.04 8GB VM for this as I had one available. There are some dependences on apt-get installed*
-    *packages, so these will need re-working to be compatible with other platforms.*
-- Copy the script over to your VM *(I put it in the home directory)* and ensure it has execute permission by running 
-```bash
-$ chmod u+x ccv0.sh
-```
-- Optionally set up some environment variables
-    - By default the script checks out the `CCv0` branches of the `kata-containers/kata-containers` and 
-      `kata-containers/tests` repositories, but it is designed to be used to test of personal forks and branches as well. 
-      If you want to build and run these you can export the `katacontainers_repo`, `katacontainers_branch`, `tests_repo`
-      and `tests_branch` variables e.g.
-      ```bash
-      $ export katacontainers_repo=github.com/stevenhorsman/kata-containers
-      $ export katacontainers_branch=stevenh/agent-pull-image-endpoint
-      $ export tests_repo=github.com/stevenhorsman/tests
-      $ export tests_branch=stevenh/add-ccv0-changes-to-build
-      ```
-      before running the script.
-    - By default the build and configuration are using `QEMU` as the hypervisor. In order to use `Cloud Hypervisor` instead
-      set:
-      ```
-      $ export KATA_HYPERVISOR="cloud-hypervisor"
-      ```
-      before running the build.
-
-- At this point you can provision a Kata confidential containers pod and container with either
-  [`crictl`](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image),
-  or [Kubernetes](#using-kubernetes-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image)
-  and then test and use it. 
-
-### Using crictl for end-to-end provisioning of a Kata confidential containers pod with an unencrypted image
-
-- Run the full build process with Kubernetes turned off, so its configuration doesn't interfere with `crictl` using:
-  ```bash
-  $ export KUBERNETES="no"
-  $ export KATA_HYPERVISOR="qemu"
-  $ ~/ccv0.sh -d build_and_install_all
-  ```
-    > **Note**: Much of this script has to be run as `sudo`, so you are likely to get prompted for your password.
-    - *I run this script sourced just so that the required installed components are accessible on the `PATH` to the rest*
-      *of the process without having to reload the session.*
-    - The steps that `build_and_install_all` takes is:
-        - Checkout the git repos for the `tests` and `kata-containers` repos as specified by the environment variables
-        (default to `CCv0` branches if they are not supplied)
-        - Use the `tests/.ci` scripts to install the build dependencies
-        - Build and install the Kata runtime
-        - Configure Kata to use containerd and for debug and confidential containers features to be enabled (including
-          enabling console access to the Kata guest shell, which should only be done in development)
-        - Create, build and install a rootfs for the Kata hypervisor to use. For 'CCv0' this is currently based on Ubuntu
-        20.04.
-        - Build the Kata guest kernel
-        - Install the hypervisor (in order to select which hypervisor will be used, the `KATA_HYPERVISOR` environment
-        variable can be used to select between `qemu` or `cloud-hypervisor`)
-    > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
-    during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
-    this, login into Docker Hub and pull the images used with:
-  >  ```bash
-  >  $ sudo docker login
-  >  $ sudo docker pull ubuntu
-  >  ```
-  >  then re-run the command.
-    - The first time this runs it may take a while, but subsequent runs will be quicker as more things are already
-      installed and they can be further cut down by not running all the above steps 
-      [see "Additional script usage" below](#additional-script-usage)
-   
-- Create a new Kata sandbox pod using `crictl` with:
-  ```bash
-  $ ~/ccv0.sh crictl_create_cc_pod
-  ```
-    - This creates a pod configuration file, creates the pod from this using 
-    `sudo crictl runp -r kata ~/pod-config.yaml` and runs `sudo crictl pods` to show the pod
-- Create a new Kata confidential container with:
-  ```bash
-  $ ~/ccv0.sh crictl_create_cc_container
-  ```
-    - This creates a container (based on `busybox:1.33.1`) in the Kata cc sandbox and prints a list of containers.
-      This will have been created based on an image pulled in the Kata pod sandbox/guest, not on the host machine.
-
-As this point you should have a `crictl` pod and container that is using the Kata confidential containers runtime.
-You can [validate that the container image was pulled on the guest](#validate-that-the-container-image-was-pulled-on-the-guest)
-or [using the Kata pod sandbox for testing with `agent-ctl` or `ctr shim`](#using-a-kata-pod-sandbox-for-testing-with-agent-ctl-or-ctr-shim)
-
-#### Clean up the `crictl` pod sandbox and container
-- When the testing is complete you can delete the container and pod by running:
-  ```bash
-  $ ~/ccv0.sh crictl_delete_cc
-  ```
-### Using Kubernetes for end-to-end provisioning of a Kata confidential containers pod with an unencrypted image
-
-- Run the full build process with the Kubernetes environment variable set to `"yes"`, so the Kubernetes cluster is
-  configured and created using the VM
-  as a single node cluster: 
-  ```bash
-  $ export KUBERNETES="yes"
-  $ ~/ccv0.sh build_and_install_all
-  ```
-    > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
-    during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
-    this, login into Docker Hub and pull the images used with:
-  >  ```bash
-  >  $ sudo docker login
-  >  $ sudo docker pull registry:2
-  >  $ sudo docker pull ubuntu:20.04
-  >  ```
-  >  then re-run the command.
-- Check that your Kubernetes cluster has been correctly set-up by running : 
-  ```bash
-  $ kubectl get nodes
-  ```
-  and checking that you see a single node e.g.
-  ```text
-  NAME                             STATUS   ROLES                  AGE   VERSION
-  stevenh-ccv0-k8s1.fyre.ibm.com   Ready    control-plane,master   43s   v1.22.0
-  ```
-- Create a Kata confidential containers pod by running:
-  ```bash
-  $ ~/ccv0.sh kubernetes_create_cc_pod
-  ```
-- Wait a few seconds for pod to start then check that the pod's status is `Running` with 
-  ```bash
-  $ kubectl get pods
-  ```
-  which should show something like:
-  ```text
-  NAME         READY   STATUS    RESTARTS   AGE
-  busybox-cc   1/1     Running   0          54s
-  ```
-
-- As this point you should have a Kubernetes pod and container running, that is using the Kata 
-confidential containers runtime.
-You can [validate that the container image was pulled on the guest](#validate-that-the-container-image-was-pulled-on-the-guest)
-or [using the Kata pod sandbox for testing with `agent-ctl` or `ctr shim`](#using-a-kata-pod-sandbox-for-testing-with-agent-ctl-or-ctr-shim)
-
-#### Clean up the Kubernetes pod sandbox and container
-- When the testing is complete you can delete the container and pod by running:
-  ```bash
-  $ ~/ccv0.sh kubernetes_delete_cc_pod
-  ```
-
-### Validate that the container image was pulled on the guest
-
-There are a couple of ways we can check that the container pull image action was offloaded to the guest, by checking
-the guest's file system for the unpacked bundle and checking the host's directories to ensure it wasn't also pulled
-there.
-- To check the guest's file system:
-    - Open a shell into the Kata guest with:
-      ```bash
-      $ ~/ccv0.sh open_kata_shell
-      ```
-    - List the files in the directory that the container image bundle should have been unpacked to with:
-      ```bash
-      $ ls -ltr /run/kata-containers/confidential-containers_signed/
-      ```
-    - This should give something like
-        ```
-        total 72
-        -rw-r--r--  1 root root  2977 Jan 20 10:03 config.json
-        drwxr-xr-x 12 root root   240 Jan 20 10:03 rootfs
-        ```
-        which shows how the image has been pulled and then unbundled on the guest.
-    - Leave the Kata guest shell by running:
-      ```bash
-      $ exit
-      ```
-- To verify that the image wasn't pulled on the host system we can look at the shared sandbox on the host and we
-  should only see a single bundle for the pause container as the `busybox` based container image should have been
-  pulled on the guest:
-    - Find all the `rootfs` directories under in the pod's shared directory with:
-      ```bash
-      $ pod_id=$(ps -ef | grep containerd-shim-kata-v2 | egrep -o "id [^,][^,].* " | awk '{print $2}')
-      $ sudo find /run/kata-containers/shared/sandboxes/${pod_id}/shared -name rootfs
-      ```
-      which should only show a single `rootfs` directory if the container image was pulled on the guest, not the host
-    - Looking that `rootfs` directory with
-      ```bash
-      $ sudo ls -ltr $(sudo find /run/kata-containers/shared/sandboxes/${pod_id}/shared -name rootfs)
-      ```
-      shows something similar to
-      ```
-      total 668
-      -rwxr-xr-x 1 root root 682696 Aug 25 13:58 pause
-      drwxr-xr-x 2 root root      6 Jan 20 02:01 proc
-      drwxr-xr-x 2 root root      6 Jan 20 02:01 dev
-      drwxr-xr-x 2 root root      6 Jan 20 02:01 sys
-      drwxr-xr-x 2 root root     25 Jan 20 02:01 etc
-      ```
-      which is clearly the pause container indicating that the `busybox` based container image is not exposed to the host.
-
-### Using a Kata pod sandbox for testing with `agent-ctl` or `ctr shim`
-
-Once you have a kata pod sandbox created as described above, either using 
-[`crictl`](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image), or [Kubernetes](#using-kubernetes-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image)
-, you can use this to test specific components of the Kata confidential
-containers architecture. This can be useful for development and debugging to isolate and test features
-that aren't broadly supported end-to-end. Here are some examples:
-
-- In the first terminal run the pull image on guest command against the Kata agent, via the shim (`containerd-shim-kata-v2`).
-This can be achieved using the [containerd](https://github.com/containerd/containerd) CLI tool, `ctr`, which can be used to
-interact with the shim directly. The command takes the form
-`ctr --namespace k8s.io shim --id <sandbox-id> pull-image <image> <new-container-id>` and can been run directly, or through
-the `ccv0.sh` script to automatically fill in the variables:
-  - Optionally, set up some environment variables to set the image and credentials used:
-    - By default the shim pull image test in `ccv0.sh` will use the `busybox:1.33.1` based test image
-     `quay.io/kata-containers/confidential-containers:signed` which requires no authentication. To use a different
-      image, set the `PULL_IMAGE` environment variable e.g. 
-      ```bash
-      $ export PULL_IMAGE="docker.io/library/busybox:latest"
-      ```
-      Currently the containerd shim pull image
-      code doesn't support using a container registry that requires authentication, so if this is required, see the 
-      below steps to run the pull image command against the agent directly.
-  - Run the pull image agent endpoint with:
-    ```bash
-    $ ~/ccv0.sh shim_pull_image
-    ```
-    which we print the `ctr shim` command for reference
-- Alternatively you can issue the command directly to the `kata-agent` pull image endpoint, which also supports
-  credentials in order to pull from an authenticated registry:
-    - Optionally set up some environment variables to set the image and credentials used:
-        - Set the `PULL_IMAGE` environment variable e.g. `export PULL_IMAGE="docker.io/library/busybox:latest"`
-          if a specific container image is required.
-        - If the container registry for the image requires authentication then this can be set with an environment
-          variable `SOURCE_CREDS`. For example to use Docker Hub (`docker.io`) as an authenticated user first run 
-          `export SOURCE_CREDS="<dockerhub username>:<dockerhub api key>"`
-            > **Note**: the credentials support on the agent request is a tactical solution for the short-term
-              proof of concept to allow more images to be pulled and tested. Once we have support for getting
-              keys into the Kata guest image using the attestation-agent and/or KBS I'd expect container registry
-              credentials to be looked up using that mechanism.
-    - Run the pull image agent endpoint with
-      ```bash
-      $ ~/ccv0.sh agent_pull_image
-      ```
-      and you should see output which includes `Command PullImage (1 of 1) returned (Ok(()), false)` to indicate
-      that the `PullImage` request was successful e.g.
-      ```
-      Finished release [optimized] target(s) in 0.21s
-      {"msg":"announce","level":"INFO","ts":"2021-09-15T08:40:14.189360410-07:00","subsystem":"rpc","name":"kata-agent-ctl","pid":"830920","version":"0.1.0","source":"kata-agent-ctl","config":"Config { server_address: \"vsock://1970354082:1024\", bundle_dir: \"/tmp/bundle\", timeout_nano: 0, interactive: false, ignore_errors: false }"}
-      {"msg":"client setup complete","level":"INFO","ts":"2021-09-15T08:40:14.193639057-07:00","pid":"830920","source":"kata-agent-ctl","name":"kata-agent-ctl","subsystem":"rpc","version":"0.1.0","server-address":"vsock://1970354082:1024"}
-      {"msg":"Run command PullImage (1 of 1)","level":"INFO","ts":"2021-09-15T08:40:14.196643765-07:00","pid":"830920","source":"kata-agent-ctl","subsystem":"rpc","name":"kata-agent-ctl","version":"0.1.0"}
-      {"msg":"response received","level":"INFO","ts":"2021-09-15T08:40:43.828200633-07:00","source":"kata-agent-ctl","name":"kata-agent-ctl","subsystem":"rpc","version":"0.1.0","pid":"830920","response":""}
-      {"msg":"Command PullImage (1 of 1) returned (Ok(()), false)","level":"INFO","ts":"2021-09-15T08:40:43.828261708-07:00","subsystem":"rpc","pid":"830920","source":"kata-agent-ctl","version":"0.1.0","name":"kata-agent-ctl"}
-      ```
-      > **Note**: The first time that `~/ccv0.sh agent_pull_image` is run, the `agent-ctl` tool will be built
-      which may take a few minutes.
-- To validate that the image pull was successful, you can open a shell into the Kata guest with:
-  ```bash
-  $ ~/ccv0.sh open_kata_shell
-  ```
-- Check the `/run/kata-containers/` directory to verify that the container image bundle has been created in a directory
-  named either `01234556789` (for the container id), or the container image name, e.g.
-  ```bash
-  $ ls -ltr /run/kata-containers/confidential-containers_signed/
-  ```
-  which should show something like
-  ```
-  total 72
-  drwxr-xr-x 10 root root   200 Jan  1  1970 rootfs
-  -rw-r--r--  1 root root  2977 Jan 20 16:45 config.json
-  ```
-- Leave the Kata shell by running:
-  ```bash
-  $ exit
-  ```
-
-## Verifying signed images
-
-For this sample demo, we use local attestation to pass through the required
-configuration to do container image signature verification. Due to this, the ability to verify images is limited
-to a pre-created selection of test images in our test
-repository [`quay.io/kata-containers/confidential-containers`](https://quay.io/repository/kata-containers/confidential-containers?tab=tags).
-For pulling images not in this test repository (called an *unprotected* registry below), we fall back to the behaviour
-of not enforcing signatures. More documentation on how to customise this to match your own containers through local,
-or remote attestation will be available in future.
-
-In our test repository there are three tagged images:
-
-| Test Image | Base Image used | Signature status | GPG key status |
-| --- | --- | --- | --- |
-| `quay.io/kata-containers/confidential-containers:signed` | `busybox:1.33.1` | [signature](https://github.com/kata-containers/tests/tree/CCv0/integration/confidential/fixtures/quay_verification/x86_64/signatures.tar) embedded in kata rootfs |  [public key](https://github.com/kata-containers/tests/tree/CCv0/integration/confidential/fixtures/quay_verification/x86_64/public.gpg) embedded in kata rootfs |
-| `quay.io/kata-containers/confidential-containers:unsigned` | `busybox:1.33.1` | not signed | not signed |
-| `quay.io/kata-containers/confidential-containers:other_signed` | `nginx:1.21.3` | [signature](https://github.com/kata-containers/tests/tree/CCv0/integration/confidential/fixtures/quay_verification/x86_64/signatures.tar) embedded in kata rootfs | GPG key not kept |
-
-Using a standard unsigned `busybox` image that can be pulled from another, *unprotected*, `quay.io` repository we can
-test a few scenarios.
-
-In this sample, with local attestation, we pass in the the public GPG key and signature files, and the [`offline_fs_kbc`
-configuration](https://github.com/confidential-containers/attestation-agent/blob/main/src/kbc_modules/offline_fs_kbc/README.md)
-into the guest image which specifies that any container image from `quay.io/kata-containers`
-must be signed with the embedded GPG key and the agent configuration needs updating to enable this. 
-With this policy set a few tests of image verification can be done to test different scenarios by attempting
-to create containers from these images using `crictl`:
-
-- If you don't already have the Kata Containers CC code built and configured for `crictl`, then follow the 
-[instructions above](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image)
-up to the `~/ccv0.sh crictl_create_cc_pod` command.
-
-- In order to enable the guest image, you will need to setup the required configuration, policy and signature files
-needed by running 
-`~/ccv0.sh copy_signature_files_to_guest` and then run `~/ccv0.sh crictl_create_cc_pod` which will delete and recreate 
-your pod - adding in the new files.
- 
-- To test the fallback behaviour works using an unsigned image from an *unprotected* registry we can pull the `busybox`
-image by running:
-  ```bash
-  $ export CONTAINER_CONFIG_FILE=container-config_unsigned-unprotected.yaml
-  $ ~/ccv0.sh crictl_create_cc_container
-  ```
-  - This finishes showing the running container e.g.
-  ```text
-  CONTAINER           IMAGE                               CREATED                  STATE               NAME                        ATTEMPT             POD ID
-  98c70fefe997a       quay.io/prometheus/busybox:latest   Less than a second ago   Running             prometheus-busybox-signed   0                   70119e0539238
-  ```
-- To test that an unsigned image from our *protected* test container registry is rejected we can run:
-  ```bash
-  $ export CONTAINER_CONFIG_FILE=container-config_unsigned-protected.yaml
-  $ ~/ccv0.sh crictl_create_cc_container
-  ```
-  - This correctly results in an error message from `crictl`:
-  `PullImage from image service failed" err="rpc error: code = Internal desc = Security validate failed: Validate image failed: The signatures do not satisfied! Reject reason: [Match reference failed.]" image="quay.io/kata-containers/confidential-containers:unsigned"`
-- To test that the signed image our *protected* test container registry is accepted we can run:
-  ```bash
-  $ export CONTAINER_CONFIG_FILE=container-config.yaml
-  $ ~/ccv0.sh crictl_create_cc_container
-  ```
-  - This finishes by showing a new `kata-cc-busybox-signed` running container e.g.
-  ```text
-  CONTAINER           IMAGE                                                    CREATED                  STATE               NAME                        ATTEMPT             POD ID
-  b4d85c2132ed9       quay.io/kata-containers/confidential-containers:signed   Less than a second ago   Running             kata-cc-busybox-signed      0                   70119e0539238
-  ...
-  ```
-- Finally to check the image with a valid signature, but invalid GPG key (the real trusted piece of information we really
-want to protect with the attestation agent in future) fails we can run: 
-  ```bash
-  $ export CONTAINER_CONFIG_FILE=container-config_signed-protected-other.yaml
-  $ ~/ccv0.sh crictl_create_cc_container
-  ```
-  - Again this results in an error message from `crictl`:
-  `"PullImage from image service failed" err="rpc error: code = Internal desc = Security validate failed: Validate image failed: The signatures do not satisfied! Reject reason: [signature verify failed! There is no pubkey can verify the signature!]" image="quay.io/kata-containers/confidential-containers:other_signed"`
-
-### Using Kubernetes to create a Kata confidential containers pod from the encrypted ssh demo sample image
-
-The [ssh-demo](https://github.com/confidential-containers/documentation/tree/main/demos/ssh-demo) explains how to
-demonstrate creating a Kata confidential containers pod from an encrypted image with the runtime created by the
-[confidential-containers operator](https://github.com/confidential-containers/documentation/blob/main/demos/operator-demo).
-To be fully confidential, this should be run on a Trusted Execution Environment, but it can be tested on generic
-hardware as well.
-
-If you wish to build the Kata confidential containers runtime to do this yourself, then you can using the following
-steps:
-
-- Run the full build process with the Kubernetes environment variable set to `"yes"`, so the Kubernetes cluster is
-  configured and created using the VM as a single node cluster and with `AA_KBC` set to `offline_fs_kbc`. 
-  ```bash
-  $ export KUBERNETES="yes"
-  $ export AA_KBC=offline_fs_kbc
-  $ ~/ccv0.sh build_and_install_all
-  ```
-    - The `AA_KBC=offline_fs_kbc` mode will ensure that, when creating the rootfs of the Kata guest, the 
-      [attestation-agent](https://github.com/confidential-containers/attestation-agent) will be added along with the
-      [sample offline KBC](https://github.com/confidential-containers/documentation/blob/main/demos/ssh-demo/aa-offline_fs_kbc-keys.json)
-      and an agent configuration file
-    > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
-    during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
-    this, login into Docker Hub and pull the images used with:
-  >  ```bash
-  >  $ sudo docker login
-  >  $ sudo docker pull registry:2
-  >  $ sudo docker pull ubuntu:20.04
-  >  ```
-  >  then re-run the command.
-- Check that your Kubernetes cluster has been correctly set-up by running : 
-  ```bash
-  $ kubectl get nodes
-  ```
-  and checking that you see a single node e.g.
-  ```text
-  NAME                             STATUS   ROLES                  AGE   VERSION
-  stevenh-ccv0-k8s1.fyre.ibm.com   Ready    control-plane,master   43s   v1.22.0
-  ```
-- Create a sample Kata confidential containers ssh pod by running:
-  ```bash
-  $ ~/ccv0.sh kubernetes_create_ssh_demo_pod
-  ```
-- As this point you should have a Kubernetes pod running the Kata confidential containers runtime that has pulled
-the [sample image](https://hub.docker.com/r/katadocker/ccv0-ssh) which was encrypted by the key file that we included
-in the rootfs.
-During the pod deployment the image was pulled and then decrypted using the key file, on the Kata guest image, without
-it ever being available to the host.
-
-- To validate that the container is working you, can connect to the image via SSH by running:
-  ```bash
-  $ ~/ccv0.sh connect_to_ssh_demo_pod
-  ```
-  - During this connection the host key fingerprint is shown and should match:
-    `ED25519 key fingerprint is SHA256:wK7uOpqpYQczcgV00fGCh+X97sJL3f6G1Ku4rvlwtR0.`
-  - After you are finished connecting then run:
-    ```bash
-    $ exit
-    ```
-  
-- To delete the sample SSH demo pod run:
-  ```bash
-  $ ~/ccv0.sh kubernetes_delete_ssh_demo_pod
-  ```
-
-## Additional script usage
-
-As well as being able to use the script as above to build all of `kata-containers` from scratch it can be used to just
-re-build bits of it by running the script with different parameters. For example after the first build you will often
-not need to re-install the dependencies, the hypervisor or the Guest kernel, but just test code changes made to the
-runtime and agent. This can be done by running `~/ccv0.sh rebuild_and_install_kata`. (*Note this does a hard checkout*
-*from git, so if your changes are only made locally it is better to do the individual steps e.g.* 
-`~/ccv0.sh build_kata_runtime && ~/ccv0.sh build_and_add_agent_to_rootfs && ~/ccv0.sh build_and_install_rootfs`).
-There are commands for a lot of steps in building, setting up and testing and the full list can be seen by running
-`~/ccv0.sh help`:
-```
-$ ~/ccv0.sh help
-Overview:
-    Build and test kata containers from source
-    Optionally set kata-containers and tests repo and branch as exported variables before running
-    e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && ~/ccv0.sh build_and_install_all
-Usage:
-    ccv0.sh [options] <command>
-Commands:
-- help:                         Display this help
-- all:                          Build and install everything, test kata with containerd and capture the logs
-- build_and_install_all:        Build and install everything
-- initialize:                   Install dependencies and check out kata-containers source
-- rebuild_and_install_kata:     Rebuild the kata runtime and agent and build and install the image
-- build_kata_runtime:           Build and install the kata runtime
-- configure:                    Configure Kata to use rootfs and enable debug
-- create_rootfs:                Create a local rootfs
-- build_and_add_agent_to_rootfs:Builds the kata-agent and adds it to the rootfs
-- build_and_install_rootfs:     Builds and installs the rootfs image
-- install_guest_kernel:         Setup, build and install the guest kernel
-- build_cloud_hypervisor        Checkout, patch, build and install Cloud Hypervisor
-- build_qemu:                   Checkout, patch, build and install QEMU
-- init_kubernetes:              initialize a Kubernetes cluster on this system
-- crictl_create_cc_pod          Use crictl to create a new kata cc pod
-- crictl_create_cc_container    Use crictl to create a new busybox container in the kata cc pod
-- crictl_delete_cc              Use crictl to delete the kata cc pod sandbox and container in it
-- kubernetes_create_cc_pod:     Create a Kata CC runtime busybox-based pod in Kubernetes
-- kubernetes_delete_cc_pod:     Delete the Kata CC runtime busybox-based pod in Kubernetes
-- open_kata_shell:              Open a shell into the kata runtime
-- agent_pull_image:             Run PullImage command against the agent with agent-ctl
-- shim_pull_image:              Run PullImage command against the shim with ctr
-- agent_create_container:       Run CreateContainer command against the agent with agent-ctl
-- test:                         Test using kata with containerd
-- test_capture_logs:            Test using kata with containerd and capture the logs in the user's home directory
-
-Options:
-    -d: Enable debug
-    -h: Display this help
-```

docs/how-to/how-to-generate-a-kata-containers-payload-for-the-confidential-containers-operator.md

@@ -1,44 +0,0 @@
-# Generating a Kata Containers payload for the Confidential Containers Operator
-
-[Confidential Containers
-Operator](https://github.com/confidential-containers/operator) consumes a Kata
-Containers payload, generated from the `CCv0` branch, and here one can find all
-the necessary info on how to build such a payload.
-
-## Requirements
-
-* `make` installed in the machine
-* Docker installed in the machine
-* `sudo` access to the machine
-
-## Process
-
-* Clone [Kata Containers](https://github.com/kata-containers/kata-containers)
-  ```sh
-  git clone --branch CCv0 https://github.com/kata-containers/kata-containers
-  ```
-  * In case you've already cloned the repo, make sure to switch to the `CCv0` branch
-    ```sh
-    git checkout CCv0
-    ```
-  * Ensure your tree is clean and in sync with upstream `CCv0`
-    ```sh
-    git clean -xfd
-    git reset --hard <upstream>/CCv0
-    ```
-* Make sure you're authenticated to `quay.io`
-    ```sh
-    sudo docker login quay.io
-    ```
-* From the top repo directory, run:
-  ```sh
-  sudo make cc-payload
-  ```
-* Make sure the image was upload to the [Confidential Containers
-  runtime-payload
-registry](https://quay.io/repository/confidential-containers/runtime-payload?tab=tags)
-
-## Notes
-
-Make sure to run it on a machine that's not the one you're hacking on, prepare a
-cup of tea, and get back to it an hour later (at least).

docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md

@@ -44,11 +44,12 @@ $ popd
 - Build a custom QEMU
 ```bash
 $ source kata-containers/tools/packaging/scripts/lib.sh
-$ qemu_url="$(get_from_kata_deps "assets.hypervisor.qemu-snp-experimental.url")"
-$ qemu_tag="$(get_from_kata_deps "assets.hypervisor.qemu-snp-experimental.tag")"
-$ git clone "${qemu_url}"
+$ qemu_url="$(get_from_kata_deps "assets.hypervisor.qemu.snp.url")"
+$ qemu_branch="$(get_from_kata_deps "assets.hypervisor.qemu.snp.branch")"
+$ qemu_commit="$(get_from_kata_deps "assets.hypervisor.qemu.snp.commit")"
+$ git clone -b "${qemu_branch}" "${qemu_url}"
 $ pushd qemu
-$ git checkout "${qemu_tag}"
+$ git checkout "${qemu_commit}"
 $ ./configure --enable-virtfs --target-list=x86_64-softmmu --enable-debug
 $ make -j "$(nproc)"
 $ popd

docs/how-to/how-to-run-rootless-vmm.md

@@ -1,5 +1,5 @@
 ## Introduction
-To improve security, Kata Container supports running the VMM process (QEMU and cloud-hypervisor) as a non-`root` user. 
+To improve security, Kata Container supports running the VMM process (currently only QEMU) as a non-`root` user.
 This document describes how to enable the rootless VMM mode and its limitations.
 
 ## Pre-requisites
@@ -27,7 +27,7 @@ Another necessary change is to move the hypervisor runtime files (e.g. `vhost-fs
 ## Limitations
 
 1. Only the VMM process is running as a non-root user. Other processes such as Kata Container shimv2 and `virtiofsd` still run as the root user.
-2. Currently, this feature is only supported in QEMU and cloud-hypervisor. For firecracker, you can use jailer to run the VMM process with a non-root user.
+2. Currently, this feature is only supported in QEMU. Still need to bring it to Firecracker and Cloud Hypervisor (see https://github.com/kata-containers/kata-containers/issues/2567).
 3. Certain features will not work when rootless VMM is enabled, including:
    1. Passing devices to the guest (`virtio-blk`, `virtio-scsi`) will not work if the non-privileged user does not have permission to access it (leading to a permission denied error). A more permissive permission (e.g. 666) may overcome this issue. However, you need to be aware of the potential security implications of reducing the security on such devices.
    2. `vfio` device will also not work because of permission denied error.

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -94,16 +94,6 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.enable_guest_swap` | `boolean` | enable swap in the guest |
 | `io.katacontainers.config.hypervisor.use_legacy_serial` | `boolean` | uses legacy serial device for guest's console (QEMU) |
 
-## Confidential Computing Options
-| Key | Value Type | Comments |
-|-------| ----- | ----- |
-| `io.katacontainers.config.pre_attestation.enabled"` | `bool` |
-determines if SEV/-ES attestation is enabled |
-| `io.katacontainers.config.pre_attestation.uri"` | `string` |
-specify the location of the attestation server |
-| `io.katacontainers.config.sev.policy"` | `uint32` |
-specify the SEV guest policy |
-
 ## Container Options
 | Key | Value Type | Comments |
 |-------| ----- | ----- |

docs/how-to/how-to-setup-swap-devices-in-guest-kernel.md

@@ -27,8 +27,6 @@ $ image="quay.io/prometheus/busybox:latest"
 $ cat << EOF > "${pod_yaml}"
 metadata:
   name: busybox-sandbox1
-  uid: $(uuidgen)
-  namespace: default
 EOF
 $ cat << EOF > "${container_yaml}"
 metadata:

docs/how-to/how-to-use-virtio-fs-nydus-with-kata.md

@@ -32,7 +32,6 @@ The `nydus-sandbox.yaml` looks like below:
 metadata:
   attempt: 1
   name: nydus-sandbox
-  uid: nydus-uid
   namespace: default
 log_directory: /tmp
 linux:

docs/how-to/how-to-use-virtio-mem-with-kata.md

@@ -42,8 +42,6 @@ $ image="quay.io/prometheus/busybox:latest"
 $ cat << EOF > "${pod_yaml}"
 metadata:
   name: busybox-sandbox1
-  uid: $(uuidgen)
-  namespace: default
 EOF
 $ cat << EOF > "${container_yaml}"
 metadata:

docs/install/README.md

@@ -19,7 +19,7 @@ Packaged installation methods uses your distribution's native package format (su
 |------------------------------------------------------|----------------------------------------------------------------------------------------------|-------------------|-----------------------------------------------------------------------------------------------|
 | [Using kata-deploy](#kata-deploy-installation)       | The preferred way to deploy the Kata Containers distributed binaries on a Kubernetes cluster | **No!**           | Best way to give it a try on kata-containers on an already up and running Kubernetes cluster. | 
 | [Using official distro packages](#official-packages) | Kata packages provided by Linux distributions official repositories                          | yes               | Recommended for most users.                                                                   |
-| ~~[Using snap](#snap-installation)~~                     | ~~Easy to install~~                                                                              | ~~yes~~               | **Snap is unmaintained!** ~~Good alternative to official distro packages.~~                                                 |
+| [Using snap](#snap-installation)                     | Easy to install                                                                              | yes               | Good alternative to official distro packages.                                                 |
 | [Automatic](#automatic-installation)                 | Run a single command to install a full system                                                | **No!**           | For those wanting the latest release quickly.                                                 |
 | [Manual](#manual-installation)                       | Follow a guide step-by-step to install a working system                                      | **No!**           | For those who want the latest release with more control.                                      |
 | [Build from source](#build-from-source-installation) | Build the software components manually                                                       | **No!**           | Power users and developers only.                                                              |
@@ -44,24 +44,9 @@ Kata packages are provided by official distribution repositories for:
 
 ### Snap Installation
 
-> **WARNING:**
->
-> The Snap package method is **unmaintained** and only provides an old
-> version of Kata Containers:
-> The [latest Kata Containers snap](https://snapcraft.io/kata-containers)
-> provides Kata Containers
-> [version 2.4.2](https://github.com/kata-containers/kata-containers/releases/tag/2.4.2)
-> but the latest stable Kata Containers release at the time of writing is
-> [version 3.1.0](https://github.com/kata-containers/kata-containers/releases/tag/3.1.0).
->
-> We recommend strongly that you switch to an alternative Kata Containers installation method.
->
-> See: https://github.com/kata-containers/kata-containers/issues/6769
-> for further details.
+The snap installation is available for all distributions which support `snapd`.
 
-~~The snap installation is available for all distributions which support `snapd`.~~
-
-~~[Use snap](snap-installation-guide.md) to install Kata Containers from https://snapcraft.io. ~~
+[Use snap](snap-installation-guide.md) to install Kata Containers from https://snapcraft.io.
 
 ### Automatic Installation
 

docs/install/aws-installation-guide.md

@@ -123,7 +123,7 @@ Refer to [this guide](https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-l
 SSH into the machine
 
 ```bash
-$ ssh -i MyKeyPair.pem ubuntu@${IP}
+$ ssh -i MyKeyPair.pen ubuntu@${IP}
 ```
 
 Go onto the next step.

docs/install/kata-containers-3.0-rust-runtime-installation-guide.md

@@ -49,14 +49,14 @@ Follow the [`kata-deploy`](../../tools/packaging/kata-deploy/README.md).
 
 * Download `Rustup` and install  `Rust`
     > **Notes:**
-    > For Rust version, please set `RUST_VERSION` to the value of `languages.rust.meta.newest-version key` in [`versions.yaml`](../../versions.yaml) or, if `yq` is available on your system, run `export RUST_VERSION=$(yq read versions.yaml languages.rust.meta.newest-version)`.
+    > Rust version 1.62.0 is needed
 
     Example for `x86_64`
     ```
     $ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
     $ source $HOME/.cargo/env
-    $ rustup install ${RUST_VERSION}
-    $ rustup default ${RUST_VERSION}-x86_64-unknown-linux-gnu
+    $ rustup install 1.62.0
+    $ rustup default 1.62.0-x86_64-unknown-linux-gnu
     ```
 
 * Musl support for fully static binary

docs/install/snap-installation-guide.md

@@ -1,20 +1,5 @@
 # Kata Containers snap package
 
-> **WARNING:**
->
-> The Snap package method is **unmaintained** and only provides an old
-> version of Kata Containers:
-> The [latest Kata Containers snap](https://snapcraft.io/kata-containers)
-> provides Kata Containers
-> [version 2.4.2](https://github.com/kata-containers/kata-containers/releases/tag/2.4.2)
-> but the latest stable Kata Containers release at the time of writing is
-> [version 3.1.0](https://github.com/kata-containers/kata-containers/releases/tag/3.1.0).
->
-> We recommend strongly that you switch to an alternative Kata Containers installation method.
->
-> See: https://github.com/kata-containers/kata-containers/issues/6769
-> for further details.
-
 ## Install Kata Containers
 
 Kata Containers can be installed in any Linux distribution that supports
@@ -22,21 +7,6 @@ Kata Containers can be installed in any Linux distribution that supports
 
 Run the following command to install **Kata Containers**:
 
-> **WARNING:**
->
-> The Snap package method is **unmaintained** and only provides an old
-> version of Kata Containers:
-> The [latest Kata Containers snap](https://snapcraft.io/kata-containers)
-> provides Kata Containers
-> [version 2.4.2](https://github.com/kata-containers/kata-containers/releases/tag/2.4.2)
-> but the latest stable Kata Containers release at the time of writing is
-> [version 3.1.0](https://github.com/kata-containers/kata-containers/releases/tag/3.1.0).
->
-> We recommend strongly that you switch to an alternative Kata Containers installation method.
->
-> See: https://github.com/kata-containers/kata-containers/issues/6769
-> for further details.
-
 ```sh
 $ sudo snap install kata-containers --stable --classic
 ```

snap/snapcraft.yaml

@@ -34,6 +34,54 @@ parts:
       mkdir -p $(dirname ${kata_dir})
       ln -sf $(realpath "${SNAPCRAFT_STAGE}/..") ${kata_dir}
 
+  godeps:
+    after: [metadata]
+    plugin: nil
+    prime:
+      - -*
+    build-packages:
+      - curl
+    override-build: |
+      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
+
+      # put everything in stage
+      cd "${SNAPCRAFT_STAGE}"
+
+      version="$(${yq} r ${kata_dir}/versions.yaml languages.golang.meta.newest-version)"
+      tarfile="go${version}.${goos}-${goarch}.tar.gz"
+      curl -LO https://golang.org/dl/${tarfile}
+      tar -xf ${tarfile} --strip-components=1
+
+  rustdeps:
+    after: [metadata]
+    plugin: nil
+    prime:
+      - -*
+    build-packages:
+      - curl
+    override-build: |
+      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
+
+      # put everything in stage
+      cd "${SNAPCRAFT_STAGE}"
+
+      version="$(${yq} r ${kata_dir}/versions.yaml languages.rust.meta.newest-version)"
+      if ! command -v rustup > /dev/null; then
+        curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain ${version}
+      fi
+
+      export PATH=${PATH}:${HOME}/.cargo/bin
+      rustup toolchain install ${version}
+      rustup default ${version}
+      if [ "${arch}" == "ppc64le" ] || [ "${arch}" == "s390x" ] ; then
+        [ "${arch}" == "ppc64le" ] && arch="powerpc64le"
+        rustup target add ${arch}-unknown-linux-gnu
+      else
+        rustup target add ${arch}-unknown-linux-musl
+        $([ "$(whoami)" != "root" ] && echo sudo) ln -sf /usr/bin/g++ /bin/musl-g++
+      fi
+      rustup component add rustfmt
+
   docker:
     after: [metadata]
     plugin: nil
@@ -63,92 +111,240 @@ parts:
       echo "Adding $USER into docker group"
       sudo -E gpasswd -a $USER docker
       echo "Starting docker"
-      # docker may fail to start using "fd://" in docker.service
-      sudo sed -i 's/fd:\/\//unix:\/\//g' /lib/systemd/system/docker.service
-      sudo systemctl daemon-reload
       sudo -E systemctl start docker || true
 
   image:
-    after: [docker]
+    after: [godeps, docker, qemu, kernel]
     plugin: nil
+    build-packages:
+      - docker.io
+      - cpio
+      - git
+      - iptables
+      - software-properties-common
+      - uidmap
+      - gnupg2
     override-build: |
       source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
 
-      cd "${SNAPCRAFT_PROJECT_DIR}"
-      sudo -E NO_TTY=true make rootfs-image-tarball
+      [ "${arch}" = "ppc64le" ] || [ "${arch}" = "s390x" ] && sudo apt-get --no-install-recommends install -y protobuf-compiler
 
-      tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-rootfs-image.tar.xz"
+      if [ -n "$http_proxy" ]; then
+        echo "Setting proxy $http_proxy"
+        sudo -E systemctl set-environment http_proxy="$http_proxy" || true
+        sudo -E systemctl set-environment https_proxy="$https_proxy" || true
+      fi
 
-      tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}"
+      # Copy yq binary. It's used in the container
+      cp -a "${yq}" "${GOPATH}/bin/"
 
+      cd "${kata_dir}/tools/osbuilder"
 
-      sudo -E NO_TTY=true make rootfs-initrd-tarball
+      # build image
+      export AGENT_INIT=yes
+      export USE_DOCKER=1
+      export DEBUG=1
+      initrd_distro=$(${yq} r -X ${kata_dir}/versions.yaml assets.initrd.architecture.${arch}.name)
+      image_distro=$(${yq} r -X ${kata_dir}/versions.yaml assets.image.architecture.${arch}.name)
+      case "$arch" in
+        x86_64)
+          # In some build systems it's impossible to build a rootfs image, try with the initrd image
+          sudo -E PATH=$PATH make image DISTRO="${image_distro}" || sudo -E PATH="$PATH" make initrd DISTRO="${initrd_distro}"
+        ;;
 
-      tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-rootfs-initrd.tar.xz"
+        aarch64|ppc64le|s390x)
+          sudo -E PATH="$PATH" make initrd DISTRO="${initrd_distro}"
+        ;;
 
-      tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}"
+        *) die "unsupported architecture: ${arch}" ;;
+      esac
 
+      # Install image
+      kata_image_dir="${SNAPCRAFT_PART_INSTALL}/usr/share/kata-containers"
+      mkdir -p "${kata_image_dir}"
+      cp kata-containers*.img "${kata_image_dir}"
 
   runtime:
-    after: [docker]
+    after: [godeps, image, cloud-hypervisor]
     plugin: nil
+    build-attributes: [no-patchelf]
     override-build: |
       source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
 
-      cd "${SNAPCRAFT_PROJECT_DIR}"
-      sudo -E NO_TTY=true make shim-v2-tarball
+      cd "${kata_dir}/src/runtime"
 
-      tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-shim-v2.tar.xz"
+      qemu_cmd="qemu-system-${qemu_arch}"
 
-      tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}"
+      # build and install runtime
+      make \
+        PREFIX="/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr" \
+        SKIP_GO_VERSION_CHECK=1 \
+        QEMUCMD="${qemu_cmd}"
 
-      mkdir -p "${SNAPCRAFT_PART_INSTALL}/usr/bin"
-      ln -sf "${SNAPCRAFT_PART_INSTALL}/opt/kata/bin/containerd-shim-kata-v2" "${SNAPCRAFT_PART_INSTALL}/usr/bin/containerd-shim-kata-v2"
-      ln -sf "${SNAPCRAFT_PART_INSTALL}/opt/kata/bin/kata-runtime" "${SNAPCRAFT_PART_INSTALL}/usr/bin/kata-runtime"
-      ln -sf "${SNAPCRAFT_PART_INSTALL}/opt/kata/bin/kata-collect-data.sh" "${SNAPCRAFT_PART_INSTALL}/usr/bin/kata-collect-data.sh"
+      make install \
+        PREFIX=/usr \
+        DESTDIR="${SNAPCRAFT_PART_INSTALL}" \
+        SKIP_GO_VERSION_CHECK=1 \
+        QEMUCMD="${qemu_cmd}"
+
+      if [ ! -f ${SNAPCRAFT_PART_INSTALL}/../../image/install/usr/share/kata-containers/kata-containers.img ]; then
+        sed -i -e "s|^image =.*|initrd = \"/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr/share/kata-containers/kata-containers-initrd.img\"|" \
+          ${SNAPCRAFT_PART_INSTALL}/usr/share/defaults/${SNAPCRAFT_PROJECT_NAME}/configuration.toml
+      fi
 
   kernel:
-    after: [docker]
+    after: [godeps]
     plugin: nil
+    build-packages:
+      - libelf-dev
+      - curl
+      - build-essential
+      - bison
+      - flex
     override-build: |
       source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
 
-      cd "${SNAPCRAFT_PROJECT_DIR}"
-      sudo -E NO_TTY=true make kernel-tarball
+      kernel_version="$(${yq} r $versions_file assets.kernel.version)"
+      #Remove extra 'v'
+      kernel_version="${kernel_version#v}"
 
-      tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-kernel.tar.xz"
+      [ "${arch}" = "s390x" ] && sudo apt-get --no-install-recommends install -y libssl-dev
 
-      tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}"
+      cd "${kata_dir}/tools/packaging/kernel"
+      kernel_dir_prefix="kata-linux-"
+
+      # Setup and build kernel
+      ./build-kernel.sh -v "${kernel_version}" -d setup
+      cd ${kernel_dir_prefix}*
+      make -j $(nproc ${CI:+--ignore 1}) EXTRAVERSION=".container"
+
+      kernel_suffix="${kernel_version}.container"
+      kata_kernel_dir="${SNAPCRAFT_PART_INSTALL}/usr/share/kata-containers"
+      mkdir -p "${kata_kernel_dir}"
+
+      # Install bz kernel
+      make install INSTALL_PATH="${kata_kernel_dir}" EXTRAVERSION=".container" || true
+      vmlinuz_name="vmlinuz-${kernel_suffix}"
+      ln -sf "${vmlinuz_name}" "${kata_kernel_dir}/vmlinuz.container"
+
+      # Install raw kernel
+      vmlinux_path="vmlinux"
+      [ "${arch}" = "s390x" ] && vmlinux_path="arch/s390/boot/vmlinux"
+      vmlinux_name="vmlinux-${kernel_suffix}"
+      cp "${vmlinux_path}" "${kata_kernel_dir}/${vmlinux_name}"
+      ln -sf "${vmlinux_name}" "${kata_kernel_dir}/vmlinux.container"
 
   qemu:
     plugin: make
-    after: [docker]
+    after: [godeps]
+    build-packages:
+      - gcc
+      - python3
+      - zlib1g-dev
+      - libcap-ng-dev
+      - libglib2.0-dev
+      - libpixman-1-dev
+      - libnuma-dev
+      - libltdl-dev
+      - libcap-dev
+      - libattr1-dev
+      - libfdt-dev
+      - curl
+      - libcapstone-dev
+      - bc
+      - libblkid-dev
+      - libffi-dev
+      - libmount-dev
+      - libseccomp-dev
+      - libselinux1-dev
+      - ninja-build
     override-build: |
       source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
 
-      cd "${SNAPCRAFT_PROJECT_DIR}"
-      sudo -E NO_TTY=true make qemu-tarball
+      branch="$(${yq} r ${versions_file} assets.hypervisor.qemu.version)"
+      url="$(${yq} r ${versions_file} assets.hypervisor.qemu.url)"
+      commit=""
+      patches_dir="${kata_dir}/tools/packaging/qemu/patches/$(echo ${branch} | sed -e 's/.[[:digit:]]*$//' -e 's/^v//').x"
+      patches_version_dir="${kata_dir}/tools/packaging/qemu/patches/tag_patches/${branch}"
 
-      tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-qemu.tar.xz"
+      # download source
+      qemu_dir="${SNAPCRAFT_STAGE}/qemu"
+      rm -rf "${qemu_dir}"
+      git clone --depth 1 --branch ${branch} --single-branch ${url} "${qemu_dir}"
+      cd "${qemu_dir}"
+      [ -z "${commit}" ] || git checkout "${commit}"
 
-      tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}"
+      [ -n "$(ls -A ui/keycodemapdb)" ] || git clone --depth 1 https://github.com/qemu/keycodemapdb ui/keycodemapdb/
+      [ -n "$(ls -A capstone)" ] || git clone --depth 1 https://github.com/qemu/capstone capstone
+
+      # Apply branch patches
+      [ -d "${patches_version_dir}" ] || mkdir "${patches_version_dir}"
+      ${kata_dir}/tools/packaging/scripts/apply_patches.sh "${patches_dir}"
+      ${kata_dir}/tools/packaging/scripts/apply_patches.sh "${patches_version_dir}"
+
+      # Only x86_64 supports libpmem
+      [ "${arch}" = "x86_64" ] && sudo apt-get --no-install-recommends install -y apt-utils ca-certificates libpmem-dev
+
+      configure_hypervisor="${kata_dir}/tools/packaging/scripts/configure-hypervisor.sh"
+      chmod +x "${configure_hypervisor}"
+      # static build. The --prefix, --libdir, --libexecdir, --datadir arguments are
+      # based on PREFIX and set by configure-hypervisor.sh
+      echo "$(PREFIX=/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr ${configure_hypervisor} -s kata-qemu) \
+        --disable-rbd " \
+        | xargs ./configure
+
+      # Copy QEMU configurations (Kconfigs)
+      case "${branch}" in
+      "v5.1.0")
+        cp -a "${kata_dir}"/tools/packaging/qemu/default-configs/* default-configs
+        ;;
+
+      *)
+        cp -a "${kata_dir}"/tools/packaging/qemu/default-configs/* configs/devices/
+        ;;
+      esac
+
+      # build and install
+      make -j $(nproc ${CI:+--ignore 1})
+      make install DESTDIR="${SNAPCRAFT_PART_INSTALL}"
+    prime:
+      - -snap/
+      - -usr/bin/qemu-ga
+      - -usr/bin/qemu-pr-helper
+      - -usr/bin/virtfs-proxy-helper
+      - -usr/include/
+      - -usr/share/applications/
+      - -usr/share/icons/
+      - -usr/var/
+      - usr/*
+      - lib/*
+    organize:
+      # Hack: move qemu to /
+      "snap/kata-containers/current/": "./"
 
   virtiofsd:
     plugin: nil
-    after: [docker]
+    after: [godeps, rustdeps, docker]
     override-build: |
       source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
 
+      echo "INFO: Building rust version of virtiofsd"
+
       cd "${SNAPCRAFT_PROJECT_DIR}"
+      # Clean-up build dir in case it already exists
       sudo -E NO_TTY=true make virtiofsd-tarball
 
-      tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-virtiofsd.tar.xz"
-
-      tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}"
+      sudo install \
+        --owner='root' \
+        --group='root' \
+        --mode=0755 \
+        -D \
+        --target-directory="${SNAPCRAFT_PART_INSTALL}/usr/libexec/" \
+        build/virtiofsd/builddir/virtiofsd/virtiofsd
 
   cloud-hypervisor:
     plugin: nil
-    after: [docker]
+    after: [godeps, docker]
     override-build: |
       source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
 
@@ -157,8 +353,13 @@ parts:
           sudo -E NO_TTY=true make cloud-hypervisor-tarball
 
           tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-cloud-hypervisor.tar.xz"
+          tmpdir=$(mktemp -d)
 
-          tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}"
+          tar -xvJpf "${tarfile}" -C "${tmpdir}"
+
+          install -D "${tmpdir}/opt/kata/bin/cloud-hypervisor" "${SNAPCRAFT_PART_INSTALL}/usr/bin/cloud-hypervisor"
+
+          rm -rf "${tmpdir}"
       fi
 
 apps:

src/agent/Cargo.toml

@@ -10,8 +10,8 @@ oci = { path = "../libs/oci" }
 rustjail = { path = "rustjail" }
 protocols = { path = "../libs/protocols", features = ["async"] }
 lazy_static = "1.3.0"
-ttrpc = { version = "0.7.1", features = ["async"], default-features = false }
-protobuf = "3.2.0"
+ttrpc = { version = "0.6.0", features = ["async"], default-features = false }
+protobuf = "2.27.0"
 libc = "0.2.58"
 nix = "0.24.2"
 capctl = "0.2.0"
@@ -23,15 +23,14 @@ regex = "1.5.6"
 serial_test = "0.5.1"
 kata-sys-util = { path = "../libs/kata-sys-util" }
 kata-types = { path = "../libs/kata-types" }
-url = "2.2.2"
 
 # Async helpers
 async-trait = "0.1.42"
 async-recursion = "0.3.2"
-futures = "0.3.28"
+futures = "0.3.17"
 
 # Async runtime
-tokio = { version = "1.28.1", features = ["full"] }
+tokio = { version = "1.14.0", features = ["full"] }
 tokio-vsock = "0.3.1"
 
 netlink-sys = { version = "0.7.0", features = ["tokio_socket",]}
@@ -49,7 +48,6 @@ slog-scope = "4.1.2"
 slog-stdlog = "4.0.0"
 log = "0.4.11"
 
-cfg-if = "1.0.0"
 prometheus = { version = "0.13.0", features = ["process"] }
 procfs = "0.12.0"
 anyhow = "1.0.32"
@@ -67,22 +65,12 @@ serde = { version = "1.0.129", features = ["derive"] }
 toml = "0.5.8"
 clap = { version = "3.0.1", features = ["derive"] }
 
-# "vendored" feature for openssl is required by musl build
-openssl = { version = "0.10.38", features = ["vendored"] }
-
-# Image pull/decrypt
-image-rs = { git = "https://github.com/confidential-containers/image-rs", tag = "v0.6.0", default-features = false, features = ["kata-cc-native-tls"] }
-
-[patch.crates-io]
-oci-distribution = { git = "https://github.com/krustlet/oci-distribution.git", rev = "f44124c" }
-
 [dev-dependencies]
 tempfile = "3.1.0"
 test-utils = { path = "../libs/test-utils" }
 which = "4.3.0"
 
 [workspace]
-resolver = "2"
 members = [
     "rustjail",
 ]

src/agent/Makefile

@@ -33,12 +33,6 @@ ifeq ($(SECCOMP),yes)
     override EXTRA_RUSTFEATURES += seccomp
 endif
 
-include ../../utils.mk
-
-ifeq ($(ARCH), ppc64le)
-    override ARCH = powerpc64le
-endif
-
 ##VAR STANDARD_OCI_RUNTIME=yes|no define if agent enables standard oci runtime feature
 STANDARD_OCI_RUNTIME := no
 
@@ -51,6 +45,8 @@ ifneq ($(EXTRA_RUSTFEATURES),)
     override EXTRA_RUSTFEATURES := --features "$(EXTRA_RUSTFEATURES)"
 endif
 
+include ../../utils.mk
+
 TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET)
 
 ##VAR DESTDIR=<path> is a directory prepended to each installed target file

src/agent/rustjail/Cargo.toml

@@ -18,7 +18,7 @@ scopeguard = "1.0.0"
 capctl = "0.2.0"
 lazy_static = "1.3.0"
 libc = "0.2.58"
-protobuf = "3.2.0"
+protobuf = "2.27.0"
 slog = "2.5.2"
 slog-scope = "4.1.2"
 scan_fmt = "0.2.6"
@@ -29,7 +29,7 @@ cgroups = { package = "cgroups-rs", version = "0.3.2" }
 rlimit = "0.5.3"
 cfg-if = "0.1.0"
 
-tokio = { version = "1.28.1", features = ["sync", "io-util", "process", "time", "macros", "rt"] }
+tokio = { version = "1.2.0", features = ["sync", "io-util", "process", "time", "macros", "rt"] }
 futures = "0.3.17"
 async-trait = "0.1.31"
 inotify = "0.9.2"

src/agent/rustjail/src/cgroups/fs/mod.rs

@@ -27,7 +27,7 @@ use oci::{
     LinuxNetwork, LinuxPids, LinuxResources,
 };
 
-use protobuf::MessageField;
+use protobuf::{CachedSize, RepeatedField, SingularPtrField, UnknownFields};
 use protocols::agent::{
     BlkioStats, BlkioStatsEntry, CgroupStats, CpuStats, CpuUsage, HugetlbStats, MemoryData,
     MemoryStats, PidsStats, ThrottlingData,
@@ -50,7 +50,7 @@ macro_rules! get_controller_or_return_singular_none {
     ($cg:ident) => {
         match $cg.controller_of() {
             Some(c) => c,
-            None => return MessageField::none(),
+            None => return SingularPtrField::none(),
         }
     };
 }
@@ -134,10 +134,11 @@ impl CgroupManager for Manager {
 
         let throttling_data = get_cpu_stats(&self.cgroup);
 
-        let cpu_stats = MessageField::some(CpuStats {
+        let cpu_stats = SingularPtrField::some(CpuStats {
             cpu_usage,
             throttling_data,
-            ..Default::default()
+            unknown_fields: UnknownFields::default(),
+            cached_size: CachedSize::default(),
         });
 
         // Memorystats
@@ -159,7 +160,8 @@ impl CgroupManager for Manager {
             pids_stats,
             blkio_stats,
             hugetlb_stats,
-            ..Default::default()
+            unknown_fields: UnknownFields::default(),
+            cached_size: CachedSize::default(),
         })
     }
 
@@ -444,14 +446,14 @@ fn set_memory_resources(cg: &cgroups::Cgroup, memory: &LinuxMemory, update: bool
         let memstat = get_memory_stats(cg)
             .into_option()
             .ok_or_else(|| anyhow!("failed to get the cgroup memory stats"))?;
-        let memusage = memstat.usage();
+        let memusage = memstat.get_usage();
 
         // When update memory limit, the kernel would check the current memory limit
         // set against the new swap setting, if the current memory limit is large than
         // the new swap, then set limit first, otherwise the kernel would complain and
         // refused to set; on the other hand, if the current memory limit is smaller than
         // the new swap, then we should set the swap first and then set the memor limit.
-        if swap == -1 || memusage.limit() < swap as u64 {
+        if swap == -1 || memusage.get_limit() < swap as u64 {
             mem_controller.set_memswap_limit(swap)?;
             set_resource!(mem_controller, set_limit, memory, limit);
         } else {
@@ -543,11 +545,8 @@ fn linux_device_to_cgroup_device(d: &LinuxDevice) -> Option<DeviceResource> {
 }
 
 fn linux_device_group_to_cgroup_device(d: &LinuxDeviceCgroup) -> Option<DeviceResource> {
-    let dev_type = match &d.r#type {
-        Some(t_s) => match DeviceType::from_char(t_s.chars().next()) {
-            Some(t_c) => t_c,
-            None => return None,
-        },
+    let dev_type = match DeviceType::from_char(d.r#type.chars().next()) {
+        Some(t) => t,
         None => return None,
     };
 
@@ -604,7 +603,7 @@ lazy_static! {
             // all mknod to all char devices
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: Some("c".to_string()),
+                r#type: "c".to_string(),
                 major: Some(WILDCARD),
                 minor: Some(WILDCARD),
                 access: "m".to_string(),
@@ -613,7 +612,7 @@ lazy_static! {
             // all mknod to all block devices
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: Some("b".to_string()),
+                r#type: "b".to_string(),
                 major: Some(WILDCARD),
                 minor: Some(WILDCARD),
                 access: "m".to_string(),
@@ -622,7 +621,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/console
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: Some("c".to_string()),
+                r#type: "c".to_string(),
                 major: Some(5),
                 minor: Some(1),
                 access: "rwm".to_string(),
@@ -631,7 +630,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/pts/<N>
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: Some("c".to_string()),
+                r#type: "c".to_string(),
                 major: Some(136),
                 minor: Some(WILDCARD),
                 access: "rwm".to_string(),
@@ -640,7 +639,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/ptmx
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: Some("c".to_string()),
+                r#type: "c".to_string(),
                 major: Some(5),
                 minor: Some(2),
                 access: "rwm".to_string(),
@@ -649,7 +648,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/net/tun
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: Some("c".to_string()),
+                r#type: "c".to_string(),
                 major: Some(10),
                 minor: Some(200),
                 access: "rwm".to_string(),
@@ -658,20 +657,21 @@ lazy_static! {
     };
 }
 
-fn get_cpu_stats(cg: &cgroups::Cgroup) -> MessageField<ThrottlingData> {
+fn get_cpu_stats(cg: &cgroups::Cgroup) -> SingularPtrField<ThrottlingData> {
     let cpu_controller: &CpuController = get_controller_or_return_singular_none!(cg);
     let stat = cpu_controller.cpu().stat;
     let h = lines_to_map(&stat);
 
-    MessageField::some(ThrottlingData {
+    SingularPtrField::some(ThrottlingData {
         periods: *h.get("nr_periods").unwrap_or(&0),
         throttled_periods: *h.get("nr_throttled").unwrap_or(&0),
         throttled_time: *h.get("throttled_time").unwrap_or(&0),
-        ..Default::default()
+        unknown_fields: UnknownFields::default(),
+        cached_size: CachedSize::default(),
     })
 }
 
-fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> MessageField<CpuUsage> {
+fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> SingularPtrField<CpuUsage> {
     if let Some(cpuacct_controller) = cg.controller_of::<CpuAcctController>() {
         let cpuacct = cpuacct_controller.cpuacct();
 
@@ -685,12 +685,13 @@ fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> MessageField<CpuUsage> {
 
         let percpu_usage = line_to_vec(&cpuacct.usage_percpu);
 
-        return MessageField::some(CpuUsage {
+        return SingularPtrField::some(CpuUsage {
             total_usage,
             percpu_usage,
             usage_in_kernelmode,
             usage_in_usermode,
-            ..Default::default()
+            unknown_fields: UnknownFields::default(),
+            cached_size: CachedSize::default(),
         });
     }
 
@@ -703,16 +704,17 @@ fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> MessageField<CpuUsage> {
     let total_usage = *h.get("usage_usec").unwrap_or(&0);
     let percpu_usage = vec![];
 
-    MessageField::some(CpuUsage {
+    SingularPtrField::some(CpuUsage {
         total_usage,
         percpu_usage,
         usage_in_kernelmode,
         usage_in_usermode,
-        ..Default::default()
+        unknown_fields: UnknownFields::default(),
+        cached_size: CachedSize::default(),
     })
 }
 
-fn get_memory_stats(cg: &cgroups::Cgroup) -> MessageField<MemoryStats> {
+fn get_memory_stats(cg: &cgroups::Cgroup) -> SingularPtrField<MemoryStats> {
     let memory_controller: &MemController = get_controller_or_return_singular_none!(cg);
 
     // cache from memory stat
@@ -724,48 +726,52 @@ fn get_memory_stats(cg: &cgroups::Cgroup) -> MessageField<MemoryStats> {
     let use_hierarchy = value == 1;
 
     // get memory data
-    let usage = MessageField::some(MemoryData {
+    let usage = SingularPtrField::some(MemoryData {
         usage: memory.usage_in_bytes,
         max_usage: memory.max_usage_in_bytes,
         failcnt: memory.fail_cnt,
         limit: memory.limit_in_bytes as u64,
-        ..Default::default()
+        unknown_fields: UnknownFields::default(),
+        cached_size: CachedSize::default(),
     });
 
     // get swap usage
     let memswap = memory_controller.memswap();
 
-    let swap_usage = MessageField::some(MemoryData {
+    let swap_usage = SingularPtrField::some(MemoryData {
         usage: memswap.usage_in_bytes,
         max_usage: memswap.max_usage_in_bytes,
         failcnt: memswap.fail_cnt,
         limit: memswap.limit_in_bytes as u64,
-        ..Default::default()
+        unknown_fields: UnknownFields::default(),
+        cached_size: CachedSize::default(),
     });
 
     // get kernel usage
     let kmem_stat = memory_controller.kmem_stat();
 
-    let kernel_usage = MessageField::some(MemoryData {
+    let kernel_usage = SingularPtrField::some(MemoryData {
         usage: kmem_stat.usage_in_bytes,
         max_usage: kmem_stat.max_usage_in_bytes,
         failcnt: kmem_stat.fail_cnt,
         limit: kmem_stat.limit_in_bytes as u64,
-        ..Default::default()
+        unknown_fields: UnknownFields::default(),
+        cached_size: CachedSize::default(),
     });
 
-    MessageField::some(MemoryStats {
+    SingularPtrField::some(MemoryStats {
         cache,
         usage,
         swap_usage,
         kernel_usage,
         use_hierarchy,
         stats: memory.stat.raw,
-        ..Default::default()
+        unknown_fields: UnknownFields::default(),
+        cached_size: CachedSize::default(),
     })
 }
 
-fn get_pids_stats(cg: &cgroups::Cgroup) -> MessageField<PidsStats> {
+fn get_pids_stats(cg: &cgroups::Cgroup) -> SingularPtrField<PidsStats> {
     let pid_controller: &PidController = get_controller_or_return_singular_none!(cg);
 
     let current = pid_controller.get_pid_current().unwrap_or(0);
@@ -779,10 +785,11 @@ fn get_pids_stats(cg: &cgroups::Cgroup) -> MessageField<PidsStats> {
         },
     } as u64;
 
-    MessageField::some(PidsStats {
+    SingularPtrField::some(PidsStats {
         current,
         limit,
-        ..Default::default()
+        unknown_fields: UnknownFields::default(),
+        cached_size: CachedSize::default(),
     })
 }
 
@@ -818,8 +825,8 @@ https://github.com/opencontainers/runc/blob/a5847db387ae28c0ca4ebe4beee1a76900c8
     Total 0
 */
 
-fn get_blkio_stat_blkiodata(blkiodata: &[BlkIoData]) -> Vec<BlkioStatsEntry> {
-    let mut m = Vec::new();
+fn get_blkio_stat_blkiodata(blkiodata: &[BlkIoData]) -> RepeatedField<BlkioStatsEntry> {
+    let mut m = RepeatedField::new();
     if blkiodata.is_empty() {
         return m;
     }
@@ -832,15 +839,16 @@ fn get_blkio_stat_blkiodata(blkiodata: &[BlkIoData]) -> Vec<BlkioStatsEntry> {
             minor: d.minor as u64,
             op: op.clone(),
             value: d.data,
-            ..Default::default()
+            unknown_fields: UnknownFields::default(),
+            cached_size: CachedSize::default(),
         });
     }
 
     m
 }
 
-fn get_blkio_stat_ioservice(services: &[IoService]) -> Vec<BlkioStatsEntry> {
-    let mut m = Vec::new();
+fn get_blkio_stat_ioservice(services: &[IoService]) -> RepeatedField<BlkioStatsEntry> {
+    let mut m = RepeatedField::new();
 
     if services.is_empty() {
         return m;
@@ -864,16 +872,17 @@ fn build_blkio_stats_entry(major: i16, minor: i16, op: &str, value: u64) -> Blki
         minor: minor as u64,
         op: op.to_string(),
         value,
-        ..Default::default()
+        unknown_fields: UnknownFields::default(),
+        cached_size: CachedSize::default(),
     }
 }
 
-fn get_blkio_stats_v2(cg: &cgroups::Cgroup) -> MessageField<BlkioStats> {
+fn get_blkio_stats_v2(cg: &cgroups::Cgroup) -> SingularPtrField<BlkioStats> {
     let blkio_controller: &BlkIoController = get_controller_or_return_singular_none!(cg);
     let blkio = blkio_controller.blkio();
 
     let mut resp = BlkioStats::new();
-    let mut blkio_stats = Vec::new();
+    let mut blkio_stats = RepeatedField::new();
 
     let stat = blkio.io_stat;
     for s in stat {
@@ -889,10 +898,10 @@ fn get_blkio_stats_v2(cg: &cgroups::Cgroup) -> MessageField<BlkioStats> {
 
     resp.io_service_bytes_recursive = blkio_stats;
 
-    MessageField::some(resp)
+    SingularPtrField::some(resp)
 }
 
-fn get_blkio_stats(cg: &cgroups::Cgroup) -> MessageField<BlkioStats> {
+fn get_blkio_stats(cg: &cgroups::Cgroup) -> SingularPtrField<BlkioStats> {
     if cg.v2() {
         return get_blkio_stats_v2(cg);
     }
@@ -925,7 +934,7 @@ fn get_blkio_stats(cg: &cgroups::Cgroup) -> MessageField<BlkioStats> {
         m.sectors_recursive = get_blkio_stat_blkiodata(&blkio.sectors_recursive);
     }
 
-    MessageField::some(m)
+    SingularPtrField::some(m)
 }
 
 fn get_hugetlb_stats(cg: &cgroups::Cgroup) -> HashMap<String, HugetlbStats> {
@@ -949,7 +958,8 @@ fn get_hugetlb_stats(cg: &cgroups::Cgroup) -> HashMap<String, HugetlbStats> {
                 usage,
                 max_usage,
                 failcnt,
-                ..Default::default()
+                unknown_fields: UnknownFields::default(),
+                cached_size: CachedSize::default(),
             },
         );
     }

src/agent/rustjail/src/cgroups/mock.rs

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use protobuf::MessageField;
+use protobuf::{CachedSize, SingularPtrField, UnknownFields};
 
 use crate::cgroups::Manager as CgroupManager;
 use crate::protocols::agent::{BlkioStats, CgroupStats, CpuStats, MemoryStats, PidsStats};
@@ -33,12 +33,13 @@ impl CgroupManager for Manager {
 
     fn get_stats(&self) -> Result<CgroupStats> {
         Ok(CgroupStats {
-            cpu_stats: MessageField::some(CpuStats::default()),
-            memory_stats: MessageField::some(MemoryStats::new()),
-            pids_stats: MessageField::some(PidsStats::new()),
-            blkio_stats: MessageField::some(BlkioStats::new()),
+            cpu_stats: SingularPtrField::some(CpuStats::default()),
+            memory_stats: SingularPtrField::some(MemoryStats::new()),
+            pids_stats: SingularPtrField::some(PidsStats::new()),
+            blkio_stats: SingularPtrField::some(BlkioStats::new()),
             hugetlb_stats: HashMap::new(),
-            ..Default::default()
+            unknown_fields: UnknownFields::default(),
+            cached_size: CachedSize::default(),
         })
     }
 

src/agent/rustjail/src/container.rs

@@ -48,7 +48,7 @@ use nix::unistd::{self, fork, ForkResult, Gid, Pid, Uid, User};
 use std::os::unix::fs::MetadataExt;
 use std::os::unix::io::AsRawFd;
 
-use protobuf::MessageField;
+use protobuf::SingularPtrField;
 
 use oci::State as OCIState;
 use regex::Regex;
@@ -374,18 +374,13 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     let buf = read_sync(crfd)?;
     let spec_str = std::str::from_utf8(&buf)?;
     let spec: oci::Spec = serde_json::from_str(spec_str)?;
+
     log_child!(cfd_log, "notify parent to send oci process");
     write_sync(cwfd, SYNC_SUCCESS, "")?;
 
     let buf = read_sync(crfd)?;
     let process_str = std::str::from_utf8(&buf)?;
     let oci_process: oci::Process = serde_json::from_str(process_str)?;
-    log_child!(cfd_log, "notify parent to send oci state");
-    write_sync(cwfd, SYNC_SUCCESS, "")?;
-
-    let buf = read_sync(crfd)?;
-    let state_str = std::str::from_utf8(&buf)?;
-    let mut state: oci::State = serde_json::from_str(state_str)?;
     log_child!(cfd_log, "notify parent to send cgroup manager");
     write_sync(cwfd, SYNC_SUCCESS, "")?;
 
@@ -748,19 +743,6 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         unistd::read(fd, buf)?;
     }
 
-    if init {
-        // StartContainer Hooks:
-        // * should be run in container namespace
-        // * should be run after container is created and before container is started (before user-specific command is executed)
-        // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#startcontainer-hooks
-        state.pid = std::process::id() as i32;
-        state.status = oci::ContainerState::Created;
-        if let Some(hooks) = spec.hooks.as_ref() {
-            let mut start_container_states = HookStates::new();
-            start_container_states.execute_hooks(&hooks.start_container, Some(state))?;
-        }
-    }
-
     // With NoNewPrivileges, we should set seccomp as close to
     // do_exec as possible in order to reduce the amount of
     // system calls in the seccomp profiles.
@@ -875,7 +857,7 @@ impl BaseContainer for LinuxContainer {
         // what about network interface stats?
 
         Ok(StatsContainerResponse {
-            cgroup_stats: MessageField::some(self.cgroup_manager.as_ref().get_stats()?),
+            cgroup_stats: SingularPtrField::some(self.cgroup_manager.as_ref().get_stats()?),
             ..Default::default()
         })
     }
@@ -1341,6 +1323,7 @@ async fn join_namespaces(
     write_async(pipe_w, SYNC_DATA, spec_str.as_str()).await?;
 
     info!(logger, "wait child received oci spec");
+
     read_async(pipe_r).await?;
 
     info!(logger, "send oci process from parent to child");
@@ -1350,13 +1333,6 @@ async fn join_namespaces(
     info!(logger, "wait child received oci process");
     read_async(pipe_r).await?;
 
-    info!(logger, "try to send state from parent to child");
-    let state_str = serde_json::to_string(st)?;
-    write_async(pipe_w, SYNC_DATA, state_str.as_str()).await?;
-
-    info!(logger, "wait child received oci state");
-    read_async(pipe_r).await?;
-
     let cm_str = if use_systemd_cgroup {
         serde_json::to_string(cm.as_any()?.downcast_ref::<SystemdManager>().unwrap())
     } else {

src/agent/rustjail/src/lib.rs

@@ -82,11 +82,11 @@ pub fn process_grpc_to_oci(p: &grpc::Process) -> oci::Process {
         let cap = p.Capabilities.as_ref().unwrap();
 
         Some(oci::LinuxCapabilities {
-            bounding: cap.Bounding.clone(),
-            effective: cap.Effective.clone(),
-            inheritable: cap.Inheritable.clone(),
-            permitted: cap.Permitted.clone(),
-            ambient: cap.Ambient.clone(),
+            bounding: cap.Bounding.clone().into_vec(),
+            effective: cap.Effective.clone().into_vec(),
+            inheritable: cap.Inheritable.clone().into_vec(),
+            permitted: cap.Permitted.clone().into_vec(),
+            ambient: cap.Ambient.clone().into_vec(),
         })
     } else {
         None
@@ -108,8 +108,8 @@ pub fn process_grpc_to_oci(p: &grpc::Process) -> oci::Process {
         terminal: p.Terminal,
         console_size,
         user,
-        args: p.Args.clone(),
-        env: p.Env.clone(),
+        args: p.Args.clone().into_vec(),
+        env: p.Env.clone().into_vec(),
         cwd: p.Cwd.clone(),
         capabilities,
         rlimits,
@@ -130,9 +130,9 @@ fn root_grpc_to_oci(root: &grpc::Root) -> oci::Root {
 fn mount_grpc_to_oci(m: &grpc::Mount) -> oci::Mount {
     oci::Mount {
         destination: m.destination.clone(),
-        r#type: m.type_.clone(),
+        r#type: m.field_type.clone(),
         source: m.source.clone(),
-        options: m.options.clone(),
+        options: m.options.clone().into_vec(),
     }
 }
 
@@ -143,8 +143,8 @@ fn hook_grpc_to_oci(h: &[grpcHook]) -> Vec<oci::Hook> {
     for e in h.iter() {
         r.push(oci::Hook {
             path: e.Path.clone(),
-            args: e.Args.clone(),
-            env: e.Env.clone(),
+            args: e.Args.clone().into_vec(),
+            env: e.Env.clone().into_vec(),
             timeout: Some(e.Timeout as i32),
         });
     }
@@ -153,17 +153,13 @@ fn hook_grpc_to_oci(h: &[grpcHook]) -> Vec<oci::Hook> {
 
 fn hooks_grpc_to_oci(h: &grpc::Hooks) -> oci::Hooks {
     let prestart = hook_grpc_to_oci(h.Prestart.as_ref());
-    let create_runtime = hook_grpc_to_oci(h.CreateRuntime.as_ref());
-    let create_container = hook_grpc_to_oci(h.CreateContainer.as_ref());
-    let start_container = hook_grpc_to_oci(h.StartContainer.as_ref());
+
     let poststart = hook_grpc_to_oci(h.Poststart.as_ref());
+
     let poststop = hook_grpc_to_oci(h.Poststop.as_ref());
 
     oci::Hooks {
         prestart,
-        create_runtime,
-        create_container,
-        start_container,
         poststart,
         poststop,
     }
@@ -241,12 +237,6 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
     let devices = {
         let mut d = Vec::new();
         for dev in res.Devices.iter() {
-            let dev_type = if dev.Type.is_empty() {
-                None
-            } else {
-                Some(dev.Type.clone())
-            };
-
             let major = if dev.Major == -1 {
                 None
             } else {
@@ -260,7 +250,7 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
             };
             d.push(oci::LinuxDeviceCgroup {
                 allow: dev.Allow,
-                r#type: dev_type,
+                r#type: dev.Type.clone(),
                 major,
                 minor,
                 access: dev.Access.clone(),
@@ -365,7 +355,7 @@ fn seccomp_grpc_to_oci(sec: &grpc::LinuxSeccomp) -> oci::LinuxSeccomp {
             let mut args = Vec::new();
 
             let errno_ret: u32 = if sys.has_errnoret() {
-                sys.errnoret()
+                sys.get_errnoret()
             } else {
                 libc::EPERM as u32
             };
@@ -380,7 +370,7 @@ fn seccomp_grpc_to_oci(sec: &grpc::LinuxSeccomp) -> oci::LinuxSeccomp {
             }
 
             r.push(oci::LinuxSyscall {
-                names: sys.Names.clone(),
+                names: sys.Names.clone().into_vec(),
                 action: sys.Action.clone(),
                 errno_ret,
                 args,
@@ -391,8 +381,8 @@ fn seccomp_grpc_to_oci(sec: &grpc::LinuxSeccomp) -> oci::LinuxSeccomp {
 
     oci::LinuxSeccomp {
         default_action: sec.DefaultAction.clone(),
-        architectures: sec.Architectures.clone(),
-        flags: sec.Flags.clone(),
+        architectures: sec.Architectures.clone().into_vec(),
+        flags: sec.Flags.clone().into_vec(),
         syscalls,
     }
 }
@@ -462,8 +452,8 @@ fn linux_grpc_to_oci(l: &grpc::Linux) -> oci::Linux {
         devices,
         seccomp,
         rootfs_propagation: l.RootfsPropagation.clone(),
-        masked_paths: l.MaskedPaths.clone(),
-        readonly_paths: l.ReadonlyPaths.clone(),
+        masked_paths: l.MaskedPaths.clone().into_vec(),
+        readonly_paths: l.ReadonlyPaths.clone().into_vec(),
         mount_label: l.MountLabel.clone(),
         intel_rdt,
     }
@@ -564,30 +554,35 @@ mod tests {
                 // All fields specified
                 grpcproc: grpc::Process {
                     Terminal: true,
-                    ConsoleSize: protobuf::MessageField::<grpc::Box>::some(grpc::Box {
+                    ConsoleSize: protobuf::SingularPtrField::<grpc::Box>::some(grpc::Box {
                         Height: 123,
                         Width: 456,
                         ..Default::default()
                     }),
-                    User: protobuf::MessageField::<grpc::User>::some(grpc::User {
+                    User: protobuf::SingularPtrField::<grpc::User>::some(grpc::User {
                         UID: 1234,
                         GID: 5678,
                         AdditionalGids: Vec::from([910, 1112]),
                         Username: String::from("username"),
                         ..Default::default()
                     }),
-                    Args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                    Env: Vec::from([String::from("env")]),
+                    Args: protobuf::RepeatedField::from(Vec::from([
+                        String::from("arg1"),
+                        String::from("arg2"),
+                    ])),
+                    Env: protobuf::RepeatedField::from(Vec::from([String::from("env")])),
                     Cwd: String::from("cwd"),
-                    Capabilities: protobuf::MessageField::some(grpc::LinuxCapabilities {
-                        Bounding: Vec::from([String::from("bnd")]),
-                        Effective: Vec::from([String::from("eff")]),
-                        Inheritable: Vec::from([String::from("inher")]),
-                        Permitted: Vec::from([String::from("perm")]),
-                        Ambient: Vec::from([String::from("amb")]),
+                    Capabilities: protobuf::SingularPtrField::some(grpc::LinuxCapabilities {
+                        Bounding: protobuf::RepeatedField::from(Vec::from([String::from("bnd")])),
+                        Effective: protobuf::RepeatedField::from(Vec::from([String::from("eff")])),
+                        Inheritable: protobuf::RepeatedField::from(Vec::from([String::from(
+                            "inher",
+                        )])),
+                        Permitted: protobuf::RepeatedField::from(Vec::from([String::from("perm")])),
+                        Ambient: protobuf::RepeatedField::from(Vec::from([String::from("amb")])),
                         ..Default::default()
                     }),
-                    Rlimits: Vec::from([
+                    Rlimits: protobuf::RepeatedField::from(Vec::from([
                         grpc::POSIXRlimit {
                             Type: String::from("r#type"),
                             Hard: 123,
@@ -600,7 +595,7 @@ mod tests {
                             Soft: 1011,
                             ..Default::default()
                         },
-                    ]),
+                    ])),
                     NoNewPrivileges: true,
                     ApparmorProfile: String::from("apparmor profile"),
                     OOMScoreAdj: 123456,
@@ -650,7 +645,7 @@ mod tests {
             TestData {
                 // None ConsoleSize
                 grpcproc: grpc::Process {
-                    ConsoleSize: protobuf::MessageField::<grpc::Box>::none(),
+                    ConsoleSize: protobuf::SingularPtrField::<grpc::Box>::none(),
                     OOMScoreAdj: 0,
                     ..Default::default()
                 },
@@ -663,7 +658,7 @@ mod tests {
             TestData {
                 // None User
                 grpcproc: grpc::Process {
-                    User: protobuf::MessageField::<grpc::User>::none(),
+                    User: protobuf::SingularPtrField::<grpc::User>::none(),
                     OOMScoreAdj: 0,
                     ..Default::default()
                 },
@@ -681,7 +676,7 @@ mod tests {
             TestData {
                 // None Capabilities
                 grpcproc: grpc::Process {
-                    Capabilities: protobuf::MessageField::none(),
+                    Capabilities: protobuf::SingularPtrField::none(),
                     OOMScoreAdj: 0,
                     ..Default::default()
                 },
@@ -782,57 +777,60 @@ mod tests {
             TestData {
                 // All specified
                 grpchooks: grpc::Hooks {
-                    Prestart: Vec::from([
+                    Prestart: protobuf::RepeatedField::from(Vec::from([
                         grpc::Hook {
                             Path: String::from("prestartpath"),
-                            Args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                            Env: Vec::from([String::from("env1"), String::from("env2")]),
+                            Args: protobuf::RepeatedField::from(Vec::from([
+                                String::from("arg1"),
+                                String::from("arg2"),
+                            ])),
+                            Env: protobuf::RepeatedField::from(Vec::from([
+                                String::from("env1"),
+                                String::from("env2"),
+                            ])),
                             Timeout: 10,
                             ..Default::default()
                         },
                         grpc::Hook {
                             Path: String::from("prestartpath2"),
-                            Args: Vec::from([String::from("arg3"), String::from("arg4")]),
-                            Env: Vec::from([String::from("env3"), String::from("env4")]),
+                            Args: protobuf::RepeatedField::from(Vec::from([
+                                String::from("arg3"),
+                                String::from("arg4"),
+                            ])),
+                            Env: protobuf::RepeatedField::from(Vec::from([
+                                String::from("env3"),
+                                String::from("env4"),
+                            ])),
                             Timeout: 25,
                             ..Default::default()
                         },
-                    ]),
-                    Poststart: Vec::from([grpc::Hook {
+                    ])),
+                    Poststart: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
                         Path: String::from("poststartpath"),
-                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                        Env: Vec::from([String::from("env1"), String::from("env2")]),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
                         Timeout: 10,
                         ..Default::default()
-                    }]),
-                    Poststop: Vec::from([grpc::Hook {
+                    }])),
+                    Poststop: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
                         Path: String::from("poststoppath"),
-                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                        Env: Vec::from([String::from("env1"), String::from("env2")]),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
                         Timeout: 10,
                         ..Default::default()
-                    }]),
-                    CreateRuntime: Vec::from([grpc::Hook {
-                        Path: String::from("createruntimepath"),
-                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                        Env: Vec::from([String::from("env1"), String::from("env2")]),
-                        Timeout: 10,
-                        ..Default::default()
-                    }]),
-                    CreateContainer: Vec::from([grpc::Hook {
-                        Path: String::from("createcontainerpath"),
-                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                        Env: Vec::from([String::from("env1"), String::from("env2")]),
-                        Timeout: 10,
-                        ..Default::default()
-                    }]),
-                    StartContainer: Vec::from([grpc::Hook {
-                        Path: String::from("startcontainerpath"),
-                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                        Env: Vec::from([String::from("env1"), String::from("env2")]),
-                        Timeout: 10,
-                        ..Default::default()
-                    }]),
+                    }])),
                     ..Default::default()
                 },
                 result: oci::Hooks {
@@ -862,65 +860,38 @@ mod tests {
                         env: Vec::from([String::from("env1"), String::from("env2")]),
                         timeout: Some(10),
                     }]),
-                    create_runtime: Vec::from([oci::Hook {
-                        path: String::from("createruntimepath"),
-                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                        env: Vec::from([String::from("env1"), String::from("env2")]),
-                        timeout: Some(10),
-                    }]),
-                    create_container: Vec::from([oci::Hook {
-                        path: String::from("createcontainerpath"),
-                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                        env: Vec::from([String::from("env1"), String::from("env2")]),
-                        timeout: Some(10),
-                    }]),
-                    start_container: Vec::from([oci::Hook {
-                        path: String::from("startcontainerpath"),
-                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                        env: Vec::from([String::from("env1"), String::from("env2")]),
-                        timeout: Some(10),
-                    }]),
                 },
             },
             TestData {
                 // Prestart empty
                 grpchooks: grpc::Hooks {
-                    Prestart: Vec::from([]),
-                    Poststart: Vec::from([grpc::Hook {
+                    Prestart: protobuf::RepeatedField::from(Vec::from([])),
+                    Poststart: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
                         Path: String::from("poststartpath"),
-                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                        Env: Vec::from([String::from("env1"), String::from("env2")]),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
                         Timeout: 10,
                         ..Default::default()
-                    }]),
-                    Poststop: Vec::from([grpc::Hook {
+                    }])),
+                    Poststop: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
                         Path: String::from("poststoppath"),
-                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                        Env: Vec::from([String::from("env1"), String::from("env2")]),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
                         Timeout: 10,
                         ..Default::default()
-                    }]),
-                    CreateRuntime: Vec::from([grpc::Hook {
-                        Path: String::from("createruntimepath"),
-                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                        Env: Vec::from([String::from("env1"), String::from("env2")]),
-                        Timeout: 10,
-                        ..Default::default()
-                    }]),
-                    CreateContainer: Vec::from([grpc::Hook {
-                        Path: String::from("createcontainerpath"),
-                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                        Env: Vec::from([String::from("env1"), String::from("env2")]),
-                        Timeout: 10,
-                        ..Default::default()
-                    }]),
-                    StartContainer: Vec::from([grpc::Hook {
-                        Path: String::from("startcontainerpath"),
-                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                        Env: Vec::from([String::from("env1"), String::from("env2")]),
-                        Timeout: 10,
-                        ..Default::default()
-                    }]),
+                    }])),
                     ..Default::default()
                 },
                 result: oci::Hooks {
@@ -937,24 +908,6 @@ mod tests {
                         env: Vec::from([String::from("env1"), String::from("env2")]),
                         timeout: Some(10),
                     }]),
-                    create_runtime: Vec::from([oci::Hook {
-                        path: String::from("createruntimepath"),
-                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                        env: Vec::from([String::from("env1"), String::from("env2")]),
-                        timeout: Some(10),
-                    }]),
-                    create_container: Vec::from([oci::Hook {
-                        path: String::from("createcontainerpath"),
-                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                        env: Vec::from([String::from("env1"), String::from("env2")]),
-                        timeout: Some(10),
-                    }]),
-                    start_container: Vec::from([oci::Hook {
-                        path: String::from("startcontainerpath"),
-                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                        env: Vec::from([String::from("env1"), String::from("env2")]),
-                        timeout: Some(10),
-                    }]),
                 },
             },
         ];
@@ -992,8 +945,11 @@ mod tests {
                 grpcmount: grpc::Mount {
                     destination: String::from("destination"),
                     source: String::from("source"),
-                    type_: String::from("fieldtype"),
-                    options: Vec::from([String::from("option1"), String::from("option2")]),
+                    field_type: String::from("fieldtype"),
+                    options: protobuf::RepeatedField::from(Vec::from([
+                        String::from("option1"),
+                        String::from("option2"),
+                    ])),
                     ..Default::default()
                 },
                 result: oci::Mount {
@@ -1007,8 +963,8 @@ mod tests {
                 grpcmount: grpc::Mount {
                     destination: String::from("destination"),
                     source: String::from("source"),
-                    type_: String::from("fieldtype"),
-                    options: Vec::new(),
+                    field_type: String::from("fieldtype"),
+                    options: protobuf::RepeatedField::from(Vec::new()),
                     ..Default::default()
                 },
                 result: oci::Mount {
@@ -1022,8 +978,8 @@ mod tests {
                 grpcmount: grpc::Mount {
                     destination: String::new(),
                     source: String::from("source"),
-                    type_: String::from("fieldtype"),
-                    options: Vec::from([String::from("option1")]),
+                    field_type: String::from("fieldtype"),
+                    options: protobuf::RepeatedField::from(Vec::from([String::from("option1")])),
                     ..Default::default()
                 },
                 result: oci::Mount {
@@ -1037,8 +993,8 @@ mod tests {
                 grpcmount: grpc::Mount {
                     destination: String::from("destination"),
                     source: String::from("source"),
-                    type_: String::new(),
-                    options: Vec::from([String::from("option1")]),
+                    field_type: String::new(),
+                    options: protobuf::RepeatedField::from(Vec::from([String::from("option1")])),
                     ..Default::default()
                 },
                 result: oci::Mount {
@@ -1098,15 +1054,27 @@ mod tests {
                 grpchook: &[
                     grpc::Hook {
                         Path: String::from("path"),
-                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                        Env: Vec::from([String::from("env1"), String::from("env2")]),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
                         Timeout: 10,
                         ..Default::default()
                     },
                     grpc::Hook {
                         Path: String::from("path2"),
-                        Args: Vec::from([String::from("arg3"), String::from("arg4")]),
-                        Env: Vec::from([String::from("env3"), String::from("env4")]),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg3"),
+                            String::from("arg4"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env3"),
+                            String::from("env4"),
+                        ])),
                         Timeout: 20,
                         ..Default::default()
                     },

src/agent/rustjail/src/mount.rs

@@ -35,7 +35,7 @@ use crate::log_child;
 // struct is populated from the content in the /proc/<pid>/mountinfo file.
 #[derive(std::fmt::Debug, PartialEq)]
 pub struct Info {
-    pub mount_point: String,
+    mount_point: String,
     optional: String,
     fstype: String,
 }
@@ -553,7 +553,7 @@ fn rootfs_parent_mount_private(path: &str) -> Result<()> {
 
 // Parse /proc/self/mountinfo because comparing Dev and ino does not work from
 // bind mounts
-pub fn parse_mount_table(mountinfo_path: &str) -> Result<Vec<Info>> {
+fn parse_mount_table(mountinfo_path: &str) -> Result<Vec<Info>> {
     let file = File::open(mountinfo_path)?;
     let reader = BufReader::new(file);
     let mut infos = Vec::new();

src/agent/src/ap.rs

@@ -1,79 +0,0 @@
-// Copyright (c) IBM Corp. 2023
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-use std::fmt;
-use std::str::FromStr;
-
-use anyhow::{anyhow, Context};
-
-// IBM Adjunct Processor (AP) is used for cryptographic operations
-// by IBM Crypto Express hardware security modules on IBM zSystem & LinuxONE (s390x).
-// In Linux, virtual cryptographic devices are called AP queues.
-// The name of an AP queue respects a format <xx>.<xxxx> in hexadecimal notation [1, p.467]:
-//   - <xx> is an adapter ID
-//   - <xxxx> is an adapter domain ID
-// [1] https://www.ibm.com/docs/en/linuxonibm/pdf/lku5dd05.pdf
-
-#[derive(Debug)]
-pub struct Address {
-    pub adapter_id: u8,
-    pub adapter_domain: u16,
-}
-
-impl Address {
-    pub fn new(adapter_id: u8, adapter_domain: u16) -> Address {
-        Address {
-            adapter_id,
-            adapter_domain,
-        }
-    }
-}
-
-impl FromStr for Address {
-    type Err = anyhow::Error;
-
-    fn from_str(s: &str) -> anyhow::Result<Self> {
-        let split: Vec<&str> = s.split('.').collect();
-        if split.len() != 2 {
-            return Err(anyhow!(
-                "Wrong AP bus format. It needs to be in the form <xx>.<xxxx> (e.g. 0a.003f), got {:?}",
-                s
-            ));
-        }
-
-        let adapter_id = u8::from_str_radix(split[0], 16).context(format!(
-            "Wrong AP bus format. AP ID needs to be in the form <xx> (e.g. 0a), got {:?}",
-            split[0]
-        ))?;
-        let adapter_domain = u16::from_str_radix(split[1], 16).context(format!(
-            "Wrong AP bus format. AP domain needs to be in the form <xxxx> (e.g. 003f), got {:?}",
-            split[1]
-        ))?;
-
-        Ok(Address::new(adapter_id, adapter_domain))
-    }
-}
-
-impl fmt::Display for Address {
-    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
-        write!(f, "{:02x}.{:04x}", self.adapter_id, self.adapter_domain)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_from_str() {
-        let device = Address::from_str("a.1").unwrap();
-        assert_eq!(format!("{}", device), "0a.0001");
-
-        assert!(Address::from_str("").is_err());
-        assert!(Address::from_str(".").is_err());
-        assert!(Address::from_str("0.0.0").is_err());
-        assert!(Address::from_str("0g.0000").is_err());
-        assert!(Address::from_str("0a.10000").is_err());
-    }
-}

src/agent/src/config.rs

@@ -11,7 +11,6 @@ use std::fs;
 use std::str::FromStr;
 use std::time;
 use tracing::instrument;
-use url::Url;
 
 use kata_types::config::default::DEFAULT_AGENT_VSOCK_PORT;
 
@@ -26,11 +25,6 @@ const LOG_VPORT_OPTION: &str = "agent.log_vport";
 const CONTAINER_PIPE_SIZE_OPTION: &str = "agent.container_pipe_size";
 const UNIFIED_CGROUP_HIERARCHY_OPTION: &str = "agent.unified_cgroup_hierarchy";
 const CONFIG_FILE: &str = "agent.config_file";
-const AA_KBC_PARAMS: &str = "agent.aa_kbc_params";
-const HTTPS_PROXY: &str = "agent.https_proxy";
-const NO_PROXY: &str = "agent.no_proxy";
-const ENABLE_DATA_INTEGRITY: &str = "agent.data_integrity";
-const ENABLE_SIGNATURE_VERIFICATION: &str = "agent.enable_signature_verification";
 
 const DEFAULT_LOG_LEVEL: slog::Level = slog::Level::Info;
 const DEFAULT_HOTPLUG_TIMEOUT: time::Duration = time::Duration::from_secs(3);
@@ -83,12 +77,6 @@ pub struct AgentConfig {
     pub tracing: bool,
     pub endpoints: AgentEndpoints,
     pub supports_seccomp: bool,
-    pub container_policy_path: String,
-    pub aa_kbc_params: String,
-    pub https_proxy: String,
-    pub no_proxy: String,
-    pub data_integrity: bool,
-    pub enable_signature_verification: bool,
 }
 
 #[derive(Debug, Deserialize)]
@@ -104,12 +92,6 @@ pub struct AgentConfigBuilder {
     pub unified_cgroup_hierarchy: Option<bool>,
     pub tracing: Option<bool>,
     pub endpoints: Option<EndpointsConfig>,
-    pub container_policy_path: Option<String>,
-    pub aa_kbc_params: Option<String>,
-    pub https_proxy: Option<String>,
-    pub no_proxy: Option<String>,
-    pub data_integrity: Option<bool>,
-    pub enable_signature_verification: Option<bool>,
 }
 
 macro_rules! config_override {
@@ -171,12 +153,6 @@ impl Default for AgentConfig {
             tracing: false,
             endpoints: Default::default(),
             supports_seccomp: rpc::have_seccomp(),
-            container_policy_path: String::from(""),
-            aa_kbc_params: String::from(""),
-            https_proxy: String::from(""),
-            no_proxy: String::from(""),
-            data_integrity: false,
-            enable_signature_verification: true,
         }
     }
 }
@@ -205,16 +181,6 @@ impl FromStr for AgentConfig {
         config_override!(agent_config_builder, agent_config, server_addr);
         config_override!(agent_config_builder, agent_config, unified_cgroup_hierarchy);
         config_override!(agent_config_builder, agent_config, tracing);
-        config_override!(agent_config_builder, agent_config, container_policy_path);
-        config_override!(agent_config_builder, agent_config, aa_kbc_params);
-        config_override!(agent_config_builder, agent_config, https_proxy);
-        config_override!(agent_config_builder, agent_config, no_proxy);
-        config_override!(agent_config_builder, agent_config, data_integrity);
-        config_override!(
-            agent_config_builder,
-            agent_config,
-            enable_signature_verification
-        );
 
         // Populate the allowed endpoints hash set, if we got any from the config file.
         if let Some(endpoints) = agent_config_builder.endpoints {
@@ -234,7 +200,7 @@ impl AgentConfig {
         let config_position = args.iter().position(|a| a == "--config" || a == "-c");
         if let Some(config_position) = config_position {
             if let Some(config_file) = args.get(config_position + 1) {
-                return AgentConfig::from_config_file(config_file).context("AgentConfig from args");
+                return AgentConfig::from_config_file(config_file);
             } else {
                 panic!("The config argument wasn't formed properly: {:?}", args);
             }
@@ -243,10 +209,6 @@ impl AgentConfig {
         let mut config: AgentConfig = Default::default();
         let cmdline = fs::read_to_string(file)?;
         let params: Vec<&str> = cmdline.split_ascii_whitespace().collect();
-
-        let mut using_config_file = false;
-        // Check if there is config file before parsing params that might
-        // override values from the config file.
         for param in params.iter() {
             // If we get a configuration file path from the command line, we
             // generate our config from it.
@@ -254,15 +216,9 @@ impl AgentConfig {
             // or if it can't be parsed properly.
             if param.starts_with(format!("{}=", CONFIG_FILE).as_str()) {
                 let config_file = get_string_value(param)?;
-                config = AgentConfig::from_config_file(&config_file)
-                    .context("AgentConfig from kernel cmdline")
-                    .unwrap();
-                using_config_file = true;
-                break;
+                return AgentConfig::from_config_file(&config_file);
             }
-        }
 
-        for param in params.iter() {
             // parse cmdline flags
             parse_cmdline_param!(param, DEBUG_CONSOLE_FLAG, config.debug_console);
             parse_cmdline_param!(param, DEV_MODE_FLAG, config.dev_mode);
@@ -322,23 +278,6 @@ impl AgentConfig {
                 config.unified_cgroup_hierarchy,
                 get_bool_value
             );
-
-            parse_cmdline_param!(param, AA_KBC_PARAMS, config.aa_kbc_params, get_string_value);
-            parse_cmdline_param!(param, HTTPS_PROXY, config.https_proxy, get_url_value);
-            parse_cmdline_param!(param, NO_PROXY, config.no_proxy, get_string_value);
-            parse_cmdline_param!(
-                param,
-                ENABLE_DATA_INTEGRITY,
-                config.data_integrity,
-                get_bool_value
-            );
-
-            parse_cmdline_param!(
-                param,
-                ENABLE_SIGNATURE_VERIFICATION,
-                config.enable_signature_verification,
-                get_bool_value
-            );
         }
 
         if let Ok(addr) = env::var(SERVER_ADDR_ENV_VAR) {
@@ -358,17 +297,14 @@ impl AgentConfig {
         }
 
         // We did not get a configuration file: allow all endpoints.
-        if !using_config_file {
-            config.endpoints.all_allowed = true;
-        }
+        config.endpoints.all_allowed = true;
 
         Ok(config)
     }
 
     #[instrument]
     pub fn from_config_file(file: &str) -> Result<AgentConfig> {
-        let config = fs::read_to_string(file)
-            .with_context(|| format!("Failed to read config file {}", file))?;
+        let config = fs::read_to_string(file)?;
         AgentConfig::from_str(&config)
     }
 
@@ -495,12 +431,6 @@ fn get_container_pipe_size(param: &str) -> Result<i32> {
     Ok(value)
 }
 
-#[instrument]
-fn get_url_value(param: &str) -> Result<String> {
-    let value = get_string_value(param)?;
-    Ok(Url::parse(&value)?.to_string())
-}
-
 #[cfg(test)]
 mod tests {
     use test_utils::assert_result;
@@ -519,8 +449,6 @@ mod tests {
         assert!(!config.dev_mode);
         assert_eq!(config.log_level, DEFAULT_LOG_LEVEL);
         assert_eq!(config.hotplug_timeout, DEFAULT_HOTPLUG_TIMEOUT);
-        assert_eq!(config.container_policy_path, "");
-        assert!(config.enable_signature_verification);
     }
 
     #[test]
@@ -539,12 +467,6 @@ mod tests {
             server_addr: &'a str,
             unified_cgroup_hierarchy: bool,
             tracing: bool,
-            container_policy_path: &'a str,
-            aa_kbc_params: &'a str,
-            https_proxy: &'a str,
-            no_proxy: &'a str,
-            data_integrity: bool,
-            enable_signature_verification: bool,
         }
 
         impl Default for TestData<'_> {
@@ -560,12 +482,6 @@ mod tests {
                     server_addr: TEST_SERVER_ADDR,
                     unified_cgroup_hierarchy: false,
                     tracing: false,
-                    container_policy_path: "",
-                    aa_kbc_params: "",
-                    https_proxy: "",
-                    no_proxy: "",
-                    data_integrity: false,
-                    enable_signature_verification: true,
                 }
             }
         }
@@ -935,81 +851,6 @@ mod tests {
                 tracing: true,
                 ..Default::default()
             },
-            TestData {
-                contents: "agent.aa_kbc_params=offline_fs_kbc::null",
-                aa_kbc_params: "offline_fs_kbc::null",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.aa_kbc_params=eaa_kbc::127.0.0.1:50000",
-                aa_kbc_params: "eaa_kbc::127.0.0.1:50000",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.https_proxy=http://proxy.url.com:81/",
-                https_proxy: "http://proxy.url.com:81/",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.https_proxy=http://192.168.1.100:81/",
-                https_proxy: "http://192.168.1.100:81/",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.no_proxy=*.internal.url.com",
-                no_proxy: "*.internal.url.com",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.no_proxy=192.168.1.0/24,172.16.0.0/12",
-                no_proxy: "192.168.1.0/24,172.16.0.0/12",
-                ..Default::default()
-            },
-            TestData {
-                contents: "",
-                data_integrity: false,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.data_integrity=true",
-                data_integrity: true,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.data_integrity=false",
-                data_integrity: false,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.data_integrity=1",
-                data_integrity: true,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.data_integrity=0",
-                data_integrity: false,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.enable_signature_verification=false",
-                enable_signature_verification: false,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.enable_signature_verification=0",
-                enable_signature_verification: false,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.enable_signature_verification=1",
-                enable_signature_verification: true,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.enable_signature_verification=foo",
-                enable_signature_verification: false,
-                ..Default::default()
-            },
         ];
 
         let dir = tempdir().expect("failed to create tmpdir");
@@ -1057,20 +898,6 @@ mod tests {
             assert_eq!(d.container_pipe_size, config.container_pipe_size, "{}", msg);
             assert_eq!(d.server_addr, config.server_addr, "{}", msg);
             assert_eq!(d.tracing, config.tracing, "{}", msg);
-            assert_eq!(
-                d.container_policy_path, config.container_policy_path,
-                "{}",
-                msg
-            );
-            assert_eq!(d.aa_kbc_params, config.aa_kbc_params, "{}", msg);
-            assert_eq!(d.https_proxy, config.https_proxy, "{}", msg);
-            assert_eq!(d.no_proxy, config.no_proxy, "{}", msg);
-            assert_eq!(d.data_integrity, config.data_integrity, "{}", msg);
-            assert_eq!(
-                d.enable_signature_verification, config.enable_signature_verification,
-                "{}",
-                msg
-            );
 
             for v in vars_to_unset {
                 env::remove_var(v);
@@ -1572,50 +1399,4 @@ Caused by:
         // Verify that the default values are valid
         assert_eq!(config.hotplug_timeout, DEFAULT_HOTPLUG_TIMEOUT);
     }
-
-    #[test]
-    fn test_config_from_cmdline_and_config_file() {
-        let dir = tempdir().expect("failed to create tmpdir");
-
-        let agent_config = r#"
-               dev_mode = false
-               server_addr = 'vsock://8:2048'
-
-               [endpoints]
-               allowed = ["CreateContainer", "StartContainer"]
-              "#;
-
-        let config_path = dir.path().join("agent-config.toml");
-        let config_filename = config_path.to_str().expect("failed to get config filename");
-
-        fs::write(config_filename, agent_config).expect("failed to write agen config");
-
-        let cmdline = format!("agent.devmode agent.config_file={}", config_filename);
-
-        let cmdline_path = dir.path().join("cmdline");
-        let cmdline_filename = cmdline_path
-            .to_str()
-            .expect("failed to get cmdline filename");
-
-        fs::write(cmdline_filename, cmdline).expect("failed to write agen config");
-
-        let config = AgentConfig::from_cmdline(cmdline_filename, vec![])
-            .expect("failed to parse command line");
-
-        // Should be overwritten by cmdline
-        assert!(config.dev_mode);
-
-        // Should be from agent config
-        assert_eq!(config.server_addr, "vsock://8:2048");
-
-        // Should be from agent config
-        assert_eq!(
-            config.endpoints.allowed,
-            vec!["CreateContainer".to_string(), "StartContainer".to_string()]
-                .iter()
-                .cloned()
-                .collect()
-        );
-        assert!(!config.endpoints.all_allowed);
-    }
 }

src/agent/src/device.rs

@@ -16,12 +16,13 @@ use std::str::FromStr;
 use std::sync::Arc;
 use tokio::sync::Mutex;
 
+#[cfg(target_arch = "s390x")]
+use crate::ccw;
 use crate::linux_abi::*;
 use crate::pci;
 use crate::sandbox::Sandbox;
 use crate::uevent::{wait_for_uevent, Uevent, UeventMatcher};
 use anyhow::{anyhow, Context, Result};
-use cfg_if::cfg_if;
 use oci::{LinuxDeviceCgroup, LinuxResources, Spec};
 use protocols::agent::Device;
 use tracing::instrument;
@@ -34,7 +35,7 @@ macro_rules! sl {
 }
 
 const VM_ROOTFS: &str = "/";
-const BLOCK: &str = "block";
+
 pub const DRIVER_9P_TYPE: &str = "9p";
 pub const DRIVER_VIRTIOFS_TYPE: &str = "virtio-fs";
 pub const DRIVER_BLK_TYPE: &str = "blk";
@@ -45,22 +46,14 @@ pub const DRIVER_NVDIMM_TYPE: &str = "nvdimm";
 pub const DRIVER_EPHEMERAL_TYPE: &str = "ephemeral";
 pub const DRIVER_LOCAL_TYPE: &str = "local";
 pub const DRIVER_WATCHABLE_BIND_TYPE: &str = "watchable-bind";
-// VFIO PCI device to be bound to a guest kernel driver
-pub const DRIVER_VFIO_PCI_GK_TYPE: &str = "vfio-pci-gk";
-// VFIO PCI device to be bound to vfio-pci and made available inside the
+// VFIO device to be bound to a guest kernel driver
+pub const DRIVER_VFIO_GK_TYPE: &str = "vfio-gk";
+// VFIO device to be bound to vfio-pci and made available inside the
 // container as a VFIO device node
-pub const DRIVER_VFIO_PCI_TYPE: &str = "vfio-pci";
-pub const DRIVER_VFIO_AP_TYPE: &str = "vfio-ap";
+pub const DRIVER_VFIO_TYPE: &str = "vfio";
 pub const DRIVER_OVERLAYFS_TYPE: &str = "overlayfs";
 pub const FS_TYPE_HUGETLB: &str = "hugetlbfs";
 
-cfg_if! {
-    if #[cfg(target_arch = "s390x")] {
-        use crate::ap;
-        use crate::ccw;
-    }
-}
-
 #[instrument]
 pub fn online_device(path: &str) -> Result<()> {
     fs::write(path, "1")?;
@@ -204,7 +197,7 @@ impl ScsiBlockMatcher {
 
 impl UeventMatcher for ScsiBlockMatcher {
     fn is_match(&self, uev: &Uevent) -> bool {
-        uev.subsystem == BLOCK && uev.devpath.contains(&self.search) && !uev.devname.is_empty()
+        uev.subsystem == "block" && uev.devpath.contains(&self.search) && !uev.devname.is_empty()
     }
 }
 
@@ -238,7 +231,7 @@ impl VirtioBlkPciMatcher {
 
 impl UeventMatcher for VirtioBlkPciMatcher {
     fn is_match(&self, uev: &Uevent) -> bool {
-        uev.subsystem == BLOCK && self.rex.is_match(&uev.devpath) && !uev.devname.is_empty()
+        uev.subsystem == "block" && self.rex.is_match(&uev.devpath) && !uev.devname.is_empty()
     }
 }
 
@@ -287,7 +280,7 @@ pub async fn get_virtio_blk_ccw_device_name(
     sandbox: &Arc<Mutex<Sandbox>>,
     device: &ccw::Device,
 ) -> Result<String> {
-    let matcher = VirtioBlkCCWMatcher::new(CCW_ROOT_BUS_PATH, device);
+    let matcher = VirtioBlkCCWMatcher::new(&create_ccw_root_bus_path(), device);
     let uev = wait_for_uevent(sandbox, matcher).await?;
     let devname = uev.devname;
     return match Path::new(SYSTEM_DEV_PATH).join(&devname).to_str() {
@@ -311,7 +304,7 @@ impl PmemBlockMatcher {
 
 impl UeventMatcher for PmemBlockMatcher {
     fn is_match(&self, uev: &Uevent) -> bool {
-        uev.subsystem == BLOCK
+        uev.subsystem == "block"
             && uev.devpath.starts_with(ACPI_DEV_PATH)
             && uev.devpath.ends_with(&self.suffix)
             && !uev.devname.is_empty()
@@ -408,81 +401,6 @@ async fn get_vfio_device_name(sandbox: &Arc<Mutex<Sandbox>>, grp: IommuGroup) ->
     Ok(format!("{}/{}", SYSTEM_DEV_PATH, &uev.devname))
 }
 
-#[cfg(target_arch = "s390x")]
-#[derive(Debug)]
-struct ApMatcher {
-    syspath: String,
-}
-
-#[cfg(target_arch = "s390x")]
-impl ApMatcher {
-    fn new(address: ap::Address) -> ApMatcher {
-        ApMatcher {
-            syspath: format!(
-                "{}/card{:02x}/{}",
-                AP_ROOT_BUS_PATH, address.adapter_id, address
-            ),
-        }
-    }
-}
-
-#[cfg(target_arch = "s390x")]
-impl UeventMatcher for ApMatcher {
-    fn is_match(&self, uev: &Uevent) -> bool {
-        uev.action == "add" && uev.devpath == self.syspath
-    }
-}
-
-#[cfg(target_arch = "s390x")]
-#[instrument]
-async fn wait_for_ap_device(sandbox: &Arc<Mutex<Sandbox>>, address: ap::Address) -> Result<()> {
-    let matcher = ApMatcher::new(address);
-    wait_for_uevent(sandbox, matcher).await?;
-    Ok(())
-}
-
-#[derive(Debug)]
-struct MmioBlockMatcher {
-    suffix: String,
-}
-
-impl MmioBlockMatcher {
-    fn new(devname: &str) -> MmioBlockMatcher {
-        MmioBlockMatcher {
-            suffix: format!(r"/block/{}", devname),
-        }
-    }
-}
-
-impl UeventMatcher for MmioBlockMatcher {
-    fn is_match(&self, uev: &Uevent) -> bool {
-        uev.subsystem == BLOCK && uev.devpath.ends_with(&self.suffix) && !uev.devname.is_empty()
-    }
-}
-
-#[instrument]
-pub async fn get_virtio_mmio_device_name(
-    sandbox: &Arc<Mutex<Sandbox>>,
-    devpath: &str,
-) -> Result<()> {
-    let devname = devpath
-        .strip_prefix("/dev/")
-        .ok_or_else(|| anyhow!("Storage source '{}' must start with /dev/", devpath))?;
-
-    let matcher = MmioBlockMatcher::new(devname);
-    let uev = wait_for_uevent(sandbox, matcher)
-        .await
-        .context("failed to wait for uevent")?;
-    if uev.devname != devname {
-        return Err(anyhow!(
-            "Unexpected device name {} for mmio device (expected {})",
-            uev.devname,
-            devname
-        ));
-    }
-    Ok(())
-}
-
 /// Scan SCSI bus for the given SCSI address(SCSI-Id and LUN)
 #[instrument]
 fn scan_scsi_bus(scsi_addr: &str) -> Result<()> {
@@ -653,15 +571,13 @@ fn update_spec_devices(spec: &mut Spec, mut updates: HashMap<&str, DevUpdate>) -
 
     if let Some(resources) = linux.resources.as_mut() {
         for r in &mut resources.devices {
-            if let (Some(host_type), Some(host_major), Some(host_minor)) =
-                (r.r#type.as_ref(), r.major, r.minor)
-            {
-                if let Some(update) = res_updates.get(&(host_type.as_str(), host_major, host_minor))
+            if let (Some(host_major), Some(host_minor)) = (r.major, r.minor) {
+                if let Some(update) = res_updates.get(&(r.r#type.as_str(), host_major, host_minor))
                 {
                     info!(
                         sl!(),
                         "update_spec_devices() updating resource";
-                        "type" => &host_type,
+                        "type" => &r.r#type,
                         "host_major" => host_major,
                         "host_minor" => host_minor,
                         "guest_major" => update.guest_major,
@@ -720,18 +636,12 @@ pub fn update_env_pci(
 #[instrument]
 async fn virtiommio_blk_device_handler(
     device: &Device,
-    sandbox: &Arc<Mutex<Sandbox>>,
+    _sandbox: &Arc<Mutex<Sandbox>>,
 ) -> Result<SpecUpdate> {
     if device.vm_path.is_empty() {
         return Err(anyhow!("Invalid path for virtio mmio blk device"));
     }
 
-    if !Path::new(&device.vm_path).exists() {
-        get_virtio_mmio_device_name(sandbox, &device.vm_path.to_string())
-            .await
-            .context("failed to get mmio device name")?;
-    }
-
     Ok(DevNumUpdate::from_vm_path(&device.vm_path)?.into())
 }
 
@@ -789,7 +699,7 @@ async fn virtio_nvdimm_device_handler(
     Ok(DevNumUpdate::from_vm_path(&device.vm_path)?.into())
 }
 
-fn split_vfio_pci_option(opt: &str) -> Option<(&str, &str)> {
+fn split_vfio_option(opt: &str) -> Option<(&str, &str)> {
     let mut tokens = opt.split('=');
     let hostbdf = tokens.next()?;
     let path = tokens.next()?;
@@ -804,18 +714,14 @@ fn split_vfio_pci_option(opt: &str) -> Option<(&str, &str)> {
 // Each option should have the form "DDDD:BB:DD.F=<pcipath>"
 //     DDDD:BB:DD.F is the device's PCI address in the host
 //     <pcipath> is a PCI path to the device in the guest (see pci.rs)
-#[instrument]
-async fn vfio_pci_device_handler(
-    device: &Device,
-    sandbox: &Arc<Mutex<Sandbox>>,
-) -> Result<SpecUpdate> {
-    let vfio_in_guest = device.type_ != DRIVER_VFIO_PCI_GK_TYPE;
+async fn vfio_device_handler(device: &Device, sandbox: &Arc<Mutex<Sandbox>>) -> Result<SpecUpdate> {
+    let vfio_in_guest = device.field_type != DRIVER_VFIO_GK_TYPE;
     let mut pci_fixups = Vec::<(pci::Address, pci::Address)>::new();
     let mut group = None;
 
     for opt in device.options.iter() {
-        let (host, pcipath) = split_vfio_pci_option(opt)
-            .ok_or_else(|| anyhow!("Malformed VFIO PCI option {:?}", opt))?;
+        let (host, pcipath) =
+            split_vfio_option(opt).ok_or_else(|| anyhow!("Malformed VFIO option {:?}", opt))?;
         let host =
             pci::Address::from_str(host).context("Bad host PCI address in VFIO option {:?}")?;
         let pcipath = pci::Path::from_str(pcipath)?;
@@ -857,28 +763,6 @@ async fn vfio_pci_device_handler(
     })
 }
 
-// The VFIO AP (Adjunct Processor) device handler takes all the APQNs provided as device options
-// and awaits them. It sets the minimum AP rescan time of 5 seconds and temporarily adds that
-// amount to the hotplug timeout.
-#[cfg(target_arch = "s390x")]
-#[instrument]
-async fn vfio_ap_device_handler(
-    device: &Device,
-    sandbox: &Arc<Mutex<Sandbox>>,
-) -> Result<SpecUpdate> {
-    // Force AP bus rescan
-    fs::write(AP_SCANS_PATH, "1")?;
-    for apqn in device.options.iter() {
-        wait_for_ap_device(sandbox, ap::Address::from_str(apqn)?).await?;
-    }
-    Ok(Default::default())
-}
-
-#[cfg(not(target_arch = "s390x"))]
-async fn vfio_ap_device_handler(_: &Device, _: &Arc<Mutex<Sandbox>>) -> Result<SpecUpdate> {
-    Err(anyhow!("AP is only supported on s390x"))
-}
-
 #[instrument]
 pub async fn add_devices(
     devices: &[Device],
@@ -924,9 +808,9 @@ pub async fn add_devices(
 async fn add_device(device: &Device, sandbox: &Arc<Mutex<Sandbox>>) -> Result<SpecUpdate> {
     // log before validation to help with debugging gRPC protocol version differences.
     info!(sl!(), "device-id: {}, device-type: {}, device-vm-path: {}, device-container-path: {}, device-options: {:?}",
-          device.id, device.type_, device.vm_path, device.container_path, device.options);
+          device.id, device.field_type, device.vm_path, device.container_path, device.options);
 
-    if device.type_.is_empty() {
+    if device.field_type.is_empty() {
         return Err(anyhow!("invalid type for device {:?}", device));
     }
 
@@ -938,17 +822,14 @@ async fn add_device(device: &Device, sandbox: &Arc<Mutex<Sandbox>>) -> Result<Sp
         return Err(anyhow!("invalid container path for device {:?}", device));
     }
 
-    match device.type_.as_str() {
+    match device.field_type.as_str() {
         DRIVER_BLK_TYPE => virtio_blk_device_handler(device, sandbox).await,
         DRIVER_BLK_CCW_TYPE => virtio_blk_ccw_device_handler(device, sandbox).await,
         DRIVER_MMIO_BLK_TYPE => virtiommio_blk_device_handler(device, sandbox).await,
         DRIVER_NVDIMM_TYPE => virtio_nvdimm_device_handler(device, sandbox).await,
         DRIVER_SCSI_TYPE => virtio_scsi_device_handler(device, sandbox).await,
-        DRIVER_VFIO_PCI_GK_TYPE | DRIVER_VFIO_PCI_TYPE => {
-            vfio_pci_device_handler(device, sandbox).await
-        }
-        DRIVER_VFIO_AP_TYPE => vfio_ap_device_handler(device, sandbox).await,
-        _ => Err(anyhow!("Unknown device type {}", device.type_)),
+        DRIVER_VFIO_GK_TYPE | DRIVER_VFIO_TYPE => vfio_device_handler(device, sandbox).await,
+        _ => Err(anyhow!("Unknown device type {}", device.field_type)),
     }
 }
 
@@ -973,7 +854,7 @@ pub fn update_device_cgroup(spec: &mut Spec) -> Result<()> {
         allow: false,
         major: Some(major),
         minor: Some(minor),
-        r#type: Some(String::from("b")),
+        r#type: String::from("b"),
         access: String::from("rw"),
     });
 
@@ -1136,13 +1017,13 @@ mod tests {
                 resources: Some(LinuxResources {
                     devices: vec![
                         oci::LinuxDeviceCgroup {
-                            r#type: Some("c".to_string()),
+                            r#type: "c".to_string(),
                             major: Some(host_major_a),
                             minor: Some(host_minor_a),
                             ..oci::LinuxDeviceCgroup::default()
                         },
                         oci::LinuxDeviceCgroup {
-                            r#type: Some("c".to_string()),
+                            r#type: "c".to_string(),
                             major: Some(host_major_b),
                             minor: Some(host_minor_b),
                             ..oci::LinuxDeviceCgroup::default()
@@ -1235,13 +1116,13 @@ mod tests {
                 resources: Some(LinuxResources {
                     devices: vec![
                         LinuxDeviceCgroup {
-                            r#type: Some("c".to_string()),
+                            r#type: "c".to_string(),
                             major: Some(host_major),
                             minor: Some(host_minor),
                             ..LinuxDeviceCgroup::default()
                         },
                         LinuxDeviceCgroup {
-                            r#type: Some("b".to_string()),
+                            r#type: "b".to_string(),
                             major: Some(host_major),
                             minor: Some(host_minor),
                             ..LinuxDeviceCgroup::default()
@@ -1444,7 +1325,7 @@ mod tests {
 
         let mut uev = crate::uevent::Uevent::default();
         uev.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
-        uev.subsystem = BLOCK.to_string();
+        uev.subsystem = "block".to_string();
         uev.devpath = devpath.clone();
         uev.devname = devname.to_string();
 
@@ -1478,7 +1359,7 @@ mod tests {
         let mut uev_a = crate::uevent::Uevent::default();
         let relpath_a = "/0000:00:0a.0";
         uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
-        uev_a.subsystem = BLOCK.to_string();
+        uev_a.subsystem = "block".to_string();
         uev_a.devname = devname.to_string();
         uev_a.devpath = format!("{}{}/virtio4/block/{}", root_bus, relpath_a, devname);
         let matcher_a = VirtioBlkPciMatcher::new(relpath_a);
@@ -1497,7 +1378,7 @@ mod tests {
     #[cfg(target_arch = "s390x")]
     #[tokio::test]
     async fn test_virtio_blk_ccw_matcher() {
-        let root_bus = CCW_ROOT_BUS_PATH;
+        let root_bus = create_ccw_root_bus_path();
         let subsystem = "block";
         let devname = "vda";
         let relpath = "0.0.0002";
@@ -1562,7 +1443,7 @@ mod tests {
         let mut uev_a = crate::uevent::Uevent::default();
         let addr_a = "0:0";
         uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
-        uev_a.subsystem = BLOCK.to_string();
+        uev_a.subsystem = "block".to_string();
         uev_a.devname = devname.to_string();
         uev_a.devpath = format!(
             "{}/0000:00:00.0/virtio0/host0/target0:0:0/0:0:{}/block/sda",
@@ -1605,41 +1486,14 @@ mod tests {
         assert!(!matcher_a.is_match(&uev_b));
     }
 
-    #[tokio::test]
-    async fn test_mmio_block_matcher() {
-        let devname_a = "vda";
-        let devname_b = "vdb";
-        let mut uev_a = crate::uevent::Uevent::default();
-        uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
-        uev_a.subsystem = BLOCK.to_string();
-        uev_a.devname = devname_a.to_string();
-        uev_a.devpath = format!(
-            "/sys/devices/virtio-mmio-cmdline/virtio-mmio.0/virtio0/block/{}",
-            devname_a
-        );
-        let matcher_a = MmioBlockMatcher::new(devname_a);
-
-        let mut uev_b = uev_a.clone();
-        uev_b.devpath = format!(
-            "/sys/devices/virtio-mmio-cmdline/virtio-mmio.4/virtio4/block/{}",
-            devname_b
-        );
-        let matcher_b = MmioBlockMatcher::new(devname_b);
-
-        assert!(matcher_a.is_match(&uev_a));
-        assert!(matcher_b.is_match(&uev_b));
-        assert!(!matcher_b.is_match(&uev_a));
-        assert!(!matcher_a.is_match(&uev_b));
-    }
-
     #[test]
-    fn test_split_vfio_pci_option() {
+    fn test_split_vfio_option() {
         assert_eq!(
-            split_vfio_pci_option("0000:01:00.0=02/01"),
+            split_vfio_option("0000:01:00.0=02/01"),
             Some(("0000:01:00.0", "02/01"))
         );
-        assert_eq!(split_vfio_pci_option("0000:01:00.0=02/01=rubbish"), None);
-        assert_eq!(split_vfio_pci_option("0000:01:00.0"), None);
+        assert_eq!(split_vfio_option("0000:01:00.0=02/01=rubbish"), None);
+        assert_eq!(split_vfio_option("0000:01:00.0"), None);
     }
 
     #[test]
@@ -1718,35 +1572,4 @@ mod tests {
         // Test dev2
         assert!(pci_iommu_group(&syspci, dev2).is_err());
     }
-
-    #[cfg(target_arch = "s390x")]
-    #[tokio::test]
-    async fn test_vfio_ap_matcher() {
-        let subsystem = "ap";
-        let card = "0a";
-        let relpath = format!("{}.0001", card);
-
-        let mut uev = Uevent::default();
-        uev.action = U_EVENT_ACTION_ADD.to_string();
-        uev.subsystem = subsystem.to_string();
-        uev.devpath = format!("{}/card{}/{}", AP_ROOT_BUS_PATH, card, relpath);
-
-        let ap_address = ap::Address::from_str(&relpath).unwrap();
-        let matcher = ApMatcher::new(ap_address);
-
-        assert!(matcher.is_match(&uev));
-
-        let mut uev_remove = uev.clone();
-        uev_remove.action = U_EVENT_ACTION_REMOVE.to_string();
-        assert!(!matcher.is_match(&uev_remove));
-
-        let mut uev_other_device = uev.clone();
-        uev_other_device.devpath = format!(
-            "{}/card{}/{}",
-            AP_ROOT_BUS_PATH,
-            card,
-            format!("{}.0002", card)
-        );
-        assert!(!matcher.is_match(&uev_other_device));
-    }
 }

src/agent/src/image_rpc.rs

@@ -1,352 +0,0 @@
-// Copyright (c) 2021 Alibaba Cloud
-// Copyright (c) 2021, 2023 IBM Corporation
-// Copyright (c) 2022 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-use std::env;
-use std::fs;
-use std::path::Path;
-use std::process::Command;
-use std::sync::atomic::{AtomicBool, AtomicU16, Ordering};
-use std::sync::Arc;
-
-use anyhow::{anyhow, Result};
-use async_trait::async_trait;
-use protocols::image;
-use tokio::sync::Mutex;
-use ttrpc::{self, error::get_rpc_status as ttrpc_error};
-
-use crate::rpc::{verify_cid, CONTAINER_BASE};
-use crate::sandbox::Sandbox;
-use crate::AGENT_CONFIG;
-
-use image_rs::image::ImageClient;
-use std::io::Write;
-
-const AA_PATH: &str = "/usr/local/bin/attestation-agent";
-
-const AA_KEYPROVIDER_URI: &str =
-    "unix:///run/confidential-containers/attestation-agent/keyprovider.sock";
-const AA_GETRESOURCE_URI: &str =
-    "unix:///run/confidential-containers/attestation-agent/getresource.sock";
-
-const OCICRYPT_CONFIG_PATH: &str = "/tmp/ocicrypt_config.json";
-// kata rootfs is readonly, use tmpfs before CC storage is implemented.
-const KATA_CC_IMAGE_WORK_DIR: &str = "/run/image/";
-const KATA_CC_PAUSE_BUNDLE: &str = "/pause_bundle";
-const CONFIG_JSON: &str = "config.json";
-
-// Convenience macro to obtain the scope logger
-macro_rules! sl {
-    () => {
-        slog_scope::logger()
-    };
-}
-
-pub struct ImageService {
-    sandbox: Arc<Mutex<Sandbox>>,
-    attestation_agent_started: AtomicBool,
-    image_client: Arc<Mutex<ImageClient>>,
-    container_count: Arc<AtomicU16>,
-}
-
-impl ImageService {
-    pub fn new(sandbox: Arc<Mutex<Sandbox>>) -> Self {
-        env::set_var("CC_IMAGE_WORK_DIR", KATA_CC_IMAGE_WORK_DIR);
-        Self {
-            sandbox,
-            attestation_agent_started: AtomicBool::new(false),
-            image_client: Arc::new(Mutex::new(ImageClient::default())),
-            container_count: Arc::new(AtomicU16::new(0)),
-        }
-    }
-
-    // pause image is packaged in rootfs for CC
-    fn unpack_pause_image(cid: &str) -> Result<()> {
-        let cc_pause_bundle = Path::new(KATA_CC_PAUSE_BUNDLE);
-        if !cc_pause_bundle.exists() {
-            return Err(anyhow!("Pause image not present in rootfs"));
-        }
-
-        info!(sl!(), "use guest pause image cid {:?}", cid);
-        let pause_bundle = Path::new(CONTAINER_BASE).join(cid);
-        let pause_rootfs = pause_bundle.join("rootfs");
-        let pause_config = pause_bundle.join(CONFIG_JSON);
-        let pause_binary = pause_rootfs.join("pause");
-        fs::create_dir_all(&pause_rootfs)?;
-        if !pause_config.exists() {
-            fs::copy(
-                cc_pause_bundle.join(CONFIG_JSON),
-                pause_bundle.join(CONFIG_JSON),
-            )?;
-        }
-        if !pause_binary.exists() {
-            fs::copy(cc_pause_bundle.join("rootfs").join("pause"), pause_binary)?;
-        }
-
-        Ok(())
-    }
-
-    // If we fail to start the AA, ocicrypt won't be able to unwrap keys
-    // and container decryption will fail.
-    fn init_attestation_agent() -> Result<()> {
-        let config_path = OCICRYPT_CONFIG_PATH;
-
-        // The image will need to be encrypted using a keyprovider
-        // that has the same name (at least according to the config).
-        let ocicrypt_config = serde_json::json!({
-            "key-providers": {
-                "attestation-agent":{
-                    "ttrpc":AA_KEYPROVIDER_URI
-                }
-            }
-        });
-
-        let mut config_file = fs::File::create(config_path)?;
-        config_file.write_all(ocicrypt_config.to_string().as_bytes())?;
-
-        // The Attestation Agent will run for the duration of the guest.
-        Command::new(AA_PATH)
-            .arg("--keyprovider_sock")
-            .arg(AA_KEYPROVIDER_URI)
-            .arg("--getresource_sock")
-            .arg(AA_GETRESOURCE_URI)
-            .spawn()?;
-        Ok(())
-    }
-
-    /// Determines the container id (cid) to use for a given request.
-    ///
-    /// If the request specifies a non-empty id, use it; otherwise derive it from the image path.
-    /// In either case, verify that the chosen id is valid.
-    fn cid_from_request(&self, req: &image::PullImageRequest) -> Result<String> {
-        let req_cid = req.container_id();
-        let cid = if !req_cid.is_empty() {
-            req_cid.to_string()
-        } else if let Some(last) = req.image().rsplit('/').next() {
-            // Support multiple containers with same image
-            let index = self.container_count.fetch_add(1, Ordering::Relaxed);
-
-            // ':' not valid for container id
-            format!("{}_{}", last.replace(':', "_"), index)
-        } else {
-            return Err(anyhow!("Invalid image name. {}", req.image()));
-        };
-        verify_cid(&cid)?;
-        Ok(cid)
-    }
-
-    async fn pull_image(&self, req: &image::PullImageRequest) -> Result<String> {
-        env::set_var("OCICRYPT_KEYPROVIDER_CONFIG", OCICRYPT_CONFIG_PATH);
-
-        let https_proxy = &AGENT_CONFIG.read().await.https_proxy;
-        if !https_proxy.is_empty() {
-            env::set_var("HTTPS_PROXY", https_proxy);
-        }
-
-        let no_proxy = &AGENT_CONFIG.read().await.no_proxy;
-        if !no_proxy.is_empty() {
-            env::set_var("NO_PROXY", no_proxy);
-        }
-
-        let cid = self.cid_from_request(req)?;
-        let image = req.image();
-        if cid.starts_with("pause") {
-            Self::unpack_pause_image(&cid)?;
-
-            let mut sandbox = self.sandbox.lock().await;
-            sandbox.images.insert(String::from(image), cid);
-            return Ok(image.to_owned());
-        }
-
-        let aa_kbc_params = &AGENT_CONFIG.read().await.aa_kbc_params;
-        if !aa_kbc_params.is_empty() {
-            match self.attestation_agent_started.compare_exchange_weak(
-                false,
-                true,
-                Ordering::SeqCst,
-                Ordering::SeqCst,
-            ) {
-                Ok(_) => Self::init_attestation_agent()?,
-                Err(_) => info!(sl!(), "Attestation Agent already running"),
-            }
-        }
-        // If the attestation-agent is being used, then enable the authenticated credentials support
-        info!(
-            sl!(),
-            "image_client.config.auth set to: {}",
-            !aa_kbc_params.is_empty()
-        );
-        self.image_client.lock().await.config.auth = !aa_kbc_params.is_empty();
-
-        // Read enable signature verification from the agent config and set it in the image_client
-        let enable_signature_verification =
-            &AGENT_CONFIG.read().await.enable_signature_verification;
-        info!(
-            sl!(),
-            "enable_signature_verification set to: {}", enable_signature_verification
-        );
-        self.image_client.lock().await.config.security_validate = *enable_signature_verification;
-
-        let source_creds = (!req.source_creds().is_empty()).then(|| req.source_creds());
-
-        let bundle_path = Path::new(CONTAINER_BASE).join(&cid);
-        fs::create_dir_all(&bundle_path)?;
-
-        let decrypt_config = format!("provider:attestation-agent:{}", aa_kbc_params);
-
-        info!(sl!(), "pull image {:?}, bundle path {:?}", cid, bundle_path);
-        // Image layers will store at KATA_CC_IMAGE_WORK_DIR, generated bundles
-        // with rootfs and config.json will store under CONTAINER_BASE/cid.
-        let res = self
-            .image_client
-            .lock()
-            .await
-            .pull_image(image, &bundle_path, &source_creds, &Some(&decrypt_config))
-            .await;
-
-        match res {
-            Ok(image) => {
-                info!(
-                    sl!(),
-                    "pull and unpack image {:?}, cid: {:?}, with image-rs succeed. ", image, cid
-                );
-            }
-            Err(e) => {
-                error!(
-                    sl!(),
-                    "pull and unpack image {:?}, cid: {:?}, with image-rs failed with {:?}. ",
-                    image,
-                    cid,
-                    e.to_string()
-                );
-                return Err(e);
-            }
-        };
-
-        let mut sandbox = self.sandbox.lock().await;
-        sandbox.images.insert(String::from(image), cid);
-        Ok(image.to_owned())
-    }
-}
-
-#[async_trait]
-impl protocols::image_ttrpc_async::Image for ImageService {
-    async fn pull_image(
-        &self,
-        _ctx: &ttrpc::r#async::TtrpcContext,
-        req: image::PullImageRequest,
-    ) -> ttrpc::Result<image::PullImageResponse> {
-        match self.pull_image(&req).await {
-            Ok(r) => {
-                let mut resp = image::PullImageResponse::new();
-                resp.image_ref = r;
-                return Ok(resp);
-            }
-            Err(e) => {
-                return Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()));
-            }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::ImageService;
-    use crate::sandbox::Sandbox;
-    use protocols::image;
-    use std::sync::Arc;
-    use tokio::sync::Mutex;
-
-    #[tokio::test]
-    async fn test_cid_from_request() {
-        struct Case {
-            cid: &'static str,
-            image: &'static str,
-            result: Option<&'static str>,
-        }
-
-        let cases = [
-            Case {
-                cid: "",
-                image: "",
-                result: None,
-            },
-            Case {
-                cid: "..",
-                image: "",
-                result: None,
-            },
-            Case {
-                cid: "",
-                image: "..",
-                result: None,
-            },
-            Case {
-                cid: "",
-                image: "abc/..",
-                result: None,
-            },
-            Case {
-                cid: "",
-                image: "abc/",
-                result: None,
-            },
-            Case {
-                cid: "",
-                image: "../abc",
-                result: Some("abc_4"),
-            },
-            Case {
-                cid: "",
-                image: "../9abc",
-                result: Some("9abc_5"),
-            },
-            Case {
-                cid: "some-string.1_2",
-                image: "",
-                result: Some("some-string.1_2"),
-            },
-            Case {
-                cid: "0some-string.1_2",
-                image: "",
-                result: Some("0some-string.1_2"),
-            },
-            Case {
-                cid: "a:b",
-                image: "",
-                result: None,
-            },
-            Case {
-                cid: "",
-                image: "prefix/a:b",
-                result: Some("a_b_6"),
-            },
-            Case {
-                cid: "",
-                image: "/a/b/c/d:e",
-                result: Some("d_e_7"),
-            },
-        ];
-
-        let logger = slog::Logger::root(slog::Discard, o!());
-        let s = Sandbox::new(&logger).unwrap();
-        let image_service = ImageService::new(Arc::new(Mutex::new(s)));
-        for case in &cases {
-            let mut req = image::PullImageRequest::new();
-            req.set_image(case.image.to_string());
-            req.set_container_id(case.cid.to_string());
-            let ret = image_service.cid_from_request(&req);
-            match (case.result, ret) {
-                (Some(expected), Ok(actual)) => assert_eq!(expected, actual),
-                (None, Err(_)) => (),
-                (None, Ok(r)) => panic!("Expected an error, got {}", r),
-                (Some(expected), Err(e)) => {
-                    panic!("Expected {} but got an error ({})", expected, e)
-                }
-            }
-        }
-    }
-}

src/agent/src/linux_abi.rs

@@ -3,8 +3,6 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use cfg_if::cfg_if;
-
 /// Linux ABI related constants.
 
 #[cfg(target_arch = "aarch64")]
@@ -66,14 +64,10 @@ pub fn create_pci_root_bus_path() -> String {
     ret
 }
 
-cfg_if! {
-    if #[cfg(target_arch = "s390x")] {
-        pub const CCW_ROOT_BUS_PATH: &str = "/devices/css0";
-        pub const AP_ROOT_BUS_PATH: &str = "/devices/ap";
-        pub const AP_SCANS_PATH: &str = "/sys/bus/ap/scans";
-    }
+#[cfg(target_arch = "s390x")]
+pub fn create_ccw_root_bus_path() -> String {
+    String::from("/devices/css0")
 }
-
 // From https://www.kernel.org/doc/Documentation/acpi/namespace.txt
 // The Linux kernel's core ACPI subsystem creates struct acpi_device
 // objects for ACPI namespace objects representing devices, power resources

src/agent/src/main.rs

@@ -20,7 +20,6 @@ extern crate scopeguard;
 extern crate slog;
 
 use anyhow::{anyhow, Context, Result};
-use cfg_if::cfg_if;
 use clap::{AppSettings, Parser};
 use nix::fcntl::OFlag;
 use nix::sys::socket::{self, AddressFamily, SockFlag, SockType, VsockAddr};
@@ -35,6 +34,8 @@ use std::process::exit;
 use std::sync::Arc;
 use tracing::{instrument, span};
 
+#[cfg(target_arch = "s390x")]
+mod ccw;
 mod config;
 mod console;
 mod device;
@@ -70,17 +71,9 @@ use tokio::{
     task::JoinHandle,
 };
 
-mod image_rpc;
 mod rpc;
 mod tracer;
 
-cfg_if! {
-    if #[cfg(target_arch = "s390x")] {
-        mod ap;
-        mod ccw;
-    }
-}
-
 const NAME: &str = "kata-agent";
 
 lazy_static! {

src/agent/src/mount.rs

@@ -21,11 +21,10 @@ use nix::unistd::{Gid, Uid};
 use regex::Regex;
 
 use crate::device::{
-    get_scsi_device_name, get_virtio_blk_pci_device_name, get_virtio_mmio_device_name,
-    online_device, wait_for_pmem_device, DRIVER_9P_TYPE, DRIVER_BLK_CCW_TYPE, DRIVER_BLK_TYPE,
-    DRIVER_EPHEMERAL_TYPE, DRIVER_LOCAL_TYPE, DRIVER_MMIO_BLK_TYPE, DRIVER_NVDIMM_TYPE,
-    DRIVER_OVERLAYFS_TYPE, DRIVER_SCSI_TYPE, DRIVER_VIRTIOFS_TYPE, DRIVER_WATCHABLE_BIND_TYPE,
-    FS_TYPE_HUGETLB,
+    get_scsi_device_name, get_virtio_blk_pci_device_name, online_device, wait_for_pmem_device,
+    DRIVER_9P_TYPE, DRIVER_BLK_CCW_TYPE, DRIVER_BLK_TYPE, DRIVER_EPHEMERAL_TYPE, DRIVER_LOCAL_TYPE,
+    DRIVER_MMIO_BLK_TYPE, DRIVER_NVDIMM_TYPE, DRIVER_OVERLAYFS_TYPE, DRIVER_SCSI_TYPE,
+    DRIVER_VIRTIOFS_TYPE, DRIVER_WATCHABLE_BIND_TYPE, FS_TYPE_HUGETLB,
 };
 use crate::linux_abi::*;
 use crate::pci;
@@ -212,10 +211,10 @@ async fn ephemeral_storage_handler(
     // By now we only support one option field: "fsGroup" which
     // isn't an valid mount option, thus we should remove it when
     // do mount.
-    if !storage.options.is_empty() {
+    if storage.options.len() > 0 {
         // ephemeral_storage didn't support mount options except fsGroup.
         let mut new_storage = storage.clone();
-        new_storage.options = Default::default();
+        new_storage.options = protobuf::RepeatedField::default();
         common_storage_handler(logger, &new_storage)?;
 
         let opts_vec: Vec<String> = storage.options.to_vec();
@@ -241,70 +240,6 @@ async fn ephemeral_storage_handler(
     Ok("".to_string())
 }
 
-// update_ephemeral_mounts takes a list of ephemeral mounts and remounts them
-// with mount options passed by the caller
-#[instrument]
-pub async fn update_ephemeral_mounts(
-    logger: Logger,
-    storages: Vec<Storage>,
-    sandbox: Arc<Mutex<Sandbox>>,
-) -> Result<()> {
-    for (_, storage) in storages.iter().enumerate() {
-        let handler_name = storage.driver.clone();
-        let logger = logger.new(o!(
-	    "msg" => "updating tmpfs storage",
-            "subsystem" => "storage",
-            "storage-type" => handler_name.to_owned()));
-
-        match handler_name.as_str() {
-            DRIVER_EPHEMERAL_TYPE => {
-                fs::create_dir_all(Path::new(&storage.mount_point))?;
-
-                if storage.options.is_empty() {
-                    continue;
-                } else {
-                    // assume that fsGid has already been set
-                    let mut opts = Vec::<&str>::new();
-                    for (_, opt) in storage.options.iter().enumerate() {
-                        if opt.starts_with(FS_GID) {
-                            continue;
-                        }
-                        opts.push(opt)
-                    }
-                    let mount_path = Path::new(&storage.mount_point);
-                    let src_path = Path::new(&storage.source);
-
-                    let (flags, options) = parse_mount_flags_and_options(opts);
-
-                    info!(logger, "mounting storage";
-                    "mount-source" => src_path.display(),
-                    "mount-destination" => mount_path.display(),
-                    "mount-fstype"  => storage.fstype.as_str(),
-                    "mount-options" => options.as_str(),
-                    );
-
-                    return baremount(
-                        src_path,
-                        mount_path,
-                        storage.fstype.as_str(),
-                        flags,
-                        options.as_str(),
-                        &logger,
-                    );
-                }
-            }
-            _ => {
-                return Err(anyhow!(
-                    "Unsupported storage type for syncing mounts {}. Only ephemeral storage update is supported",
-                    storage.driver.to_owned()
-                ));
-            }
-        };
-    }
-
-    Ok(())
-}
-
 #[instrument]
 async fn overlayfs_storage_handler(
     logger: &Logger,
@@ -474,14 +409,8 @@ async fn virtiommio_blk_storage_handler(
     storage: &Storage,
     sandbox: Arc<Mutex<Sandbox>>,
 ) -> Result<String> {
-    let storage = storage.clone();
-    if !Path::new(&storage.source).exists() {
-        get_virtio_mmio_device_name(&sandbox, &storage.source)
-            .await
-            .context("failed to get mmio device name")?;
-    }
     //The source path is VmPath
-    common_storage_handler(logger, &storage)
+    common_storage_handler(logger, storage)
 }
 
 // virtiofs_storage_handler handles the storage for virtio-fs.
@@ -661,7 +590,7 @@ pub fn set_ownership(logger: &Logger, storage: &Storage) -> Result<()> {
     if storage.fs_group.is_none() {
         return Ok(());
     }
-    let fs_group = storage.fs_group();
+    let fs_group = storage.get_fs_group();
 
     let mut read_only = false;
     let opts_vec: Vec<String> = storage.options.to_vec();
@@ -678,7 +607,7 @@ pub fn set_ownership(logger: &Logger, storage: &Storage) -> Result<()> {
         err
     })?;
 
-    if fs_group.group_change_policy == FSGroupChangePolicy::OnRootMismatch.into()
+    if fs_group.group_change_policy == FSGroupChangePolicy::OnRootMismatch
         && metadata.gid() == fs_group.group_id
     {
         let mut mask = if read_only { RO_MASK } else { RW_MASK };
@@ -1101,6 +1030,7 @@ fn parse_options(option_list: Vec<String>) -> HashMap<String, String> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use protobuf::RepeatedField;
     use protocols::agent::FSGroup;
     use std::fs::File;
     use std::fs::OpenOptions;
@@ -2021,8 +1951,9 @@ mod tests {
                 mount_path: "rw_mount",
                 fs_group: Some(FSGroup {
                     group_id: 3000,
-                    group_change_policy: FSGroupChangePolicy::Always.into(),
-                    ..Default::default()
+                    group_change_policy: FSGroupChangePolicy::Always,
+                    unknown_fields: Default::default(),
+                    cached_size: Default::default(),
                 }),
                 read_only: false,
                 expected_group_id: 3000,
@@ -2032,8 +1963,9 @@ mod tests {
                 mount_path: "ro_mount",
                 fs_group: Some(FSGroup {
                     group_id: 3000,
-                    group_change_policy: FSGroupChangePolicy::OnRootMismatch.into(),
-                    ..Default::default()
+                    group_change_policy: FSGroupChangePolicy::OnRootMismatch,
+                    unknown_fields: Default::default(),
+                    cached_size: Default::default(),
                 }),
                 read_only: true,
                 expected_group_id: 3000,
@@ -2053,7 +1985,10 @@ mod tests {
             let directory_mode = mount_dir.as_path().metadata().unwrap().permissions().mode();
             let mut storage_data = Storage::new();
             if d.read_only {
-                storage_data.set_options(vec!["foo".to_string(), "ro".to_string()]);
+                storage_data.set_options(RepeatedField::from_slice(&[
+                    "foo".to_string(),
+                    "ro".to_string(),
+                ]));
             }
             if let Some(fs_group) = d.fs_group.clone() {
                 storage_data.set_fs_group(fs_group);

src/agent/src/netlink.rs

@@ -7,6 +7,7 @@ use anyhow::{anyhow, Context, Result};
 use futures::{future, StreamExt, TryStreamExt};
 use ipnetwork::{IpNetwork, Ipv4Network, Ipv6Network};
 use nix::errno::Errno;
+use protobuf::RepeatedField;
 use protocols::types::{ARPNeighbor, IPAddress, IPFamily, Interface, Route};
 use rtnetlink::{new_connection, packet, IpVersion};
 use std::convert::{TryFrom, TryInto};
@@ -82,8 +83,8 @@ impl Handle {
 
         // Add new ip addresses from request
         for ip_address in &iface.IPAddresses {
-            let ip = IpAddr::from_str(ip_address.address())?;
-            let mask = ip_address.mask().parse::<u8>()?;
+            let ip = IpAddr::from_str(ip_address.get_address())?;
+            let mask = ip_address.get_mask().parse::<u8>()?;
 
             self.add_addresses(link.index(), std::iter::once(IpNetwork::new(ip, mask)?))
                 .await?;
@@ -151,7 +152,7 @@ impl Handle {
                 .map(|p| p.try_into())
                 .collect::<Result<Vec<IPAddress>>>()?;
 
-            iface.IPAddresses = ips;
+            iface.IPAddresses = RepeatedField::from_vec(ips);
 
             list.push(iface);
         }
@@ -333,7 +334,7 @@ impl Handle {
 
             // `rtnetlink` offers a separate request builders for different IP versions (IP v4 and v6).
             // This if branch is a bit clumsy because it does almost the same.
-            if route.family() == IPFamily::v6 {
+            if route.get_family() == IPFamily::v6 {
                 let dest_addr = if !route.dest.is_empty() {
                     Ipv6Network::from_str(&route.dest)?
                 } else {
@@ -367,9 +368,9 @@ impl Handle {
                     if Errno::from_i32(message.code.abs()) != Errno::EEXIST {
                         return Err(anyhow!(
                             "Failed to add IP v6 route (src: {}, dst: {}, gtw: {},Err: {})",
-                            route.source(),
-                            route.dest(),
-                            route.gateway(),
+                            route.get_source(),
+                            route.get_dest(),
+                            route.get_gateway(),
                             message
                         ));
                     }
@@ -408,9 +409,9 @@ impl Handle {
                     if Errno::from_i32(message.code.abs()) != Errno::EEXIST {
                         return Err(anyhow!(
                             "Failed to add IP v4 route (src: {}, dst: {}, gtw: {},Err: {})",
-                            route.source(),
-                            route.dest(),
-                            route.gateway(),
+                            route.get_source(),
+                            route.get_dest(),
+                            route.get_gateway(),
                             message
                         ));
                     }
@@ -505,7 +506,7 @@ impl Handle {
             self.add_arp_neighbor(&neigh).await.map_err(|err| {
                 anyhow!(
                     "Failed to add ARP neighbor {}: {:?}",
-                    neigh.toIPAddress().address(),
+                    neigh.get_toIPAddress().get_address(),
                     err
                 )
             })?;
@@ -724,7 +725,7 @@ impl TryFrom<Address> for IPAddress {
         let mask = format!("{}", value.0.header.prefix_len);
 
         Ok(IPAddress {
-            family: family.into(),
+            family,
             address,
             mask,
             ..Default::default()

src/agent/src/network.rs

@@ -7,7 +7,6 @@ use anyhow::{anyhow, Result};
 use nix::mount::{self, MsFlags};
 use slog::Logger;
 use std::fs;
-use std::path;
 
 const KATA_GUEST_SANDBOX_DNS_FILE: &str = "/run/kata-containers/sandbox/resolv.conf";
 const GUEST_DNS_FILE: &str = "/etc/resolv.conf";
@@ -65,12 +64,6 @@ fn do_setup_guest_dns(logger: Logger, dns_list: Vec<String>, src: &str, dst: &st
         .map(|x| x.trim())
         .collect::<Vec<&str>>()
         .join("\n");
-
-    // make sure the src file's parent path exist.
-    let file_path = path::Path::new(src);
-    if let Some(p) = file_path.parent() {
-        fs::create_dir_all(p)?;
-    }
     fs::write(src, content)?;
 
     // bind mount to /etc/resolv.conf

src/agent/src/rpc.rs

@@ -21,29 +21,22 @@ use ttrpc::{
 use anyhow::{anyhow, Context, Result};
 use cgroups::freezer::FreezerState;
 use oci::{LinuxNamespace, Root, Spec};
-use protobuf::{MessageDyn, MessageField};
+use protobuf::{Message, RepeatedField, SingularPtrField};
 use protocols::agent::{
     AddSwapRequest, AgentDetails, CopyFileRequest, GetIPTablesRequest, GetIPTablesResponse,
     GuestDetailsResponse, Interfaces, Metrics, OOMEvent, ReadStreamResponse, Routes,
     SetIPTablesRequest, SetIPTablesResponse, StatsContainerResponse, VolumeStatsRequest,
     WaitProcessResponse, WriteStreamResponse,
 };
-use protocols::csi::{
-    volume_usage::Unit as VolumeUsage_Unit, VolumeCondition, VolumeStatsResponse, VolumeUsage,
-};
+use protocols::csi::{VolumeCondition, VolumeStatsResponse, VolumeUsage, VolumeUsage_Unit};
 use protocols::empty::Empty;
 use protocols::health::{
-    health_check_response::ServingStatus as HealthCheckResponse_ServingStatus, HealthCheckResponse,
-    VersionCheckResponse,
+    HealthCheckResponse, HealthCheckResponse_ServingStatus, VersionCheckResponse,
 };
 use protocols::types::Interface;
-use protocols::{
-    agent_ttrpc_async as agent_ttrpc, health_ttrpc_async as health_ttrpc,
-    image_ttrpc_async as image_ttrpc,
-};
+use protocols::{agent_ttrpc_async as agent_ttrpc, health_ttrpc_async as health_ttrpc};
 use rustjail::cgroups::notifier;
 use rustjail::container::{BaseContainer, Container, LinuxContainer, SYSTEMD_CGROUP_PATH_FORMAT};
-use rustjail::mount::parse_mount_table;
 use rustjail::process::Process;
 use rustjail::specconv::CreateOpts;
 
@@ -56,10 +49,9 @@ use rustjail::process::ProcessOperations;
 use crate::device::{
     add_devices, get_virtio_blk_pci_device_name, update_device_cgroup, update_env_pci,
 };
-use crate::image_rpc;
 use crate::linux_abi::*;
 use crate::metrics::get_metrics;
-use crate::mount::{add_storages, baremount, update_ephemeral_mounts, STORAGE_HANDLER_LIST};
+use crate::mount::{add_storages, baremount, STORAGE_HANDLER_LIST};
 use crate::namespace::{NSTYPEIPC, NSTYPEPID, NSTYPEUTS};
 use crate::network::setup_guest_dns;
 use crate::pci;
@@ -88,12 +80,8 @@ use std::io::{BufRead, BufReader, Write};
 use std::os::unix::fs::FileExt;
 use std::path::PathBuf;
 
-pub const CONTAINER_BASE: &str = "/run/kata-containers";
+const CONTAINER_BASE: &str = "/run/kata-containers";
 const MODPROBE_PATH: &str = "/sbin/modprobe";
-const ANNO_K8S_IMAGE_NAME: &str = "io.kubernetes.cri.image-name";
-const CONFIG_JSON: &str = "config.json";
-const INIT_TRUSTED_STORAGE: &str = "/usr/bin/kata-init-trusted-storage";
-const TRUSTED_STORAGE_DEVICE: &str = "/dev/trusted_store";
 
 /// the iptables seriers binaries could appear either in /sbin
 /// or /usr/sbin, we need to check both of them
@@ -105,7 +93,6 @@ const USR_IP6TABLES_SAVE: &str = "/usr/sbin/ip6tables-save";
 const IP6TABLES_SAVE: &str = "/sbin/ip6tables-save";
 const USR_IP6TABLES_RESTORE: &str = "/usr/sbin/ip6tables-save";
 const IP6TABLES_RESTORE: &str = "/sbin/ip6tables-restore";
-const KATA_GUEST_SHARE_DIR: &str = "/run/kata-containers/shared/containers/";
 
 const ERR_CANNOT_GET_WRITER: &str = "Cannot get writer";
 const ERR_INVALID_BLOCK_SIZE: &str = "Invalid block size";
@@ -137,11 +124,11 @@ macro_rules! is_allowed {
         if !AGENT_CONFIG
             .read()
             .await
-            .is_allowed_endpoint($req.descriptor_dyn().name())
+            .is_allowed_endpoint($req.descriptor().name())
         {
             return Err(ttrpc_error!(
                 ttrpc::Code::UNIMPLEMENTED,
-                format!("{} is blocked", $req.descriptor_dyn().name()),
+                format!("{} is blocked", $req.descriptor().name()),
             ));
         }
     };
@@ -153,41 +140,6 @@ pub struct AgentService {
     init_mode: bool,
 }
 
-// A container ID must match this regex:
-//
-//     ^[a-zA-Z0-9][a-zA-Z0-9_.-]+$
-//
-pub fn verify_cid(id: &str) -> Result<()> {
-    let mut chars = id.chars();
-
-    let valid = matches!(chars.next(), Some(first) if first.is_alphanumeric()
-                && id.len() > 1
-                && chars.all(|c| c.is_alphanumeric() || ['.', '-', '_'].contains(&c)));
-
-    match valid {
-        true => Ok(()),
-        false => Err(anyhow!("invalid container ID: {:?}", id)),
-    }
-}
-
-// Partially merge an OCI process specification into another one.
-fn merge_oci_process(target: &mut oci::Process, source: &oci::Process) {
-    if target.args.is_empty() && !source.args.is_empty() {
-        target.args.append(&mut source.args.clone());
-    }
-
-    if target.cwd == "/" && source.cwd != "/" {
-        target.cwd = String::from(&source.cwd);
-    }
-
-    for source_env in &source.env {
-        let variable_name: Vec<&str> = source_env.split('=').collect();
-        if !target.env.iter().any(|i| i.contains(variable_name[0])) {
-            target.env.push(source_env.to_string());
-        }
-    }
-}
-
 impl AgentService {
     #[instrument]
     async fn do_create_container(
@@ -199,7 +151,7 @@ impl AgentService {
         kata_sys_util::validate::verify_id(&cid)?;
 
         let mut oci_spec = req.OCI.clone();
-        let use_sandbox_pidns = req.sandbox_pidns();
+        let use_sandbox_pidns = req.get_sandbox_pidns();
 
         let sandbox;
         let mut s;
@@ -218,9 +170,6 @@ impl AgentService {
             "receive createcontainer, storages: {:?}", &req.storages
         );
 
-        // Merge the image bundle OCI spec into the container creation request OCI spec.
-        self.merge_bundle_oci(&mut oci).await?;
-
         // Some devices need some extra processing (the ones invoked with
         // --device for instance), and that's what this call is doing. It
         // updates the devices listed in the OCI spec, so that they actually
@@ -228,30 +177,6 @@ impl AgentService {
         // cannot predict everything from the caller.
         add_devices(&req.devices.to_vec(), &mut oci, &self.sandbox).await?;
 
-        let linux = oci
-            .linux
-            .as_mut()
-            .ok_or_else(|| anyhow!("Spec didn't contain linux field"))?;
-
-        for specdev in &mut linux.devices {
-            let dev_major_minor = format!("{}:{}", specdev.major, specdev.minor);
-
-            if specdev.path == TRUSTED_STORAGE_DEVICE {
-                let data_integrity = AGENT_CONFIG.read().await.data_integrity;
-                info!(
-                    sl!(),
-                    "trusted_store device major:min {}, enable data integrity {}",
-                    dev_major_minor,
-                    data_integrity.to_string()
-                );
-
-                Command::new(INIT_TRUSTED_STORAGE)
-                    .args([&dev_major_minor, &data_integrity.to_string()])
-                    .output()
-                    .expect("Failed to initialize confidential storage");
-            }
-        }
-
         // Both rootfs and volumes (invoked with --volume for instance) will
         // be processed the same way. The idea is to always mount any provided
         // storage to the specified MountPoint, so that it will match what's
@@ -713,54 +638,6 @@ impl AgentService {
             }
         }
     }
-
-    // When being passed an image name through a container annotation, merge its
-    // corresponding bundle OCI specification into the passed container creation one.
-    async fn merge_bundle_oci(&self, container_oci: &mut oci::Spec) -> Result<()> {
-        if let Some(image_name) = container_oci
-            .annotations
-            .get(&ANNO_K8S_IMAGE_NAME.to_string())
-        {
-            if let Some(container_id) = self.sandbox.clone().lock().await.images.get(image_name) {
-                let image_oci_config_path = Path::new(CONTAINER_BASE)
-                    .join(container_id)
-                    .join(CONFIG_JSON);
-                debug!(
-                    sl!(),
-                    "Image bundle config path: {:?}", image_oci_config_path
-                );
-
-                let image_oci =
-                    oci::Spec::load(image_oci_config_path.to_str().ok_or_else(|| {
-                        anyhow!(
-                            "Invalid container image OCI config path {:?}",
-                            image_oci_config_path
-                        )
-                    })?)
-                    .context("load image bundle")?;
-
-                if let Some(container_root) = container_oci.root.as_mut() {
-                    if let Some(image_root) = image_oci.root.as_ref() {
-                        let root_path = Path::new(CONTAINER_BASE)
-                            .join(container_id)
-                            .join(image_root.path.clone());
-                        container_root.path =
-                            String::from(root_path.to_str().ok_or_else(|| {
-                                anyhow!("Invalid container image root path {:?}", root_path)
-                            })?);
-                    }
-                }
-
-                if let Some(container_process) = container_oci.process.as_mut() {
-                    if let Some(image_process) = image_oci.process.as_ref() {
-                        merge_oci_process(container_process, image_process);
-                    }
-                }
-            }
-        }
-
-        Ok(())
-    }
 }
 
 #[async_trait]
@@ -908,7 +785,7 @@ impl agent_ttrpc::AgentService for AgentService {
     ) -> ttrpc::Result<protocols::empty::Empty> {
         trace_rpc_call!(ctx, "pause_container", req);
         is_allowed!(req);
-        let cid = req.container_id();
+        let cid = req.get_container_id();
         let s = Arc::clone(&self.sandbox);
         let mut sandbox = s.lock().await;
 
@@ -932,7 +809,7 @@ impl agent_ttrpc::AgentService for AgentService {
     ) -> ttrpc::Result<protocols::empty::Empty> {
         trace_rpc_call!(ctx, "resume_container", req);
         is_allowed!(req);
-        let cid = req.container_id();
+        let cid = req.get_container_id();
         let s = Arc::clone(&self.sandbox);
         let mut sandbox = s.lock().await;
 
@@ -949,29 +826,6 @@ impl agent_ttrpc::AgentService for AgentService {
         Ok(Empty::new())
     }
 
-    async fn remove_stale_virtiofs_share_mounts(
-        &self,
-        ctx: &TtrpcContext,
-        req: protocols::agent::RemoveStaleVirtiofsShareMountsRequest,
-    ) -> ttrpc::Result<Empty> {
-        trace_rpc_call!(ctx, "remove_stale_virtiofs_share_mounts", req);
-        is_allowed!(req);
-        let mount_infos = parse_mount_table("/proc/self/mountinfo")
-            .map_err(|e| ttrpc_error!(ttrpc::Code::INTERNAL, e))?;
-        for m in &mount_infos {
-            if m.mount_point.starts_with(KATA_GUEST_SHARE_DIR) {
-                // stat the mount point, virtiofs daemon will remove the stale cache and release the fds if the mount point doesn't exist any more.
-                // More details in https://github.com/kata-containers/kata-containers/issues/6455#issuecomment-1477137277
-                match stat::stat(Path::new(&m.mount_point)) {
-                    Ok(_) => info!(sl!(), "stat {} success", m.mount_point),
-                    Err(e) => info!(sl!(), "stat {} failed: {}", m.mount_point, e),
-                }
-            }
-        }
-
-        Ok(Empty::new())
-    }
-
     async fn write_stdin(
         &self,
         _ctx: &TtrpcContext,
@@ -1110,12 +964,16 @@ impl agent_ttrpc::AgentService for AgentService {
         trace_rpc_call!(ctx, "update_routes", req);
         is_allowed!(req);
 
-        let new_routes = req.routes.into_option().map(|r| r.Routes).ok_or_else(|| {
-            ttrpc_error!(
-                ttrpc::Code::INVALID_ARGUMENT,
-                "empty update routes request".to_string(),
-            )
-        })?;
+        let new_routes = req
+            .routes
+            .into_option()
+            .map(|r| r.Routes.into_vec())
+            .ok_or_else(|| {
+                ttrpc_error!(
+                    ttrpc::Code::INVALID_ARGUMENT,
+                    "empty update routes request".to_string(),
+                )
+            })?;
 
         let mut sandbox = self.sandbox.lock().await;
 
@@ -1134,28 +992,11 @@ impl agent_ttrpc::AgentService for AgentService {
         })?;
 
         Ok(protocols::agent::Routes {
-            Routes: list,
+            Routes: RepeatedField::from_vec(list),
             ..Default::default()
         })
     }
 
-    async fn update_ephemeral_mounts(
-        &self,
-        ctx: &TtrpcContext,
-        req: protocols::agent::UpdateEphemeralMountsRequest,
-    ) -> ttrpc::Result<Empty> {
-        trace_rpc_call!(ctx, "update_mounts", req);
-        is_allowed!(req);
-
-        match update_ephemeral_mounts(sl!(), req.storages.to_vec(), self.sandbox.clone()).await {
-            Ok(_) => Ok(Empty::new()),
-            Err(e) => Err(ttrpc_error!(
-                ttrpc::Code::INTERNAL,
-                format!("Failed to update mounts: {:?}", e),
-            )),
-        }
-    }
-
     async fn get_ip_tables(
         &self,
         ctx: &TtrpcContext,
@@ -1333,7 +1174,7 @@ impl agent_ttrpc::AgentService for AgentService {
             })?;
 
         Ok(protocols::agent::Interfaces {
-            Interfaces: list,
+            Interfaces: RepeatedField::from_vec(list),
             ..Default::default()
         })
     }
@@ -1356,7 +1197,7 @@ impl agent_ttrpc::AgentService for AgentService {
             .map_err(|e| ttrpc_error!(ttrpc::Code::INTERNAL, format!("list routes: {:?}", e)))?;
 
         Ok(protocols::agent::Routes {
-            Routes: list,
+            Routes: RepeatedField::from_vec(list),
             ..Default::default()
         })
     }
@@ -1472,7 +1313,7 @@ impl agent_ttrpc::AgentService for AgentService {
         let neighs = req
             .neighbors
             .into_option()
-            .map(|n| n.ARPNeighbors)
+            .map(|n| n.ARPNeighbors.into_vec())
             .ok_or_else(|| {
                 ttrpc_error!(
                     ttrpc::Code::INVALID_ARGUMENT,
@@ -1556,7 +1397,7 @@ impl agent_ttrpc::AgentService for AgentService {
 
         // to get agent details
         let detail = get_agent_details();
-        resp.agent_details = MessageField::some(detail);
+        resp.agent_details = SingularPtrField::some(detail);
 
         Ok(resp)
     }
@@ -1681,8 +1522,8 @@ impl agent_ttrpc::AgentService for AgentService {
             .map(|u| usage_vec.push(u))
             .map_err(|e| ttrpc_error!(ttrpc::Code::INTERNAL, e))?;
 
-        resp.usage = usage_vec;
-        resp.volume_condition = MessageField::some(condition);
+        resp.usage = RepeatedField::from_vec(usage_vec);
+        resp.volume_condition = SingularPtrField::some(condition);
         Ok(resp)
     }
 
@@ -1786,7 +1627,7 @@ fn get_volume_capacity_stats(path: &str) -> Result<VolumeUsage> {
     usage.total = stat.blocks() * block_size;
     usage.available = stat.blocks_free() * block_size;
     usage.used = usage.total - usage.available;
-    usage.unit = VolumeUsage_Unit::BYTES.into();
+    usage.unit = VolumeUsage_Unit::BYTES;
 
     Ok(usage)
 }
@@ -1798,7 +1639,7 @@ fn get_volume_inode_stats(path: &str) -> Result<VolumeUsage> {
     usage.total = stat.files();
     usage.available = stat.files_free();
     usage.used = usage.total - usage.available;
-    usage.unit = VolumeUsage_Unit::INODES.into();
+    usage.unit = VolumeUsage_Unit::INODES;
 
     Ok(usage)
 }
@@ -1818,12 +1659,14 @@ fn get_agent_details() -> AgentDetails {
     detail.set_supports_seccomp(have_seccomp());
     detail.init_daemon = unistd::getpid() == Pid::from_raw(1);
 
-    detail.device_handlers = Vec::new();
-    detail.storage_handlers = STORAGE_HANDLER_LIST
-        .to_vec()
-        .iter()
-        .map(|x| x.to_string())
-        .collect();
+    detail.device_handlers = RepeatedField::new();
+    detail.storage_handlers = RepeatedField::from_vec(
+        STORAGE_HANDLER_LIST
+            .to_vec()
+            .iter()
+            .map(|x| x.to_string())
+            .collect(),
+    );
 
     detail
 }
@@ -1844,7 +1687,7 @@ async fn read_stream(reader: Arc<Mutex<ReadHalf<PipeStream>>>, l: usize) -> Resu
 
 pub fn start(s: Arc<Mutex<Sandbox>>, server_address: &str, init_mode: bool) -> Result<TtrpcServer> {
     let agent_service = Box::new(AgentService {
-        sandbox: s.clone(),
+        sandbox: s,
         init_mode,
     }) as Box<dyn agent_ttrpc::AgentService + Send + Sync>;
 
@@ -1853,20 +1696,14 @@ pub fn start(s: Arc<Mutex<Sandbox>>, server_address: &str, init_mode: bool) -> R
     let health_service = Box::new(HealthService {}) as Box<dyn health_ttrpc::Health + Send + Sync>;
     let health_worker = Arc::new(health_service);
 
-    let image_service =
-        Box::new(image_rpc::ImageService::new(s)) as Box<dyn image_ttrpc::Image + Send + Sync>;
-
     let aservice = agent_ttrpc::create_agent_service(agent_worker);
 
     let hservice = health_ttrpc::create_health(health_worker);
 
-    let iservice = image_ttrpc::create_image(Arc::new(image_service));
-
     let server = TtrpcServer::new()
         .bind(server_address)?
         .register_service(aservice)
-        .register_service(hservice)
-        .register_service(iservice);
+        .register_service(hservice);
 
     info!(sl!(), "ttRPC server started"; "address" => server_address);
 
@@ -2054,50 +1891,23 @@ fn do_copy_file(req: &CopyFileRequest) -> Result<()> {
         ));
     }
 
-    if let Some(parent) = path.parent() {
-        if !parent.exists() {
-            let dir = parent.to_path_buf();
-            if let Err(e) = fs::create_dir_all(&dir) {
-                if e.kind() != std::io::ErrorKind::AlreadyExists {
-                    return Err(e.into());
-                }
-            } else {
-                std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(req.dir_mode))?;
-            }
+    let parent = path.parent();
+
+    let dir = if let Some(parent) = parent {
+        parent.to_path_buf()
+    } else {
+        PathBuf::from("/")
+    };
+
+    fs::create_dir_all(&dir).or_else(|e| {
+        if e.kind() != std::io::ErrorKind::AlreadyExists {
+            return Err(e);
         }
-    }
 
-    let sflag = stat::SFlag::from_bits_truncate(req.file_mode);
+        Ok(())
+    })?;
 
-    if sflag.contains(stat::SFlag::S_IFDIR) {
-        fs::create_dir(path.clone()).or_else(|e| {
-            if e.kind() != std::io::ErrorKind::AlreadyExists {
-                return Err(e);
-            }
-            Ok(())
-        })?;
-
-        std::fs::set_permissions(path.clone(), std::fs::Permissions::from_mode(req.file_mode))?;
-
-        unistd::chown(
-            &path,
-            Some(Uid::from_raw(req.uid as u32)),
-            Some(Gid::from_raw(req.gid as u32)),
-        )?;
-
-        return Ok(());
-    }
-
-    if sflag.contains(stat::SFlag::S_IFLNK) {
-        let src = PathBuf::from(String::from_utf8(req.data.clone()).unwrap());
-
-        unistd::symlinkat(&src, None, &path)?;
-        let path_str = CString::new(path.to_str().unwrap())?;
-        let ret = unsafe { libc::lchown(path_str.as_ptr(), req.uid as u32, req.gid as u32) };
-        Errno::result(ret).map(drop)?;
-
-        return Ok(());
-    }
+    std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(req.dir_mode))?;
 
     let mut tmpfile = path.clone();
     tmpfile.set_extension("tmp");
@@ -2164,26 +1974,18 @@ pub fn setup_bundle(cid: &str, spec: &mut Spec) -> Result<PathBuf> {
     let spec_root_path = Path::new(&spec_root.path);
 
     let bundle_path = Path::new(CONTAINER_BASE).join(cid);
-    let config_path = bundle_path.join(CONFIG_JSON);
+    let config_path = bundle_path.join("config.json");
     let rootfs_path = bundle_path.join("rootfs");
 
-    let rootfs_exists = Path::new(&rootfs_path).exists();
-    info!(
-        sl!(),
-        "The rootfs_path is {:?} and exists: {}", rootfs_path, rootfs_exists
-    );
-
-    if !rootfs_exists {
-        fs::create_dir_all(&rootfs_path)?;
-        baremount(
-            spec_root_path,
-            &rootfs_path,
-            "bind",
-            MsFlags::MS_BIND,
-            "",
-            &sl!(),
-        )?;
-    }
+    fs::create_dir_all(&rootfs_path)?;
+    baremount(
+        spec_root_path,
+        &rootfs_path,
+        "bind",
+        MsFlags::MS_BIND,
+        "",
+        &sl!(),
+    )?;
 
     let rootfs_path_name = rootfs_path
         .to_str()
@@ -2223,7 +2025,7 @@ fn load_kernel_module(module: &protocols::agent::KernelModule) -> Result<()> {
 
     let mut args = vec!["-v".to_string(), module.name.clone()];
 
-    if !module.parameters.is_empty() {
+    if module.parameters.len() > 0 {
         args.extend(module.parameters.to_vec())
     }
 
@@ -3164,135 +2966,4 @@ COMMIT
             "We should see the resulting rule"
         );
     }
-
-    #[tokio::test]
-    async fn test_merge_cwd() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            container_process_cwd: &'a str,
-            image_process_cwd: &'a str,
-            expected: &'a str,
-        }
-
-        let tests = &[
-            // Image cwd should override blank container cwd
-            // TODO - how can we tell the user didn't specifically set it to `/` vs not setting at all? Is that scenario valid?
-            TestData {
-                container_process_cwd: "/",
-                image_process_cwd: "/imageDir",
-                expected: "/imageDir",
-            },
-            // Container cwd should override image cwd
-            TestData {
-                container_process_cwd: "/containerDir",
-                image_process_cwd: "/imageDir",
-                expected: "/containerDir",
-            },
-            // Container cwd should override blank image cwd
-            TestData {
-                container_process_cwd: "/containerDir",
-                image_process_cwd: "/",
-                expected: "/containerDir",
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let mut container_process = oci::Process {
-                cwd: d.container_process_cwd.to_string(),
-                ..Default::default()
-            };
-
-            let image_process = oci::Process {
-                cwd: d.image_process_cwd.to_string(),
-                ..Default::default()
-            };
-
-            merge_oci_process(&mut container_process, &image_process);
-
-            assert_eq!(d.expected, container_process.cwd, "{}", msg);
-        }
-    }
-
-    #[tokio::test]
-    async fn test_merge_env() {
-        #[derive(Debug)]
-        struct TestData {
-            container_process_env: Vec<String>,
-            image_process_env: Vec<String>,
-            expected: Vec<String>,
-        }
-
-        let tests = &[
-            // Test that the pods environment overrides the images
-            TestData {
-                container_process_env: vec!["ISPRODUCTION=true".to_string()],
-                image_process_env: vec!["ISPRODUCTION=false".to_string()],
-                expected: vec!["ISPRODUCTION=true".to_string()],
-            },
-            // Test that multiple environment variables can be overrided
-            TestData {
-                container_process_env: vec![
-                    "ISPRODUCTION=true".to_string(),
-                    "ISDEVELOPMENT=false".to_string(),
-                ],
-                image_process_env: vec![
-                    "ISPRODUCTION=false".to_string(),
-                    "ISDEVELOPMENT=true".to_string(),
-                ],
-                expected: vec![
-                    "ISPRODUCTION=true".to_string(),
-                    "ISDEVELOPMENT=false".to_string(),
-                ],
-            },
-            // Test that when none of the variables match do not override them
-            TestData {
-                container_process_env: vec!["ANOTHERENV=TEST".to_string()],
-                image_process_env: vec![
-                    "ISPRODUCTION=false".to_string(),
-                    "ISDEVELOPMENT=true".to_string(),
-                ],
-                expected: vec![
-                    "ANOTHERENV=TEST".to_string(),
-                    "ISPRODUCTION=false".to_string(),
-                    "ISDEVELOPMENT=true".to_string(),
-                ],
-            },
-            // Test a mix of both overriding and not
-            TestData {
-                container_process_env: vec![
-                    "ANOTHERENV=TEST".to_string(),
-                    "ISPRODUCTION=true".to_string(),
-                ],
-                image_process_env: vec![
-                    "ISPRODUCTION=false".to_string(),
-                    "ISDEVELOPMENT=true".to_string(),
-                ],
-                expected: vec![
-                    "ANOTHERENV=TEST".to_string(),
-                    "ISPRODUCTION=true".to_string(),
-                    "ISDEVELOPMENT=true".to_string(),
-                ],
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let mut container_process = oci::Process {
-                env: d.container_process_env.clone(),
-                ..Default::default()
-            };
-
-            let image_process = oci::Process {
-                env: d.image_process_env.clone(),
-                ..Default::default()
-            };
-
-            merge_oci_process(&mut container_process, &image_process);
-
-            assert_eq!(d.expected, container_process.env, "{}", msg);
-        }
-    }
 }

src/agent/src/sandbox.rs

@@ -60,7 +60,6 @@ pub struct Sandbox {
     pub event_tx: Option<Sender<String>>,
     pub bind_watcher: BindWatcher,
     pub pcimap: HashMap<pci::Address, pci::Address>,
-    pub images: HashMap<String, String>,
 }
 
 impl Sandbox {
@@ -94,7 +93,6 @@ impl Sandbox {
             event_tx: Some(tx),
             bind_watcher: BindWatcher::new(),
             pcimap: HashMap::new(),
-            images: HashMap::new(),
         })
     }
 

src/agent/src/signal.rs

@@ -24,7 +24,7 @@ async fn handle_sigchild(logger: Logger, sandbox: Arc<Mutex<Sandbox>>) -> Result
     loop {
         // Avoid reaping the undesirable child's signal, e.g., execute_hook's
         // The lock should be released immediately.
-        let _locker = rustjail::container::WAIT_PID_LOCKER.lock().await;
+        rustjail::container::WAIT_PID_LOCKER.lock().await;
         let result = wait::waitpid(
             Some(Pid::from_raw(-1)),
             Some(WaitPidFlag::WNOHANG | WaitPidFlag::__WALL),

src/agent/vsock-exporter/Cargo.toml

@@ -18,4 +18,4 @@ bincode = "1.3.3"
 byteorder = "1.4.3"
 slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug"] }
 async-trait = "0.1.50"
-tokio = "1.28.1"
+tokio = "1.2.0"

src/dragonball/Cargo.toml

@@ -19,11 +19,11 @@ dbs-boot = "0.3.0"
 dbs-device = "0.2.0"
 dbs-interrupt = { version = "0.2.0", features = ["kvm-irq"] }
 dbs-legacy-devices = "0.1.0"
-dbs-upcall = { version = "0.2.0", optional = true }
+dbs-upcall = { version = "0.1.0", optional = true }
 dbs-utils = "0.2.0"
-dbs-virtio-devices = { version = "0.2.0", optional = true, features = ["virtio-mmio"] }
-kvm-bindings = "0.6.0"
-kvm-ioctls = "0.12.0"
+dbs-virtio-devices = { version = "0.1.0", optional = true, features = ["virtio-mmio"] }
+kvm-bindings = "0.5.0"
+kvm-ioctls = "0.11.0"
 lazy_static = "1.2"
 libc = "0.2.39"
 linux-loader = "0.6.0"
@@ -37,9 +37,8 @@ slog = "2.5.2"
 slog-scope = "4.4.0"
 thiserror = "1"
 vmm-sys-util = "0.11.0"
-virtio-queue = { version = "0.6.0", optional = true }
+virtio-queue = { version = "0.4.0", optional = true }
 vm-memory = { version = "0.9.0", features = ["backend-mmap"] }
-crossbeam-channel = "0.5.6"
 
 [dev-dependencies]
 slog-term = "2.9.0"
@@ -48,7 +47,7 @@ test-utils = { path = "../libs/test-utils" }
 
 [features]
 acpi = []
-atomic-guest-memory = ["vm-memory/backend-atomic"]
+atomic-guest-memory = [ "vm-memory/backend-atomic" ]
 hotplug = ["virtio-vsock"]
 virtio-vsock = ["dbs-virtio-devices/virtio-vsock", "virtio-queue"]
 virtio-blk = ["dbs-virtio-devices/virtio-blk", "virtio-queue"]

src/dragonball/src/api/v1/instance_info.rs

@@ -5,15 +5,6 @@
 
 use serde_derive::{Deserialize, Serialize};
 
-/// This struct represents the strongly typed equivalent of the json body
-/// from confidential container related requests.
-#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Serialize)]
-#[serde(deny_unknown_fields)]
-pub enum ConfidentialVmType {
-    /// Intel Trusted Domain
-    TDX = 2,
-}
-
 /// The microvm state.
 ///
 /// When Dragonball starts, the instance state is Uninitialized. Once start_microvm method is
@@ -59,38 +50,27 @@ pub struct InstanceInfo {
     pub vmm_version: String,
     /// The pid of the current VMM process.
     pub pid: u32,
-    /// The tid of the current VMM master thread.
-    pub master_tid: u32,
     /// The state of async actions.
     pub async_state: AsyncState,
     /// List of tids of vcpu threads (vcpu index, tid)
     pub tids: Vec<(u8, u32)>,
     /// Last instance downtime
     pub last_instance_downtime: u64,
-    /// confidential vm type
-    pub confidential_vm_type: Option<ConfidentialVmType>,
 }
 
 impl InstanceInfo {
-    /// create instance info object with given id, version, platform type and confidential vm type.
+    /// create instance info object with given id, version, and platform type
     pub fn new(id: String, vmm_version: String) -> Self {
         InstanceInfo {
             id,
             state: InstanceState::Uninitialized,
             vmm_version,
             pid: std::process::id(),
-            master_tid: 0,
             async_state: AsyncState::Uninitialized,
             tids: Vec::new(),
             last_instance_downtime: 0,
-            confidential_vm_type: None,
         }
     }
-
-    /// return true if VM confidential type is TDX
-    pub fn is_tdx_enabled(&self) -> bool {
-        matches!(self.confidential_vm_type, Some(ConfidentialVmType::TDX))
-    }
 }
 
 impl Default for InstanceInfo {
@@ -100,11 +80,9 @@ impl Default for InstanceInfo {
             state: InstanceState::Uninitialized,
             vmm_version: env!("CARGO_PKG_VERSION").to_string(),
             pid: std::process::id(),
-            master_tid: 0,
             async_state: AsyncState::Uninitialized,
             tids: Vec::new(),
             last_instance_downtime: 0,
-            confidential_vm_type: None,
         }
     }
 }

src/dragonball/src/api/v1/mod.rs

@@ -12,7 +12,7 @@ pub use self::boot_source::{BootSourceConfig, BootSourceConfigError, DEFAULT_KER
 
 /// Wrapper over the microVM general information.
 mod instance_info;
-pub use self::instance_info::{ConfidentialVmType, InstanceInfo, InstanceState};
+pub use self::instance_info::{InstanceInfo, InstanceState};
 
 /// Wrapper for configuring the memory and CPU of the microVM.
 mod machine_config;

src/dragonball/src/api/v1/vmm_action.rs

@@ -7,8 +7,8 @@
 // found in the THIRD-PARTY file.
 
 use std::fs::File;
+use std::sync::mpsc::{Receiver, Sender, TryRecvError};
 
-use crossbeam_channel::{Receiver, Sender, TryRecvError};
 use log::{debug, error, info, warn};
 
 use crate::error::{Result, StartMicroVmError, StopMicrovmError};
@@ -486,9 +486,7 @@ impl VmmService {
                 VmmActionError::Block(BlockDeviceError::UpdateNotAllowedPostBoot)
             })?;
 
-        vm.device_manager_mut()
-            .block_manager
-            .insert_device(ctx, config)
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, config)
             .map(|_| VmmData::Empty)
             .map_err(VmmActionError::Block)
     }
@@ -502,9 +500,7 @@ impl VmmService {
     ) -> VmmRequestResult {
         let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?;
 
-        vm.device_manager_mut()
-            .block_manager
-            .update_device_ratelimiters(config)
+        BlockDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), config)
             .map(|_| VmmData::Empty)
             .map_err(VmmActionError::Block)
     }
@@ -522,9 +518,7 @@ impl VmmService {
             .create_device_op_context(Some(event_mgr.epoll_manager()))
             .map_err(|_| VmmActionError::Block(BlockDeviceError::UpdateNotAllowedPostBoot))?;
 
-        vm.device_manager_mut()
-            .block_manager
-            .remove_device(ctx, drive_id)
+        BlockDeviceMgr::remove_device(vm.device_manager_mut(), ctx, drive_id)
             .map(|_| VmmData::Empty)
             .map_err(VmmActionError::Block)
     }
@@ -549,9 +543,7 @@ impl VmmService {
                 }
             })?;
 
-        vm.device_manager_mut()
-            .virtio_net_manager
-            .insert_device(ctx, config)
+        VirtioNetDeviceMgr::insert_device(vm.device_manager_mut(), ctx, config)
             .map(|_| VmmData::Empty)
             .map_err(VmmActionError::VirtioNet)
     }
@@ -564,9 +556,7 @@ impl VmmService {
     ) -> VmmRequestResult {
         let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?;
 
-        vm.device_manager_mut()
-            .virtio_net_manager
-            .update_device_ratelimiters(config)
+        VirtioNetDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), config)
             .map(|_| VmmData::Empty)
             .map_err(VmmActionError::VirtioNet)
     }
@@ -686,9 +676,9 @@ fn handle_cpu_topology(
 
 #[cfg(test)]
 mod tests {
+    use std::sync::mpsc::channel;
     use std::sync::{Arc, Mutex};
 
-    use crossbeam_channel::unbounded;
     use dbs_utils::epoll_manager::EpollManager;
     use test_utils::skip_if_not_root;
     use vmm_sys_util::tempfile::TempFile;
@@ -712,8 +702,8 @@ mod tests {
         }
 
         fn check_request(&mut self) {
-            let (to_vmm, from_api) = unbounded();
-            let (to_api, from_vmm) = unbounded();
+            let (to_vmm, from_api) = channel();
+            let (to_api, from_vmm) = channel();
 
             let epoll_mgr = EpollManager::default();
             let vmm = Arc::new(Mutex::new(create_vmm_instance(epoll_mgr.clone())));
@@ -738,8 +728,8 @@ mod tests {
     fn test_vmm_action_receive_unknown() {
         skip_if_not_root!();
 
-        let (_to_vmm, from_api) = unbounded();
-        let (to_api, _from_vmm) = unbounded();
+        let (_to_vmm, from_api) = channel();
+        let (to_api, _from_vmm) = channel();
         let epoll_mgr = EpollManager::default();
         let vmm = Arc::new(Mutex::new(create_vmm_instance(epoll_mgr.clone())));
         let mut vservice = VmmService::new(from_api, to_api);
@@ -752,8 +742,8 @@ mod tests {
     #[should_panic]
     #[test]
     fn test_vmm_action_disconnected() {
-        let (to_vmm, from_api) = unbounded();
-        let (to_api, _from_vmm) = unbounded();
+        let (to_vmm, from_api) = channel();
+        let (to_api, _from_vmm) = channel();
         let epoll_mgr = EpollManager::default();
         let vmm = Arc::new(Mutex::new(create_vmm_instance(epoll_mgr.clone())));
         let mut vservice = VmmService::new(from_api, to_api);

src/dragonball/src/config_manager.rs

@@ -231,7 +231,7 @@ where
                         info.config.check_conflicts(config)?;
                     }
                 }
-                self.info_list[index].config = config.clone();
+                self.info_list[index] = device_info;
                 index
             }
             None => {
@@ -278,11 +278,6 @@ where
         self.info_list.iter_mut()
     }
 
-    /// Remove the last device config info from the `info_list`.
-    pub fn pop(&mut self) -> Option<DeviceConfigInfo<T>> {
-        self.info_list.pop()
-    }
-
     fn get_index_by_id(&self, config: &T) -> Option<usize> {
         self.info_list
             .iter()

src/dragonball/src/device_manager/blk_dev_mgr.rs

@@ -340,7 +340,7 @@ impl BlockDeviceMgr {
     /// the existing entry.
     /// Inserting a secondary root block device will fail.
     pub fn insert_device(
-        &mut self,
+        device_mgr: &mut DeviceManager,
         mut ctx: DeviceOpContext,
         config: BlockDeviceConfigInfo,
     ) -> std::result::Result<(), BlockDeviceError> {
@@ -348,8 +348,10 @@ impl BlockDeviceMgr {
             return Err(BlockDeviceError::UpdateNotAllowedPostBoot);
         }
 
+        let mgr = &mut device_mgr.block_manager;
+
         // If the id of the drive already exists in the list, the operation is update.
-        match self.get_index_of_drive_id(config.id()) {
+        match mgr.get_index_of_drive_id(config.id()) {
             Some(index) => {
                 // No support for runtime update yet.
                 if ctx.is_hotplug {
@@ -357,19 +359,19 @@ impl BlockDeviceMgr {
                         config.path_on_host.clone(),
                     ))
                 } else {
-                    for (idx, info) in self.info_list.iter().enumerate() {
+                    for (idx, info) in mgr.info_list.iter().enumerate() {
                         if idx != index {
                             info.config.check_conflicts(&config)?;
                         }
                     }
-                    self.update(index, config)
+                    mgr.update(index, config)
                 }
             }
             None => {
-                for info in self.info_list.iter() {
+                for info in mgr.info_list.iter() {
                     info.config.check_conflicts(&config)?;
                 }
-                let index = self.create(config.clone())?;
+                let index = mgr.create(config.clone())?;
                 if !ctx.is_hotplug {
                     return Ok(());
                 }
@@ -381,16 +383,17 @@ impl BlockDeviceMgr {
                         let dev = DeviceManager::create_mmio_virtio_device(
                             device,
                             &mut ctx,
-                            config.use_shared_irq.unwrap_or(self.use_shared_irq),
+                            config.use_shared_irq.unwrap_or(mgr.use_shared_irq),
                             config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ),
                         )
                         .map_err(BlockDeviceError::DeviceManager)?;
-                        self.update_device_by_index(index, Arc::clone(&dev))?;
+                        mgr.update_device_by_index(index, Arc::clone(&dev))?;
                         // live-upgrade need save/restore device from info.device.
-                        self.info_list[index].set_device(dev.clone());
+                        mgr.info_list[index].set_device(dev.clone());
                         ctx.insert_hotplug_mmio_device(&dev, None).map_err(|e| {
                             let logger = ctx.logger().new(slog::o!());
-                            self.remove_device(ctx, &config.drive_id).unwrap();
+                            BlockDeviceMgr::remove_device(device_mgr, ctx, &config.drive_id)
+                                .unwrap();
                             error!(
                                 logger,
                                 "failed to hot-add virtio block device {}, {:?}",
@@ -463,7 +466,7 @@ impl BlockDeviceMgr {
 
     /// remove a block device, it basically is the inverse operation of `insert_device``
     pub fn remove_device(
-        &mut self,
+        dev_mgr: &mut DeviceManager,
         mut ctx: DeviceOpContext,
         drive_id: &str,
     ) -> std::result::Result<(), BlockDeviceError> {
@@ -471,7 +474,8 @@ impl BlockDeviceMgr {
             return Err(BlockDeviceError::UpdateNotAllowedPostBoot);
         }
 
-        match self.remove(drive_id) {
+        let mgr = &mut dev_mgr.block_manager;
+        match mgr.remove(drive_id) {
             Some(mut info) => {
                 info!(ctx.logger(), "remove drive {}", info.config.drive_id);
                 if let Some(device) = info.device.take() {
@@ -727,14 +731,15 @@ impl BlockDeviceMgr {
 
     /// Update the ratelimiter settings of a virtio blk device.
     pub fn update_device_ratelimiters(
-        &mut self,
+        device_mgr: &mut DeviceManager,
         new_cfg: BlockDeviceConfigUpdateInfo,
     ) -> std::result::Result<(), BlockDeviceError> {
-        match self.get_index_of_drive_id(&new_cfg.drive_id) {
+        let mgr = &mut device_mgr.block_manager;
+        match mgr.get_index_of_drive_id(&new_cfg.drive_id) {
             Some(index) => {
-                let config = &mut self.info_list[index].config;
+                let config = &mut mgr.info_list[index].config;
                 config.rate_limiter = new_cfg.rate_limiter.clone();
-                let device = self.info_list[index]
+                let device = mgr.info_list[index]
                     .device
                     .as_mut()
                     .ok_or_else(|| BlockDeviceError::InvalidDeviceId("".to_owned()))?;
@@ -771,623 +776,3 @@ impl Default for BlockDeviceMgr {
         }
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use test_utils::skip_if_not_root;
-    use vmm_sys_util::tempfile::TempFile;
-
-    use super::*;
-    use crate::test_utils::tests::create_vm_for_test;
-
-    #[test]
-    fn test_block_device_type() {
-        let dev_type = BlockDeviceType::get_type("spool:/device1");
-        assert_eq!(dev_type, BlockDeviceType::Spool);
-        let dev_type = BlockDeviceType::get_type("/device1");
-        assert_eq!(dev_type, BlockDeviceType::RawBlock);
-    }
-
-    #[test]
-    fn test_create_block_devices_configs() {
-        let mgr = BlockDeviceMgr::default();
-        assert!(!mgr.has_root_block_device());
-        assert!(!mgr.has_part_uuid_root());
-        assert!(!mgr.is_read_only_root());
-        assert_eq!(mgr.get_index_of_drive_id(""), None);
-        assert_eq!(mgr.info_list.len(), 0);
-    }
-
-    #[test]
-    fn test_add_non_root_block_device() {
-        skip_if_not_root!();
-        let dummy_file = TempFile::new().unwrap();
-        let dummy_path = dummy_file.as_path().to_owned();
-        let dummy_id = String::from("1");
-        let dummy_block_device = BlockDeviceConfigInfo {
-            path_on_host: dummy_path.clone(),
-            device_type: BlockDeviceType::RawBlock,
-            is_root_device: false,
-            part_uuid: None,
-            is_read_only: false,
-            is_direct: false,
-            no_drop: false,
-            drive_id: dummy_id.clone(),
-            rate_limiter: None,
-            num_queues: BlockDeviceConfigInfo::default_num_queues(),
-            queue_size: 128,
-            use_shared_irq: None,
-            use_generic_irq: None,
-        };
-
-        let mut vm = crate::vm::tests::create_vm_instance();
-        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        assert!(vm
-            .device_manager_mut()
-            .block_manager
-            .insert_device(ctx, dummy_block_device.clone(),)
-            .is_ok());
-
-        assert_eq!(vm.device_manager().block_manager.info_list.len(), 1);
-        assert!(!vm.device_manager().block_manager.has_root_block_device());
-        assert!(!vm.device_manager().block_manager.has_part_uuid_root());
-        assert!(!vm.device_manager().block_manager.is_read_only_root());
-        assert_eq!(vm.device_manager().block_manager.info_list.len(), 1);
-        assert_eq!(
-            vm.device_manager().block_manager.info_list[0]
-                .config
-                .device_type(),
-            BlockDeviceType::RawBlock
-        );
-        assert_eq!(
-            vm.device_manager().block_manager.info_list[0]
-                .config
-                .queue_sizes(),
-            [128u16]
-        );
-
-        let dev_config = vm.device_manager().block_manager.iter().next().unwrap();
-        assert_eq!(dev_config.config, dummy_block_device);
-        assert!(vm
-            .device_manager()
-            .block_manager
-            .get_index_of_drive_path(&dummy_path)
-            .is_some());
-        assert!(vm
-            .device_manager()
-            .block_manager
-            .get_index_of_drive_id(&dummy_id)
-            .is_some());
-    }
-
-    #[test]
-    fn test_update_blk_device_ratelimiters() {
-        skip_if_not_root!();
-        //Init vm for test.
-        let mut vm = create_vm_for_test();
-        let device_op_ctx = DeviceOpContext::new(
-            Some(vm.epoll_manager().clone()),
-            vm.device_manager(),
-            Some(vm.vm_as().unwrap().clone()),
-            None,
-            false,
-        );
-
-        let dummy_file = TempFile::new().unwrap();
-        let dummy_path = dummy_file.as_path().to_owned();
-
-        let dummy_block_device = BlockDeviceConfigInfo {
-            path_on_host: dummy_path,
-            device_type: BlockDeviceType::RawBlock,
-            is_root_device: true,
-            part_uuid: None,
-            is_read_only: true,
-            is_direct: false,
-            no_drop: false,
-            drive_id: String::from("1"),
-            rate_limiter: None,
-            num_queues: BlockDeviceConfigInfo::default_num_queues(),
-            queue_size: 128,
-            use_shared_irq: None,
-            use_generic_irq: None,
-        };
-        vm.device_manager_mut()
-            .block_manager
-            .insert_device(device_op_ctx, dummy_block_device)
-            .unwrap();
-
-        let cfg = BlockDeviceConfigUpdateInfo {
-            drive_id: String::from("1"),
-            rate_limiter: None,
-        };
-
-        let mut device_op_ctx = DeviceOpContext::new(
-            Some(vm.epoll_manager().clone()),
-            vm.device_manager(),
-            Some(vm.vm_as().unwrap().clone()),
-            None,
-            false,
-        );
-
-        vm.device_manager_mut()
-            .block_manager
-            .attach_devices(&mut device_op_ctx)
-            .unwrap();
-        assert_eq!(vm.device_manager().block_manager.info_list.len(), 1);
-
-        //Patch while the epoll handler is invalid.
-        let expected_error = "could not send patch message to the block epoll handler".to_string();
-
-        assert_eq!(
-            vm.device_manager_mut()
-                .block_manager
-                .update_device_ratelimiters(cfg)
-                .unwrap_err()
-                .to_string(),
-            expected_error
-        );
-
-        //Invalid drive id
-        let cfg2 = BlockDeviceConfigUpdateInfo {
-            drive_id: String::from("2"),
-            rate_limiter: None,
-        };
-
-        let expected_error = format!("invalid block device id '{0}'", cfg2.drive_id);
-
-        assert_eq!(
-            vm.device_manager_mut()
-                .block_manager
-                .update_device_ratelimiters(cfg2)
-                .unwrap_err()
-                .to_string(),
-            expected_error
-        );
-    }
-
-    #[test]
-    fn test_add_one_root_block_device() {
-        skip_if_not_root!();
-        let dummy_file = TempFile::new().unwrap();
-        let dummy_path = dummy_file.as_path().to_owned();
-        let dummy_block_device = BlockDeviceConfigInfo {
-            path_on_host: dummy_path,
-            device_type: BlockDeviceType::RawBlock,
-            is_root_device: true,
-            part_uuid: None,
-            is_read_only: true,
-            is_direct: false,
-            no_drop: false,
-            drive_id: String::from("1"),
-            rate_limiter: None,
-            num_queues: BlockDeviceConfigInfo::default_num_queues(),
-            queue_size: 128,
-            use_shared_irq: None,
-            use_generic_irq: None,
-        };
-
-        let mut vm = crate::vm::tests::create_vm_instance();
-        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        assert!(vm
-            .device_manager_mut()
-            .block_manager
-            .insert_device(ctx, dummy_block_device.clone(),)
-            .is_ok());
-
-        assert_eq!(vm.device_manager().block_manager.info_list.len(), 1);
-        assert!(vm.device_manager().block_manager.has_root_block);
-        assert!(!vm.device_manager().block_manager.has_part_uuid_root);
-        assert!(vm.device_manager().block_manager.read_only_root);
-        assert_eq!(vm.device_manager().block_manager.info_list.len(), 1);
-
-        let dev_config = vm.device_manager().block_manager.iter().next().unwrap();
-        assert_eq!(dev_config.config, dummy_block_device);
-        assert!(vm.device_manager().block_manager.is_read_only_root());
-    }
-
-    #[test]
-    fn test_add_two_root_block_devices_configs() {
-        skip_if_not_root!();
-        let dummy_file_1 = TempFile::new().unwrap();
-        let dummy_path_1 = dummy_file_1.as_path().to_owned();
-        let root_block_device_1 = BlockDeviceConfigInfo {
-            path_on_host: dummy_path_1,
-            device_type: BlockDeviceType::RawBlock,
-            is_root_device: true,
-            part_uuid: None,
-            is_read_only: false,
-            is_direct: false,
-            no_drop: false,
-            drive_id: String::from("1"),
-            rate_limiter: None,
-            num_queues: BlockDeviceConfigInfo::default_num_queues(),
-            queue_size: 128,
-            use_shared_irq: None,
-            use_generic_irq: None,
-        };
-
-        let dummy_file_2 = TempFile::new().unwrap();
-        let dummy_path_2 = dummy_file_2.as_path().to_owned();
-        let root_block_device_2 = BlockDeviceConfigInfo {
-            path_on_host: dummy_path_2,
-            device_type: BlockDeviceType::RawBlock,
-            is_root_device: true,
-            part_uuid: None,
-            is_read_only: false,
-            is_direct: false,
-            no_drop: false,
-            drive_id: String::from("2"),
-            rate_limiter: None,
-            num_queues: BlockDeviceConfigInfo::default_num_queues(),
-            queue_size: 128,
-            use_shared_irq: None,
-            use_generic_irq: None,
-        };
-
-        let mut vm = crate::vm::tests::create_vm_instance();
-        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        vm.device_manager_mut()
-            .block_manager
-            .insert_device(ctx, root_block_device_1)
-            .unwrap();
-        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        assert!(vm
-            .device_manager_mut()
-            .block_manager
-            .insert_device(ctx, root_block_device_2)
-            .is_err());
-    }
-
-    #[test]
-    // Test BlockDevicesConfigs::add when you first add the root device and then the other devices.
-    fn test_add_root_block_device_first() {
-        skip_if_not_root!();
-        let dummy_file_1 = TempFile::new().unwrap();
-        let dummy_path_1 = dummy_file_1.as_path().to_owned();
-        let root_block_device = BlockDeviceConfigInfo {
-            path_on_host: dummy_path_1,
-            device_type: BlockDeviceType::RawBlock,
-            is_root_device: true,
-            part_uuid: None,
-            is_read_only: false,
-            is_direct: false,
-            no_drop: false,
-            drive_id: String::from("1"),
-            rate_limiter: None,
-            num_queues: BlockDeviceConfigInfo::default_num_queues(),
-            queue_size: 128,
-            use_shared_irq: None,
-            use_generic_irq: None,
-        };
-
-        let dummy_file_2 = TempFile::new().unwrap();
-        let dummy_path_2 = dummy_file_2.as_path().to_owned();
-        let dummy_block_device_2 = BlockDeviceConfigInfo {
-            path_on_host: dummy_path_2,
-            device_type: BlockDeviceType::RawBlock,
-            is_root_device: false,
-            part_uuid: None,
-            is_read_only: false,
-            is_direct: false,
-            no_drop: false,
-            drive_id: String::from("2"),
-            rate_limiter: None,
-            num_queues: BlockDeviceConfigInfo::default_num_queues(),
-            queue_size: 128,
-            use_shared_irq: None,
-            use_generic_irq: None,
-        };
-
-        let dummy_file_3 = TempFile::new().unwrap();
-        let dummy_path_3 = dummy_file_3.as_path().to_owned();
-        let dummy_block_device_3 = BlockDeviceConfigInfo {
-            path_on_host: dummy_path_3,
-            device_type: BlockDeviceType::RawBlock,
-            is_root_device: false,
-            part_uuid: None,
-            is_read_only: false,
-            is_direct: false,
-            no_drop: false,
-            drive_id: String::from("3"),
-            rate_limiter: None,
-            num_queues: BlockDeviceConfigInfo::default_num_queues(),
-            queue_size: 128,
-            use_shared_irq: None,
-            use_generic_irq: None,
-        };
-
-        let mut vm = crate::vm::tests::create_vm_instance();
-        vm.device_manager_mut()
-            .block_manager
-            .create(root_block_device.clone())
-            .unwrap();
-        vm.device_manager_mut()
-            .block_manager
-            .create(dummy_block_device_2.clone())
-            .unwrap();
-        vm.device_manager_mut()
-            .block_manager
-            .create(dummy_block_device_3.clone())
-            .unwrap();
-
-        assert!(vm.device_manager().block_manager.has_root_block_device(),);
-        assert!(!vm.device_manager().block_manager.has_part_uuid_root());
-        assert_eq!(vm.device_manager().block_manager.info_list.len(), 3);
-
-        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        vm.device_manager_mut()
-            .block_manager
-            .insert_device(ctx, root_block_device)
-            .unwrap();
-
-        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        vm.device_manager_mut()
-            .block_manager
-            .insert_device(ctx, dummy_block_device_2)
-            .unwrap();
-
-        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        vm.device_manager_mut()
-            .block_manager
-            .insert_device(ctx, dummy_block_device_3)
-            .unwrap();
-    }
-
-    #[test]
-    // Test BlockDevicesConfigs::add when you add other devices first and then the root device.
-    fn test_root_block_device_add_last() {
-        skip_if_not_root!();
-        let dummy_file_1 = TempFile::new().unwrap();
-        let dummy_path_1 = dummy_file_1.as_path().to_owned();
-        let root_block_device = BlockDeviceConfigInfo {
-            path_on_host: dummy_path_1,
-            device_type: BlockDeviceType::RawBlock,
-            is_root_device: true,
-            part_uuid: None,
-            is_read_only: false,
-            is_direct: false,
-            no_drop: false,
-            drive_id: String::from("1"),
-            rate_limiter: None,
-            num_queues: BlockDeviceConfigInfo::default_num_queues(),
-            queue_size: 128,
-            use_shared_irq: None,
-            use_generic_irq: None,
-        };
-
-        let dummy_file_2 = TempFile::new().unwrap();
-        let dummy_path_2 = dummy_file_2.as_path().to_owned();
-        let dummy_block_device_2 = BlockDeviceConfigInfo {
-            path_on_host: dummy_path_2,
-            device_type: BlockDeviceType::RawBlock,
-            is_root_device: false,
-            part_uuid: None,
-            is_read_only: false,
-            is_direct: false,
-            no_drop: false,
-            drive_id: String::from("2"),
-            rate_limiter: None,
-            num_queues: BlockDeviceConfigInfo::default_num_queues(),
-            queue_size: 128,
-            use_shared_irq: None,
-            use_generic_irq: None,
-        };
-
-        let dummy_file_3 = TempFile::new().unwrap();
-        let dummy_path_3 = dummy_file_3.as_path().to_owned();
-        let dummy_block_device_3 = BlockDeviceConfigInfo {
-            path_on_host: dummy_path_3,
-            device_type: BlockDeviceType::RawBlock,
-            is_root_device: false,
-            part_uuid: None,
-            is_read_only: false,
-            is_direct: false,
-            no_drop: false,
-            drive_id: String::from("3"),
-            rate_limiter: None,
-            num_queues: BlockDeviceConfigInfo::default_num_queues(),
-            queue_size: 128,
-            use_shared_irq: None,
-            use_generic_irq: None,
-        };
-
-        let mut vm = crate::vm::tests::create_vm_instance();
-
-        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        vm.device_manager_mut()
-            .block_manager
-            .insert_device(ctx, dummy_block_device_2.clone())
-            .unwrap();
-        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        vm.device_manager_mut()
-            .block_manager
-            .insert_device(ctx, dummy_block_device_3.clone())
-            .unwrap();
-        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        vm.device_manager_mut()
-            .block_manager
-            .insert_device(ctx, root_block_device.clone())
-            .unwrap();
-
-        assert!(vm.device_manager().block_manager.has_root_block_device(),);
-        assert!(!vm.device_manager().block_manager.has_part_uuid_root());
-        assert_eq!(vm.device_manager().block_manager.info_list.len(), 3);
-
-        let mut block_dev_iter = vm.device_manager().block_manager.iter();
-        // The root device should be first in the list no matter of the order in
-        // which the devices were added.
-        assert_eq!(
-            block_dev_iter.next().unwrap().config.drive_id,
-            root_block_device.drive_id
-        );
-        assert_eq!(
-            block_dev_iter.next().unwrap().config.drive_id,
-            dummy_block_device_2.drive_id
-        );
-        assert_eq!(
-            block_dev_iter.next().unwrap().config.drive_id,
-            dummy_block_device_3.drive_id
-        );
-    }
-
-    #[test]
-    fn test_block_device_update() {
-        skip_if_not_root!();
-        let dummy_file_1 = TempFile::new().unwrap();
-        let dummy_path_1 = dummy_file_1.as_path().to_owned();
-        let root_block_device = BlockDeviceConfigInfo {
-            path_on_host: dummy_path_1.clone(),
-            device_type: BlockDeviceType::RawBlock,
-            is_root_device: true,
-            part_uuid: None,
-            is_read_only: false,
-            is_direct: false,
-            no_drop: false,
-            drive_id: String::from("1"),
-            rate_limiter: None,
-            num_queues: BlockDeviceConfigInfo::default_num_queues(),
-            queue_size: 128,
-            use_shared_irq: None,
-            use_generic_irq: None,
-        };
-
-        let dummy_file_2 = TempFile::new().unwrap();
-        let dummy_path_2 = dummy_file_2.as_path().to_owned();
-        let mut dummy_block_device_2 = BlockDeviceConfigInfo {
-            path_on_host: dummy_path_2.clone(),
-            device_type: BlockDeviceType::RawBlock,
-            is_root_device: false,
-            part_uuid: None,
-            is_read_only: false,
-            is_direct: false,
-            no_drop: false,
-            drive_id: String::from("2"),
-            rate_limiter: None,
-            num_queues: BlockDeviceConfigInfo::default_num_queues(),
-            queue_size: 128,
-            use_shared_irq: None,
-            use_generic_irq: None,
-        };
-
-        let mut vm = crate::vm::tests::create_vm_instance();
-
-        // Add 2 block devices.
-        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        vm.device_manager_mut()
-            .block_manager
-            .insert_device(ctx, root_block_device)
-            .unwrap();
-        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        vm.device_manager_mut()
-            .block_manager
-            .insert_device(ctx, dummy_block_device_2.clone())
-            .unwrap();
-
-        // Get index zero.
-        assert_eq!(
-            vm.device_manager()
-                .block_manager
-                .get_index_of_drive_id(&String::from("1"))
-                .unwrap(),
-            0
-        );
-
-        // Get None.
-        assert!(vm
-            .device_manager()
-            .block_manager
-            .get_index_of_drive_id(&String::from("foo"))
-            .is_none());
-
-        // Test several update cases using dummy_block_device_2.
-        // Validate `dummy_block_device_2` is already in the list
-        assert!(vm
-            .device_manager()
-            .block_manager
-            .get_index_of_drive_id(&dummy_block_device_2.drive_id)
-            .is_some());
-        // Update OK.
-        dummy_block_device_2.is_read_only = true;
-        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        vm.device_manager_mut()
-            .block_manager
-            .insert_device(ctx, dummy_block_device_2.clone())
-            .unwrap();
-
-        let index = vm
-            .device_manager()
-            .block_manager
-            .get_index_of_drive_id(&dummy_block_device_2.drive_id)
-            .unwrap();
-        // Validate update was successful.
-        assert!(
-            vm.device_manager().block_manager.info_list[index]
-                .config
-                .is_read_only
-        );
-
-        // Update with invalid path.
-        let dummy_filename_3 = String::from("test_update_3");
-        let dummy_path_3 = PathBuf::from(dummy_filename_3);
-        dummy_block_device_2.path_on_host = dummy_path_3;
-        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        assert!(vm
-            .device_manager_mut()
-            .block_manager
-            .insert_device(ctx, dummy_block_device_2.clone(),)
-            .is_err());
-
-        // Update with 2 root block devices.
-        dummy_block_device_2.path_on_host = dummy_path_2.clone();
-        dummy_block_device_2.is_root_device = true;
-        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        assert!(vm
-            .device_manager_mut()
-            .block_manager
-            .insert_device(ctx, dummy_block_device_2,)
-            .is_err(),);
-
-        // Switch roots and add a PARTUUID for the new one.
-        let root_block_device_old = BlockDeviceConfigInfo {
-            path_on_host: dummy_path_1,
-            device_type: BlockDeviceType::RawBlock,
-            is_root_device: false,
-            part_uuid: None,
-            is_read_only: false,
-            is_direct: false,
-            no_drop: false,
-            drive_id: String::from("1"),
-            rate_limiter: None,
-            num_queues: BlockDeviceConfigInfo::default_num_queues(),
-            queue_size: 128,
-            use_shared_irq: None,
-            use_generic_irq: None,
-        };
-        let root_block_device_new = BlockDeviceConfigInfo {
-            path_on_host: dummy_path_2,
-            device_type: BlockDeviceType::RawBlock,
-            is_root_device: true,
-            part_uuid: Some("0eaa91a0-01".to_string()),
-            is_read_only: false,
-            is_direct: false,
-            no_drop: false,
-            drive_id: String::from("2"),
-            rate_limiter: None,
-            num_queues: BlockDeviceConfigInfo::default_num_queues(),
-            queue_size: 128,
-            use_shared_irq: None,
-            use_generic_irq: None,
-        };
-        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        vm.device_manager_mut()
-            .block_manager
-            .insert_device(ctx, root_block_device_old)
-            .unwrap();
-        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        vm.device_manager_mut()
-            .block_manager
-            .insert_device(ctx, root_block_device_new)
-            .unwrap();
-        assert!(vm.device_manager().block_manager.has_part_uuid_root);
-    }
-}

src/dragonball/src/device_manager/mod.rs

@@ -147,13 +147,17 @@ pub type Result<T> = ::std::result::Result<T, DeviceMgrError>;
 /// Type of the dragonball virtio devices.
 #[cfg(feature = "dbs-virtio-devices")]
 pub type DbsVirtioDevice = Box<
-    dyn VirtioDevice<GuestAddressSpaceImpl, virtio_queue::QueueSync, vm_memory::GuestRegionMmap>,
+    dyn VirtioDevice<
+        GuestAddressSpaceImpl,
+        virtio_queue::QueueStateSync,
+        vm_memory::GuestRegionMmap,
+    >,
 >;
 
 /// Type of the dragonball virtio mmio devices.
 #[cfg(feature = "dbs-virtio-devices")]
 pub type DbsMmioV2Device =
-    MmioV2Device<GuestAddressSpaceImpl, virtio_queue::QueueSync, vm_memory::GuestRegionMmap>;
+    MmioV2Device<GuestAddressSpaceImpl, virtio_queue::QueueStateSync, vm_memory::GuestRegionMmap>;
 
 /// Struct to support transactional operations for device management.
 pub struct DeviceManagerTx {
@@ -714,14 +718,6 @@ impl DeviceManager {
 
         #[cfg(feature = "virtio-blk")]
         self.block_manager.remove_devices(&mut ctx)?;
-        // FIXME: To acquire the full abilities for gracefully removing
-        // virtio-net and virtio-vsock devices, updating dragonball-sandbox
-        // is required.
-        #[cfg(feature = "virtio-net")]
-        self.virtio_net_manager.remove_devices(&mut ctx)?;
-        #[cfg(feature = "virtio-vsock")]
-        self.vsock_manager.remove_devices(&mut ctx)?;
-
         Ok(())
     }
 }

src/dragonball/src/device_manager/virtio_net_dev_mgr.rs

@@ -223,7 +223,7 @@ impl VirtioNetDeviceMgr {
 
     /// Insert or update a virtio net device into the manager.
     pub fn insert_device(
-        &mut self,
+        device_mgr: &mut DeviceManager,
         mut ctx: DeviceOpContext,
         config: VirtioNetDeviceConfigInfo,
     ) -> std::result::Result<(), VirtioNetDeviceError> {
@@ -234,6 +234,8 @@ impl VirtioNetDeviceMgr {
             return Err(VirtioNetDeviceError::UpdateNotAllowedPostBoot);
         }
 
+        let mgr = &mut device_mgr.virtio_net_manager;
+
         slog::info!(
             ctx.logger(),
             "add virtio-net device configuration";
@@ -242,7 +244,7 @@ impl VirtioNetDeviceMgr {
             "host_dev_name" => &config.host_dev_name,
         );
 
-        let device_index = self.info_list.insert_or_update(&config)?;
+        let device_index = mgr.info_list.insert_or_update(&config)?;
 
         if ctx.is_hotplug {
             slog::info!(
@@ -258,17 +260,17 @@ impl VirtioNetDeviceMgr {
                     let dev = DeviceManager::create_mmio_virtio_device(
                         device,
                         &mut ctx,
-                        config.use_shared_irq.unwrap_or(self.use_shared_irq),
+                        config.use_shared_irq.unwrap_or(mgr.use_shared_irq),
                         config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ),
                     )
                     .map_err(VirtioNetDeviceError::DeviceManager)?;
                     ctx.insert_hotplug_mmio_device(&dev, None)
                         .map_err(VirtioNetDeviceError::DeviceManager)?;
                     // live-upgrade need save/restore device from info.device.
-                    self.info_list[device_index].set_device(dev);
+                    mgr.info_list[device_index].set_device(dev);
                 }
                 Err(e) => {
-                    self.info_list.remove(device_index);
+                    mgr.info_list.remove(device_index);
                     return Err(VirtioNetDeviceError::Virtio(e));
                 }
             }
@@ -279,15 +281,16 @@ impl VirtioNetDeviceMgr {
 
     /// Update the ratelimiter settings of a virtio net device.
     pub fn update_device_ratelimiters(
-        &mut self,
+        device_mgr: &mut DeviceManager,
         new_cfg: VirtioNetDeviceConfigUpdateInfo,
     ) -> std::result::Result<(), VirtioNetDeviceError> {
-        match self.get_index_of_iface_id(&new_cfg.iface_id) {
+        let mgr = &mut device_mgr.virtio_net_manager;
+        match mgr.get_index_of_iface_id(&new_cfg.iface_id) {
             Some(index) => {
-                let config = &mut self.info_list[index].config;
+                let config = &mut mgr.info_list[index].config;
                 config.rx_rate_limiter = new_cfg.rx_rate_limiter.clone();
                 config.tx_rate_limiter = new_cfg.tx_rate_limiter.clone();
-                let device = self.info_list[index].device.as_mut().ok_or_else(|| {
+                let device = mgr.info_list[index].device.as_mut().ok_or_else(|| {
                     VirtioNetDeviceError::InvalidIfaceId(new_cfg.iface_id.clone())
                 })?;
 
@@ -371,21 +374,6 @@ impl VirtioNetDeviceMgr {
 
         Ok(Box::new(net_device))
     }
-
-    /// Remove all virtio-net devices.
-    pub fn remove_devices(&mut self, ctx: &mut DeviceOpContext) -> Result<(), DeviceMgrError> {
-        while let Some(mut info) = self.info_list.pop() {
-            slog::info!(
-                ctx.logger(),
-                "remove virtio-net device: {}",
-                info.config.iface_id
-            );
-            if let Some(device) = info.device.take() {
-                DeviceManager::destroy_mmio_virtio_device(device, ctx)?;
-            }
-        }
-        Ok(())
-    }
 }
 
 impl Default for VirtioNetDeviceMgr {

src/dragonball/src/device_manager/vsock_dev_mgr.rs

@@ -17,7 +17,7 @@ use dbs_virtio_devices::vsock::Vsock;
 use dbs_virtio_devices::Error as VirtioError;
 use serde_derive::{Deserialize, Serialize};
 
-use super::{DeviceMgrError, StartMicroVmError};
+use super::StartMicroVmError;
 use crate::config_manager::{ConfigItem, DeviceConfigInfo, DeviceConfigInfos};
 use crate::device_manager::{DeviceManager, DeviceOpContext};
 
@@ -284,21 +284,6 @@ impl VsockDeviceMgr {
         // safe to unwrap, because we created the inner connector before
         Ok(self.default_inner_connector.clone().unwrap())
     }
-
-    /// Remove all virtio-vsock devices
-    pub fn remove_devices(&mut self, ctx: &mut DeviceOpContext) -> Result<(), DeviceMgrError> {
-        while let Some(mut info) = self.info_list.pop() {
-            slog::info!(
-                ctx.logger(),
-                "remove virtio-vsock device: {}",
-                info.config.id
-            );
-            if let Some(device) = info.device.take() {
-                DeviceManager::destroy_mmio_virtio_device(device, ctx)?;
-            }
-        }
-        Ok(())
-    }
 }
 
 impl Default for VsockDeviceMgr {

src/dragonball/src/error.rs

@@ -9,8 +9,6 @@
 
 //! Error codes for the virtual machine monitor subsystem.
 
-#[cfg(target_arch = "aarch64")]
-use dbs_arch::pmu::PmuError;
 #[cfg(feature = "dbs-virtio-devices")]
 use dbs_virtio_devices::Error as VirtIoError;
 
@@ -63,11 +61,6 @@ pub enum Error {
     #[error("failed to write MP table to guest memory: {0}")]
     MpTableSetup(#[source] dbs_boot::mptable::Error),
 
-    /// Create pmu device error
-    #[cfg(target_arch = "aarch64")]
-    #[error("Create pmu device error: {0}")]
-    PmuDeviceError(#[source] PmuError),
-
     /// Fail to boot system
     #[error("failed to boot system: {0}")]
     BootSystem(#[source] dbs_boot::Error),
@@ -75,10 +68,6 @@ pub enum Error {
     /// Cannot open the VM file descriptor.
     #[error(transparent)]
     Vm(vm::VmError),
-
-    /// confidential vm type Error
-    #[error("confidential-vm-type can only be used in x86_64 now")]
-    ConfidentialVmType,
 }
 
 /// Errors associated with starting the instance.

src/dragonball/src/kvm_context.rs

@@ -10,7 +10,7 @@ use kvm_bindings::KVM_API_VERSION;
 use kvm_ioctls::{Cap, Kvm, VmFd};
 use std::os::unix::io::{FromRawFd, RawFd};
 
-use crate::error::{Error as VmError, Result};
+use crate::error::{Error, Result};
 
 /// Describes a KVM context that gets attached to the micro VM instance.
 /// It gives access to the functionality of the KVM wrapper as long as every required
@@ -29,11 +29,11 @@ impl KvmContext {
             // Safe because we expect kvm_fd to contain a valid fd number when is_some() == true.
             unsafe { Kvm::from_raw_fd(fd) }
         } else {
-            Kvm::new().map_err(VmError::Kvm)?
+            Kvm::new().map_err(Error::Kvm)?
         };
 
         if kvm.get_api_version() != KVM_API_VERSION as i32 {
-            return Err(VmError::KvmApiVersion(kvm.get_api_version()));
+            return Err(Error::KvmApiVersion(kvm.get_api_version()));
         }
 
         Self::check_cap(&kvm, Cap::Irqchip)?;
@@ -44,8 +44,7 @@ impl KvmContext {
         Self::check_cap(&kvm, Cap::SetTssAddr)?;
 
         #[cfg(target_arch = "x86_64")]
-        let supported_msrs =
-            dbs_arch::msr::supported_guest_msrs(&kvm).map_err(VmError::GuestMSRs)?;
+        let supported_msrs = dbs_arch::msr::supported_guest_msrs(&kvm).map_err(Error::GuestMSRs)?;
         let max_memslots = kvm.get_nr_memslots();
 
         Ok(KvmContext {
@@ -68,7 +67,7 @@ impl KvmContext {
 
     /// Create a virtual machine object.
     pub fn create_vm(&self) -> Result<VmFd> {
-        self.kvm.create_vm().map_err(VmError::Kvm)
+        self.kvm.create_vm().map_err(Error::Kvm)
     }
 
     /// Get the max vcpu count supported by kvm
@@ -76,9 +75,9 @@ impl KvmContext {
         self.kvm.get_max_vcpus()
     }
 
-    fn check_cap(kvm: &Kvm, cap: Cap) -> std::result::Result<(), VmError> {
+    fn check_cap(kvm: &Kvm, cap: Cap) -> std::result::Result<(), Error> {
         if !kvm.check_extension(cap) {
-            return Err(VmError::KvmCap(cap));
+            return Err(Error::KvmCap(cap));
         }
         Ok(())
     }
@@ -92,18 +91,6 @@ mod x86_64 {
     use std::collections::HashSet;
 
     impl KvmContext {
-        /// Create a virtual machine object with specific type.
-        /// vm_type: u64
-        ///      0: legacy vm
-        ///      2: tdx vm
-        pub fn create_vm_with_type(&self, vm_type: u64) -> Result<VmFd> {
-            let fd = self
-                .kvm
-                .create_vm_with_type(vm_type)
-                .map_err(VmError::Kvm)?;
-            Ok(fd)
-        }
-
         /// Get information about supported CPUID of x86 processor.
         pub fn supported_cpuid(
             &self,
@@ -123,7 +110,7 @@ mod x86_64 {
         // It's very sensible to manipulate MSRs, so please be careful to change code below.
         fn build_msrs_list(kvm: &Kvm) -> Result<Msrs> {
             let mut mset: HashSet<u32> = HashSet::new();
-            let supported_msr_list = kvm.get_msr_index_list().map_err(VmError::Kvm)?;
+            let supported_msr_list = kvm.get_msr_index_list().map_err(super::Error::Kvm)?;
             for msr in supported_msr_list.as_slice() {
                 mset.insert(*msr);
             }
@@ -216,7 +203,7 @@ mod x86_64 {
                 })
                 .collect();
 
-            Msrs::from_entries(&msrs).map_err(VmError::Msr)
+            Msrs::from_entries(&msrs).map_err(super::Error::Msr)
         }
     }
 }
@@ -270,20 +257,4 @@ mod tests {
 
         let _ = c.create_vm().unwrap();
     }
-
-    #[test]
-    fn test_create_vm_with_type() {
-        let c = KvmContext::new(None).unwrap();
-        #[cfg(not(target_arch = "aarch64"))]
-        let _ = c.create_vm_with_type(0_u64).unwrap();
-        #[cfg(target_arch = "aarch64")]
-        {
-            /// aarch64 is using ipa_size to create vm
-            let mut ipa_size = 0; // Create using default VM type
-            if c.check_extension(kvm_ioctls::Cap::ArmVmIPASize) {
-                ipa_size = c.kvm.get_host_ipa_limit();
-            }
-            let _ = c.create_vm_with_type(ipa_size as u64).unwrap();
-        }
-    }
 }

src/dragonball/src/vcpu/aarch64.rs

@@ -11,7 +11,7 @@ use std::sync::mpsc::{channel, Sender};
 use std::sync::Arc;
 
 use crate::IoManagerCached;
-use dbs_arch::{regs, VpmuFeatureLevel};
+use dbs_arch::regs;
 use dbs_boot::get_fdt_addr;
 use dbs_utils::time::TimestampUs;
 use kvm_ioctls::{VcpuFd, VmFd};
@@ -81,7 +81,7 @@ impl Vcpu {
     /// * `_pgtable_addr` - pgtable address for ap vcpu (not used in aarch64)
     pub fn configure(
         &mut self,
-        vcpu_config: &VcpuConfig,
+        _vcpu_config: &VcpuConfig,
         vm_fd: &VmFd,
         vm_as: &GuestAddressSpaceImpl,
         kernel_load_addr: Option<GuestAddress>,
@@ -99,9 +99,6 @@ impl Vcpu {
         if self.id > 0 {
             kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_POWER_OFF;
         }
-        if vcpu_config.vpmu_feature == VpmuFeatureLevel::FullyEnabled {
-            kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_PMU_V3;
-        }
 
         self.fd.vcpu_init(&kvi).map_err(VcpuError::VcpuArmInit)?;
 

src/dragonball/src/vcpu/mod.rs

@@ -7,7 +7,9 @@ mod sm;
 mod vcpu_impl;
 mod vcpu_manager;
 
-use dbs_arch::VpmuFeatureLevel;
+#[cfg(target_arch = "x86_64")]
+use dbs_arch::cpuid::VpmuFeatureLevel;
+
 pub use vcpu_manager::{VcpuManager, VcpuManagerError, VcpuResizeInfo};
 
 #[cfg(feature = "hotplug")]
@@ -30,6 +32,6 @@ pub struct VcpuConfig {
     /// if vpmu feature is Disabled, it means vpmu feature is off (by default)
     /// if vpmu feature is LimitedlyEnabled, it means minimal vpmu counters are supported (cycles and instructions)
     /// if vpmu feature is FullyEnabled, it means all vpmu counters are supported
-    /// For aarch64, VpmuFeatureLevel only supports Disabled and FullyEnabled.
+    #[cfg(target_arch = "x86_64")]
     pub vpmu_feature: VpmuFeatureLevel,
 }

src/dragonball/src/vcpu/vcpu_impl.rs

@@ -441,77 +441,75 @@ impl Vcpu {
     /// Returns error or enum specifying whether emulation was handled or interrupted.
     fn run_emulation(&mut self) -> Result<VcpuEmulation> {
         match Vcpu::emulate(&self.fd) {
-            Ok(run) => {
-                match run {
-                    #[cfg(target_arch = "x86_64")]
-                    VcpuExit::IoIn(addr, data) => {
-                        let _ = self.io_mgr.pio_read(addr, data);
-                        METRICS.vcpu.exit_io_in.inc();
-                        Ok(VcpuEmulation::Handled)
-                    }
-                    #[cfg(target_arch = "x86_64")]
-                    VcpuExit::IoOut(addr, data) => {
-                        if !self.check_io_port_info(addr, data)? {
-                            let _ = self.io_mgr.pio_write(addr, data);
-                        }
-                        METRICS.vcpu.exit_io_out.inc();
-                        Ok(VcpuEmulation::Handled)
-                    }
-                    VcpuExit::MmioRead(addr, data) => {
-                        let _ = self.io_mgr.mmio_read(addr, data);
-                        METRICS.vcpu.exit_mmio_read.inc();
-                        Ok(VcpuEmulation::Handled)
-                    }
-                    VcpuExit::MmioWrite(addr, data) => {
-                        let _ = self.io_mgr.mmio_write(addr, data);
-                        METRICS.vcpu.exit_mmio_write.inc();
-                        Ok(VcpuEmulation::Handled)
-                    }
-                    VcpuExit::Hlt => {
-                        info!("Received KVM_EXIT_HLT signal");
-                        Err(VcpuError::VcpuUnhandledKvmExit)
-                    }
-                    VcpuExit::Shutdown => {
-                        info!("Received KVM_EXIT_SHUTDOWN signal");
-                        Err(VcpuError::VcpuUnhandledKvmExit)
-                    }
-                    // Documentation specifies that below kvm exits are considered errors.
-                    VcpuExit::FailEntry(reason, cpu) => {
-                        METRICS.vcpu.failures.inc();
-                        error!("Received KVM_EXIT_FAIL_ENTRY signal, reason {reason}, cpu number {cpu}");
-                        Err(VcpuError::VcpuUnhandledKvmExit)
-                    }
-                    VcpuExit::InternalError => {
-                        METRICS.vcpu.failures.inc();
-                        error!("Received KVM_EXIT_INTERNAL_ERROR signal");
-                        Err(VcpuError::VcpuUnhandledKvmExit)
-                    }
-                    VcpuExit::SystemEvent(event_type, event_flags) => match event_type {
-                        KVM_SYSTEM_EVENT_RESET | KVM_SYSTEM_EVENT_SHUTDOWN => {
-                            info!(
-                                "Received KVM_SYSTEM_EVENT: type: {}, event: {}",
-                                event_type, event_flags
-                            );
-                            Ok(VcpuEmulation::Stopped)
-                        }
-                        _ => {
-                            METRICS.vcpu.failures.inc();
-                            error!(
-                                "Received KVM_SYSTEM_EVENT signal type: {}, flag: {}",
-                                event_type, event_flags
-                            );
-                            Err(VcpuError::VcpuUnhandledKvmExit)
-                        }
-                    },
-                    r => {
-                        METRICS.vcpu.failures.inc();
-                        // TODO: Are we sure we want to finish running a vcpu upon
-                        // receiving a vm exit that is not necessarily an error?
-                        error!("Unexpected exit reason on vcpu run: {:?}", r);
-                        Err(VcpuError::VcpuUnhandledKvmExit)
-                    }
+            Ok(run) => match run {
+                #[cfg(target_arch = "x86_64")]
+                VcpuExit::IoIn(addr, data) => {
+                    let _ = self.io_mgr.pio_read(addr, data);
+                    METRICS.vcpu.exit_io_in.inc();
+                    Ok(VcpuEmulation::Handled)
                 }
-            }
+                #[cfg(target_arch = "x86_64")]
+                VcpuExit::IoOut(addr, data) => {
+                    if !self.check_io_port_info(addr, data)? {
+                        let _ = self.io_mgr.pio_write(addr, data);
+                    }
+                    METRICS.vcpu.exit_io_out.inc();
+                    Ok(VcpuEmulation::Handled)
+                }
+                VcpuExit::MmioRead(addr, data) => {
+                    let _ = self.io_mgr.mmio_read(addr, data);
+                    METRICS.vcpu.exit_mmio_read.inc();
+                    Ok(VcpuEmulation::Handled)
+                }
+                VcpuExit::MmioWrite(addr, data) => {
+                    let _ = self.io_mgr.mmio_write(addr, data);
+                    METRICS.vcpu.exit_mmio_write.inc();
+                    Ok(VcpuEmulation::Handled)
+                }
+                VcpuExit::Hlt => {
+                    info!("Received KVM_EXIT_HLT signal");
+                    Err(VcpuError::VcpuUnhandledKvmExit)
+                }
+                VcpuExit::Shutdown => {
+                    info!("Received KVM_EXIT_SHUTDOWN signal");
+                    Err(VcpuError::VcpuUnhandledKvmExit)
+                }
+                // Documentation specifies that below kvm exits are considered errors.
+                VcpuExit::FailEntry => {
+                    METRICS.vcpu.failures.inc();
+                    error!("Received KVM_EXIT_FAIL_ENTRY signal");
+                    Err(VcpuError::VcpuUnhandledKvmExit)
+                }
+                VcpuExit::InternalError => {
+                    METRICS.vcpu.failures.inc();
+                    error!("Received KVM_EXIT_INTERNAL_ERROR signal");
+                    Err(VcpuError::VcpuUnhandledKvmExit)
+                }
+                VcpuExit::SystemEvent(event_type, event_flags) => match event_type {
+                    KVM_SYSTEM_EVENT_RESET | KVM_SYSTEM_EVENT_SHUTDOWN => {
+                        info!(
+                            "Received KVM_SYSTEM_EVENT: type: {}, event: {}",
+                            event_type, event_flags
+                        );
+                        Ok(VcpuEmulation::Stopped)
+                    }
+                    _ => {
+                        METRICS.vcpu.failures.inc();
+                        error!(
+                            "Received KVM_SYSTEM_EVENT signal type: {}, flag: {}",
+                            event_type, event_flags
+                        );
+                        Err(VcpuError::VcpuUnhandledKvmExit)
+                    }
+                },
+                r => {
+                    METRICS.vcpu.failures.inc();
+                    // TODO: Are we sure we want to finish running a vcpu upon
+                    // receiving a vm exit that is not necessarily an error?
+                    error!("Unexpected exit reason on vcpu run: {:?}", r);
+                    Err(VcpuError::VcpuUnhandledKvmExit)
+                }
+            },
             // The unwrap on raw_os_error can only fail if we have a logic
             // error in our code in which case it is better to panic.
             Err(ref e) => {
@@ -760,11 +758,6 @@ impl Vcpu {
         // State machine reached its end.
         StateMachine::finish(Self::exited)
     }
-
-    /// Get vcpu file descriptor.
-    pub fn vcpu_fd(&self) -> &VcpuFd {
-        self.fd.as_ref()
-    }
 }
 
 impl Drop for Vcpu {
@@ -793,7 +786,7 @@ pub mod tests {
         MmioWrite,
         Hlt,
         Shutdown,
-        FailEntry(u64, u32),
+        FailEntry,
         InternalError,
         Unknown,
         SystemEvent(u32, u64),
@@ -814,9 +807,7 @@ pub mod tests {
                 EmulationCase::MmioWrite => Ok(VcpuExit::MmioWrite(0, &[])),
                 EmulationCase::Hlt => Ok(VcpuExit::Hlt),
                 EmulationCase::Shutdown => Ok(VcpuExit::Shutdown),
-                EmulationCase::FailEntry(error_type, cpu_num) => {
-                    Ok(VcpuExit::FailEntry(*error_type, *cpu_num))
-                }
+                EmulationCase::FailEntry => Ok(VcpuExit::FailEntry),
                 EmulationCase::InternalError => Ok(VcpuExit::InternalError),
                 EmulationCase::Unknown => Ok(VcpuExit::Unknown),
                 EmulationCase::SystemEvent(event_type, event_flags) => {
@@ -859,8 +850,6 @@ pub mod tests {
 
     #[cfg(target_arch = "aarch64")]
     fn create_vcpu() -> (Vcpu, Receiver<VcpuStateEvent>) {
-        use kvm_ioctls::Kvm;
-        use std::os::fd::AsRawFd;
         // Call for kvm too frequently would cause error in some host kernel.
         std::thread::sleep(std::time::Duration::from_millis(5));
 
@@ -929,7 +918,7 @@ pub mod tests {
         assert!(matches!(res, Err(VcpuError::VcpuUnhandledKvmExit)));
 
         // KVM_EXIT_FAIL_ENTRY signal
-        *(EMULATE_RES.lock().unwrap()) = EmulationCase::FailEntry(0, 0);
+        *(EMULATE_RES.lock().unwrap()) = EmulationCase::FailEntry;
         let res = vcpu.run_emulation();
         assert!(matches!(res, Err(VcpuError::VcpuUnhandledKvmExit)));
 

src/dragonball/src/vcpu/vcpu_manager.rs

@@ -15,7 +15,6 @@ use std::sync::mpsc::{channel, Receiver, RecvError, RecvTimeoutError, Sender};
 use std::sync::{Arc, Barrier, Mutex, RwLock};
 use std::time::Duration;
 
-use dbs_arch::VpmuFeatureLevel;
 #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))]
 use dbs_upcall::{DevMgrService, UpcallClient};
 use dbs_utils::epoll_manager::{EpollManager, EventOps, EventSet, Events, MutEventSubscriber};
@@ -282,20 +281,11 @@ impl VcpuManager {
         let supported_cpuid = kvm_context
             .supported_cpuid(kvm_bindings::KVM_MAX_CPUID_ENTRIES)
             .map_err(VcpuManagerError::Kvm)?;
-        #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))]
+        #[cfg(target_arch = "x86_64")]
         let vpmu_feature_level = match vm_config_info.vpmu_feature {
-            #[cfg(target_arch = "x86_64")]
-            1 => VpmuFeatureLevel::LimitedlyEnabled,
-            #[cfg(target_arch = "aarch64")]
-            1 => {
-                log::warn!(
-                    "Limitedly enabled vpmu feature isn't supported on aarch64 for now.\
-                       This will be supported in the future. The vpmu_feature will be set disabled!"
-                );
-                VpmuFeatureLevel::Disabled
-            }
-            2 => VpmuFeatureLevel::FullyEnabled,
-            _ => VpmuFeatureLevel::Disabled,
+            1 => dbs_arch::cpuid::VpmuFeatureLevel::LimitedlyEnabled,
+            2 => dbs_arch::cpuid::VpmuFeatureLevel::FullyEnabled,
+            _ => dbs_arch::cpuid::VpmuFeatureLevel::Disabled,
         };
 
         let vcpu_manager = Arc::new(Mutex::new(VcpuManager {
@@ -307,6 +297,7 @@ impl VcpuManager {
                 cores_per_die: vm_config_info.cpu_topology.cores_per_die,
                 dies_per_socket: vm_config_info.cpu_topology.dies_per_socket,
                 sockets: vm_config_info.cpu_topology.sockets,
+                #[cfg(target_arch = "x86_64")]
                 vpmu_feature: vpmu_feature_level,
             },
             vcpu_seccomp_filter,
@@ -808,11 +799,6 @@ impl VcpuManager {
         )
         .map_err(VcpuManagerError::Vcpu)
     }
-
-    /// get vpmu_feature config
-    pub fn vpmu_feature(&self) -> VpmuFeatureLevel {
-        self.vcpu_config.vpmu_feature
-    }
 }
 
 #[cfg(feature = "hotplug")]
@@ -901,9 +887,7 @@ mod hotplug {
             cpu_ids_array[..cpu_ids.len()].copy_from_slice(&cpu_ids[..cpu_ids.len()]);
             let req = DevMgrRequest::AddVcpu(CpuDevRequest {
                 count: cpu_ids.len() as u8,
-                #[cfg(target_arch = "x86_64")]
                 apic_ids: cpu_ids_array,
-                #[cfg(target_arch = "x86_64")]
                 apic_ver: APIC_VERSION,
             });
             self.send_upcall_action(upcall_client, req)?;
@@ -940,9 +924,7 @@ mod hotplug {
             cpu_ids_array[..cpu_ids.len()].copy_from_slice(&cpu_ids[..cpu_ids.len()]);
             let req = DevMgrRequest::DelVcpu(CpuDevRequest {
                 count: cpu_num_to_be_del as u8,
-                #[cfg(target_arch = "x86_64")]
                 apic_ids: cpu_ids_array,
-                #[cfg(target_arch = "x86_64")]
                 apic_ver: APIC_VERSION,
             });
             self.send_upcall_action(upcall_client, req)?;
@@ -987,10 +969,7 @@ mod hotplug {
                                 vcpu_state_sender
                                     .send(VcpuStateEvent::Hotplug((
                                         result,
-                                        #[cfg(target_arch = "x86_64")]
                                         resp.info.apic_id_index,
-                                        #[cfg(target_arch = "aarch64")]
-                                        resp.info.cpu_id,
                                     )))
                                     .unwrap();
                                 vcpu_state_event.write(1).unwrap();

src/dragonball/src/vm/aarch64.rs

@@ -11,8 +11,7 @@ use std::fmt::Debug;
 use std::ops::Deref;
 
 use dbs_arch::gic::GICDevice;
-use dbs_arch::pmu::initialize_pmu;
-use dbs_arch::{DeviceInfoForFDT, DeviceType, VpmuFeatureLevel};
+use dbs_arch::{DeviceInfoForFDT, DeviceType};
 use dbs_boot::InitrdConfig;
 use dbs_utils::epoll_manager::EpollManager;
 use dbs_utils::time::TimestampUs;
@@ -44,7 +43,6 @@ fn configure_system<T: DeviceInfoForFDT + Clone + Debug, M: GuestMemory>(
     device_info: Option<&HashMap<(DeviceType, String), T>>,
     gic_device: &Box<dyn GICDevice>,
     initrd: &Option<super::InitrdConfig>,
-    vpmu_feature: &VpmuFeatureLevel,
 ) -> super::Result<()> {
     dbs_boot::fdt::create_fdt(
         guest_mem,
@@ -53,7 +51,6 @@ fn configure_system<T: DeviceInfoForFDT + Clone + Debug, M: GuestMemory>(
         device_info,
         gic_device,
         initrd,
-        vpmu_feature,
     )
     .map_err(Error::BootSystem)?;
     Ok(())
@@ -79,23 +76,6 @@ impl Vm {
         Ok(())
     }
 
-    /// Setup pmu devices for guest vm.
-    pub fn setup_pmu_devices(&mut self) -> std::result::Result<(), StartMicroVmError> {
-        let vm = self.vm_fd();
-        let mut vcpu_manager = self.vcpu_manager().map_err(StartMicroVmError::Vcpu)?;
-        let vpmu_feature = vcpu_manager.vpmu_feature();
-        if vpmu_feature == VpmuFeatureLevel::Disabled {
-            return Ok(());
-        }
-
-        for vcpu in vcpu_manager.vcpus_mut() {
-            initialize_pmu(vm, vcpu.vcpu_fd())
-                .map_err(|e| StartMicroVmError::ConfigureVm(VmError::SetupPmu(e)))?;
-        }
-
-        Ok(())
-    }
-
     /// Initialize the virtual machine instance.
     ///
     /// It initialize the virtual machine instance by:
@@ -133,7 +113,6 @@ impl Vm {
             .create_boot_vcpus(request_ts, kernel_loader_result.kernel_load)
             .map_err(StartMicroVmError::Vcpu)?;
         self.setup_interrupt_controller()?;
-        self.setup_pmu_devices()?;
         self.init_devices(epoll_mgr)?;
 
         Ok(())
@@ -150,7 +129,6 @@ impl Vm {
         initrd: Option<InitrdConfig>,
     ) -> std::result::Result<(), StartMicroVmError> {
         let vcpu_manager = self.vcpu_manager().map_err(StartMicroVmError::Vcpu)?;
-        let vpmu_feature = vcpu_manager.vpmu_feature();
         let vcpu_mpidr = vcpu_manager
             .vcpus()
             .into_iter()
@@ -165,7 +143,6 @@ impl Vm {
             self.device_manager.get_mmio_device_info(),
             self.get_irqchip(),
             &initrd,
-            &vpmu_feature,
         )
         .map_err(StartMicroVmError::ConfigureSystem)
     }

src/dragonball/src/vm/mod.rs

@@ -10,8 +10,6 @@ use std::sync::{Arc, Mutex, RwLock};
 use dbs_address_space::AddressSpace;
 #[cfg(target_arch = "aarch64")]
 use dbs_arch::gic::GICDevice;
-#[cfg(target_arch = "aarch64")]
-use dbs_arch::pmu::PmuError;
 use dbs_boot::InitrdConfig;
 use dbs_utils::epoll_manager::EpollManager;
 use dbs_utils::time::TimestampUs;
@@ -71,11 +69,6 @@ pub enum VmError {
     #[cfg(target_arch = "aarch64")]
     #[error("failed to configure GIC")]
     SetupGIC(GICError),
-
-    /// Cannot setup pmu device
-    #[cfg(target_arch = "aarch64")]
-    #[error("failed to setup pmu device")]
-    SetupPmu(#[source] PmuError),
 }
 
 /// Configuration information for user defined NUMA nodes.
@@ -215,26 +208,7 @@ impl Vm {
         let id = api_shared_info.read().unwrap().id.clone();
         let logger = slog_scope::logger().new(slog::o!("id" => id));
         let kvm = KvmContext::new(kvm_fd)?;
-        let vm_fd = match api_shared_info
-            .as_ref()
-            .read()
-            .unwrap()
-            .confidential_vm_type
-        {
-            None => Arc::new(kvm.create_vm()?),
-            Some(confidential_vm_type) => {
-                #[cfg(not(any(target_arch = "x86_64")))]
-                {
-                    error!(
-                        "confidential-vm-type {} only can be used in x86_64",
-                        confidential_vm_type as u64
-                    );
-                    return Err(Error::ConfidentialVmType);
-                }
-                #[cfg(target_arch = "x86_64")]
-                Arc::new(kvm.create_vm_with_type(confidential_vm_type as u64)?)
-            }
-        };
+        let vm_fd = Arc::new(kvm.create_vm()?);
         let resource_manager = Arc::new(ResourceManager::new(Some(kvm.max_memslots())));
         let device_manager = DeviceManager::new(
             vm_fd.clone(),
@@ -362,15 +336,6 @@ impl Vm {
         instance_state == InstanceState::Running
     }
 
-    /// return true if VM confidential type is TDX
-    pub fn is_tdx_enabled(&self) -> bool {
-        let shared_info = self
-            .shared_info()
-            .read()
-            .expect("failed to get instance state, because shared info is poisoned lock");
-        shared_info.is_tdx_enabled()
-    }
-
     /// Save VM instance exit state
     pub fn vm_exit(&self, exit_code: i32) {
         if let Ok(mut info) = self.shared_info.write() {
@@ -1040,7 +1005,6 @@ pub mod tests {
         assert!(vm.remove_devices().is_ok());
     }
 
-    #[cfg(target_arch = "x86_64")]
     #[test]
     fn test_run_code() {
         skip_if_not_root!();

src/dragonball/src/vm/x86_64.rs

@@ -7,6 +7,7 @@
 // found in the THIRD-PARTY file.
 
 use std::convert::TryInto;
+use std::mem;
 use std::ops::Deref;
 
 use dbs_address_space::AddressSpace;
@@ -15,9 +16,8 @@ use dbs_utils::epoll_manager::EpollManager;
 use dbs_utils::time::TimestampUs;
 use kvm_bindings::{kvm_irqchip, kvm_pit_config, kvm_pit_state2, KVM_PIT_SPEAKER_DUMMY};
 use linux_loader::cmdline::Cmdline;
-use linux_loader::configurator::{linux::LinuxBootConfigurator, BootConfigurator, BootParams};
 use slog::info;
-use vm_memory::{Address, GuestAddress, GuestAddressSpace, GuestMemory};
+use vm_memory::{Address, Bytes, GuestAddress, GuestAddressSpace, GuestMemory};
 
 use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl};
 use crate::error::{Error, Result, StartMicroVmError};
@@ -110,11 +110,15 @@ fn configure_system<M: GuestMemory>(
         }
     }
 
-    LinuxBootConfigurator::write_bootparams(
-        &BootParams::new(&params, GuestAddress(layout::ZERO_PAGE_START)),
-        guest_mem,
-    )
-    .map_err(|_| Error::ZeroPageSetup)
+    let zero_page_addr = GuestAddress(layout::ZERO_PAGE_START);
+    guest_mem
+        .checked_offset(zero_page_addr, mem::size_of::<bootparam::boot_params>())
+        .ok_or(Error::ZeroPagePastRamEnd)?;
+    guest_mem
+        .write_obj(params, zero_page_addr)
+        .map_err(|_| Error::ZeroPageSetup)?;
+
+    Ok(())
 }
 
 impl Vm {

src/libs/Cargo.lock

@@ -186,7 +186,7 @@ checksum = "ec3245a0ca564e7f3c797d20d833a6870f57a728ac967d5225b3ffdef4465011"
 dependencies = [
  "lazy_static",
  "log",
- "rand",
+ "rand 0.8.5",
 ]
 
 [[package]]
@@ -299,6 +299,17 @@ dependencies = [
  "slab",
 ]
 
+[[package]]
+name = "getrandom"
+version = "0.1.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi 0.9.0+wasi-snapshot-preview1",
+]
+
 [[package]]
 name = "getrandom"
 version = "0.2.6"
@@ -484,7 +495,7 @@ dependencies = [
  "num_cpus",
  "oci",
  "once_cell",
- "rand",
+ "rand 0.7.3",
  "serde_json",
  "serial_test",
  "slog",
@@ -692,9 +703,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.17.1"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
+checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5"
 
 [[package]]
 name = "parking_lot"
@@ -834,16 +845,9 @@ name = "protobuf"
 version = "2.27.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96"
-
-[[package]]
-name = "protobuf"
-version = "3.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b55bad9126f378a853655831eb7363b7b01b81d19f8cb1218861086ca4a1a61e"
 dependencies = [
- "once_cell",
- "protobuf-support",
- "thiserror",
+ "serde",
+ "serde_derive",
 ]
 
 [[package]]
@@ -852,47 +856,17 @@ version = "2.27.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "aec1632b7c8f2e620343439a7dfd1f3c47b18906c4be58982079911482b5d707"
 dependencies = [
- "protobuf 2.27.1",
+ "protobuf",
 ]
 
 [[package]]
-name = "protobuf-codegen"
-version = "3.2.0"
+name = "protobuf-codegen-pure"
+version = "2.27.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0dd418ac3c91caa4032d37cb80ff0d44e2ebe637b2fb243b6234bf89cdac4901"
+checksum = "9f8122fdb18e55190c796b088a16bdb70cd7acdcd48f7a8b796b58c62e532cc6"
 dependencies = [
- "anyhow",
- "once_cell",
- "protobuf 3.2.0",
- "protobuf-parse",
- "regex",
- "tempfile",
- "thiserror",
-]
-
-[[package]]
-name = "protobuf-parse"
-version = "3.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d39b14605eaa1f6a340aec7f320b34064feb26c93aec35d6a9a2272a8ddfa49"
-dependencies = [
- "anyhow",
- "indexmap",
- "log",
- "protobuf 3.2.0",
- "protobuf-support",
- "tempfile",
- "thiserror",
- "which",
-]
-
-[[package]]
-name = "protobuf-support"
-version = "3.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a5d4d7b8601c814cfb36bcebb79f0e61e45e1e93640cf778837833bbed05c372"
-dependencies = [
- "thiserror",
+ "protobuf",
+ "protobuf-codegen",
 ]
 
 [[package]]
@@ -901,7 +875,7 @@ version = "0.1.0"
 dependencies = [
  "async-trait",
  "oci",
- "protobuf 3.2.0",
+ "protobuf",
  "serde",
  "serde_json",
  "ttrpc",
@@ -917,6 +891,19 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "rand"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
+dependencies = [
+ "getrandom 0.1.16",
+ "libc",
+ "rand_chacha 0.2.2",
+ "rand_core 0.5.1",
+ "rand_hc",
+]
+
 [[package]]
 name = "rand"
 version = "0.8.5"
@@ -924,8 +911,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
 dependencies = [
  "libc",
- "rand_chacha",
- "rand_core",
+ "rand_chacha 0.3.1",
+ "rand_core 0.6.3",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.5.1",
 ]
 
 [[package]]
@@ -935,7 +932,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
 dependencies = [
  "ppv-lite86",
- "rand_core",
+ "rand_core 0.6.3",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
+dependencies = [
+ "getrandom 0.1.16",
 ]
 
 [[package]]
@@ -944,7 +950,16 @@ version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
 dependencies = [
- "getrandom",
+ "getrandom 0.2.6",
+]
+
+[[package]]
+name = "rand_hc"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
+dependencies = [
+ "rand_core 0.5.1",
 ]
 
 [[package]]
@@ -1299,9 +1314,9 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
 
 [[package]]
 name = "ttrpc"
-version = "0.7.1"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a35f22a2964bea14afee161665bb260b83cb48e665e0260ca06ec0e775c8b06c"
+checksum = "2ecfff459a859c6ba6668ff72b34c2f1d94d9d58f7088414c2674ad0f31cc7d8"
 dependencies = [
  "async-trait",
  "byteorder",
@@ -1309,8 +1324,8 @@ dependencies = [
  "libc",
  "log",
  "nix 0.23.1",
- "protobuf 3.2.0",
- "protobuf-codegen 3.2.0",
+ "protobuf",
+ "protobuf-codegen-pure",
  "thiserror",
  "tokio",
  "tokio-vsock",
@@ -1318,28 +1333,28 @@ dependencies = [
 
 [[package]]
 name = "ttrpc-codegen"
-version = "0.4.2"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94d7f7631d7a9ebed715a47cd4cb6072cbc7ae1d4ec01598971bbec0024340c2"
+checksum = "809eda4e459820237104e4b61d6b41bbe6c9e1ce6adf4057955e6e6722a90408"
 dependencies = [
- "protobuf 2.27.1",
- "protobuf-codegen 3.2.0",
- "protobuf-support",
+ "protobuf",
+ "protobuf-codegen",
+ "protobuf-codegen-pure",
  "ttrpc-compiler",
 ]
 
 [[package]]
 name = "ttrpc-compiler"
-version = "0.6.1"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec3cb5dbf1f0865a34fe3f722290fe776cacb16f50428610b779467b76ddf647"
+checksum = "2978ed3fa047d8fd55cbeb4d4a61d461fb3021a90c9618519c73ce7e5bb66c15"
 dependencies = [
  "derive-new",
  "prost",
  "prost-build",
  "prost-types",
- "protobuf 2.27.1",
- "protobuf-codegen 2.27.1",
+ "protobuf",
+ "protobuf-codegen",
  "tempfile",
 ]
 
@@ -1375,6 +1390,12 @@ dependencies = [
  "try-lock",
 ]
 
+[[package]]
+name = "wasi"
+version = "0.9.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
+
 [[package]]
 name = "wasi"
 version = "0.10.2+wasi-snapshot-preview1"

src/libs/kata-sys-util/Cargo.toml

@@ -24,7 +24,7 @@ serde_json = "1.0.73"
 slog = "2.5.2"
 slog-scope = "4.4.0"
 subprocess = "0.2.8"
-rand = "0.8.5"
+rand = "0.7.2"
 thiserror = "1.0.30"
 
 kata-types = { path = "../kata-types" }

src/libs/kata-types/src/annotations/mod.rs

@@ -308,14 +308,6 @@ pub const KATA_ANNO_CFG_DISABLE_NEW_NETNS: &str =
 /// A sandbox annotation to specify how attached VFIO devices should be treated.
 pub const KATA_ANNO_CFG_VFIO_MODE: &str = "io.katacontainers.config.runtime.vfio_mode";
 
-/// A sandbox annotation used to specify prefetch_files.list host path container image
-/// being used,
-/// and runtime will pass it to Hypervisor to  search for corresponding prefetch list file.
-/// "io.katacontainers.config.hypervisor.prefetch_files.list"
-///                                = "/path/to/<uid>/xyz.com/fedora:36/prefetch_file.list"
-pub const KATA_ANNO_CFG_HYPERVISOR_PREFETCH_FILES_LIST: &str =
-    "io.katacontainers.config.hypervisor.prefetch_files.list";
-
 /// A helper structure to query configuration information by check annotations.
 #[derive(Debug, Default, Deserialize)]
 pub struct Annotation {
@@ -417,10 +409,10 @@ impl Annotation {
         match self.get_value::<u32>(KATA_ANNO_CONTAINER_RES_SWAPPINESS) {
             Ok(r) => {
                 if r.unwrap_or_default() > 100 {
-                    Err(io::Error::new(
+                    return Err(io::Error::new(
                         io::ErrorKind::InvalidData,
                         format!("{} greater than 100", r.unwrap_or_default()),
-                    ))
+                    ));
                 } else {
                     Ok(r)
                 }
@@ -681,9 +673,6 @@ impl Annotation {
                         hv.machine_info.validate_entropy_source(value)?;
                         hv.machine_info.entropy_source = value.to_string();
                     }
-                    KATA_ANNO_CFG_HYPERVISOR_PREFETCH_FILES_LIST => {
-                        hv.prefetch_list_path = value.to_string();
-                    }
                     // Hypervisor Memory related annotations
                     KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY => {
                         match byte_unit::Byte::from_str(value) {