Merge pull request #7523 from fidencio/3.2.0-rc0-branch-bump

# Kata Containers 3.2.0-rc0

.github/workflows/PR-wip-checks.yaml

@@ -9,6 +9,10 @@ on:
       - labeled
       - unlabeled
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   pr_wip_check:
     runs-on: ubuntu-latest

.github/workflows/add-backport-label.yaml

@@ -10,6 +10,10 @@ on:
       - labeled
       - unlabeled
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   check-issues:
     if: ${{ github.event.label.name != 'auto-backport' }}

.github/workflows/add-issues-to-project.yaml

@@ -11,6 +11,10 @@ on:
       - opened
       - reopened
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   add-new-issues-to-backlog:
     runs-on: ubuntu-latest

.github/workflows/add-pr-sizing-label.yaml

@@ -12,6 +12,10 @@ on:
       - reopened
       - synchronize
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   add-pr-size-label:
     runs-on: ubuntu-latest

.github/workflows/auto-backport.yaml

@@ -2,6 +2,10 @@ on:
   pull_request_target:
     types: ["labeled", "closed"]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   backport:
     name: Backport PR

.github/workflows/build-kata-static-tarball-amd64.yaml

@@ -2,6 +2,10 @@ name: CI | Build kata-static tarball for amd64
 on:
   workflow_call:
     inputs:
+      stage:
+        required: false
+        type: string
+        default: test
       tarball-suffix:
         required: false
         type: string
@@ -9,6 +13,9 @@ on:
         required: false
         type: string
         default: no
+      commit-hash:
+        required: false
+        type: string
 
 jobs:
   build-asset:
@@ -17,6 +24,7 @@ jobs:
       matrix:
         asset:
           - cloud-hypervisor
+          - cloud-hypervisor-glibc
           - firecracker
           - kernel
           - kernel-sev
@@ -32,11 +40,18 @@ jobs:
           - qemu-snp-experimental
           - qemu-tdx-experimental
           - rootfs-image
+          - rootfs-image-tdx
           - rootfs-initrd
+          - rootfs-initrd-mariner
           - rootfs-initrd-sev
           - shim-v2
           - tdvf
           - virtiofsd
+        stage:
+          - ${{ inputs.stage }}
+        exclude:
+          - asset: cloud-hypervisor-glibc
+            stage: release
     steps:
       - name: Login to Kata Containers quay.io
         if: ${{ inputs.push-to-registry == 'yes' }}
@@ -48,7 +63,7 @@ jobs:
 
       - uses: actions/checkout@v3
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ inputs.commit-hash }}
           fetch-depth: 0 # This is needed in order to keep the commit ids history
 
       - name: Build ${{ matrix.asset }}
@@ -76,7 +91,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ inputs.commit-hash }}
       - name: get-artifacts
         uses: actions/download-artifact@v3
         with:
@@ -84,7 +99,7 @@ jobs:
           path: kata-artifacts
       - name: merge-artifacts
         run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts versions.yaml
       - name: store-artifacts
         uses: actions/upload-artifact@v3
         with:

.github/workflows/build-kata-static-tarball-arm64.yaml

@@ -2,6 +2,10 @@ name: CI | Build kata-static tarball for arm64
 on:
   workflow_call:
     inputs:
+      stage:
+        required: false
+        type: string
+        default: test
       tarball-suffix:
         required: false
         type: string
@@ -9,6 +13,9 @@ on:
         required: false
         type: string
         default: no
+      commit-hash:
+        required: false
+        type: string
 
 jobs:
   build-asset:
@@ -26,6 +33,8 @@ jobs:
           - rootfs-initrd
           - shim-v2
           - virtiofsd
+        stage:
+          - ${{ inputs.stage }}
     steps:
       - name: Adjust a permission for repo
         run: |
@@ -41,7 +50,7 @@ jobs:
 
       - uses: actions/checkout@v3
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ inputs.commit-hash }}
           fetch-depth: 0 # This is needed in order to keep the commit ids history
       - name: Build ${{ matrix.asset }}
         run: |
@@ -72,7 +81,7 @@ jobs:
 
       - uses: actions/checkout@v3
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ inputs.commit-hash }}
       - name: get-artifacts
         uses: actions/download-artifact@v3
         with:
@@ -80,7 +89,7 @@ jobs:
           path: kata-artifacts
       - name: merge-artifacts
         run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts versions.yaml
       - name: store-artifacts
         uses: actions/upload-artifact@v3
         with:

.github/workflows/build-kata-static-tarball-s390x.yaml

@@ -2,6 +2,10 @@ name: CI | Build kata-static tarball for s390x
 on:
   workflow_call:
     inputs:
+      stage:
+        required: false
+        type: string
+        default: test
       tarball-suffix:
         required: false
         type: string
@@ -9,6 +13,9 @@ on:
         required: false
         type: string
         default: no
+      commit-hash:
+        required: false
+        type: string
 
 jobs:
   build-asset:
@@ -22,6 +29,8 @@ jobs:
           - rootfs-initrd
           - shim-v2
           - virtiofsd
+        stage:
+          - ${{ inputs.stage }}
     steps:
       - name: Adjust a permission for repo
         run: |
@@ -37,7 +46,7 @@ jobs:
 
       - uses: actions/checkout@v3
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ inputs.commit-hash }}
           fetch-depth: 0 # This is needed in order to keep the commit ids history
       - name: Build ${{ matrix.asset }}
         run: |
@@ -69,7 +78,7 @@ jobs:
 
       - uses: actions/checkout@v3
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ inputs.commit-hash }}
       - name: get-artifacts
         uses: actions/download-artifact@v3
         with:
@@ -77,7 +86,7 @@ jobs:
           path: kata-artifacts
       - name: merge-artifacts
         run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts versions.yaml
       - name: store-artifacts
         uses: actions/upload-artifact@v3
         with:

.github/workflows/cargo-deny-runner.yaml

@@ -7,6 +7,11 @@ on:
       - reopened
       - synchronize
     paths-ignore: [ '**.md', '**.png', '**.jpg', '**.jpeg', '**.svg', '/docs/**' ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   cargo-deny-runner:
     runs-on: ubuntu-latest

.github/workflows/cc-payload-after-push-amd64.yaml

@@ -1,161 +0,0 @@
-name: CI | Publish CC runtime payload for amd64
-on:
-  workflow_call:
-    inputs:
-      target-arch:
-        required: true
-        type: string
-
-jobs:
-  build-asset:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        asset:
-          - cc-cloud-hypervisor
-          - cc-kernel
-          - cc-qemu
-          - cc-rootfs-image
-          - cc-virtiofsd
-          - cc-sev-kernel
-          - cc-sev-ovmf
-          - cc-x86_64-ovmf
-          - cc-snp-qemu
-          - cc-sev-rootfs-initrd
-          - cc-tdx-kernel
-          - cc-tdx-rootfs-image
-          - cc-tdx-qemu
-          - cc-tdx-td-shim
-          - cc-tdx-tdvf
-    steps:
-      - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # This is needed in order to keep the commit ids history
-      - name: Build ${{ matrix.asset }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-          PUSH_TO_REGISTRY: yes
-
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-      - name: store-artifact root_hash_tdx.txt 
-        uses: actions/upload-artifact@v3
-        with:
-          name: root_hash_tdx.txt
-          path: tools/osbuilder/root_hash_tdx.txt
-          retention-days: 1
-          if-no-files-found: ignore
-
-      - name: store-artifact root_hash_vanilla.txt 
-        uses: actions/upload-artifact@v3
-        with:
-          name: root_hash_vanilla.txt
-          path: tools/osbuilder/root_hash_vanilla.txt
-          retention-days: 1
-          if-no-files-found: ignore
-
-  build-asset-cc-shim-v2:
-    runs-on: ubuntu-latest
-    needs: build-asset
-    steps:
-      - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - uses: actions/checkout@v3
-
-      - name: Get root_hash_tdx.txt
-        uses: actions/download-artifact@v3
-        with:
-          name: root_hash_tdx.txt
-          path: tools/osbuilder/
-
-      - name: Get root_hash_vanilla.txt
-        uses: actions/download-artifact@v3
-        with:
-          name: root_hash_vanilla.txt
-          path: tools/osbuilder/
-
-      - name: Build cc-shim-v2
-        run: |
-          make cc-shim-v2-tarball
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          PUSH_TO_REGISTRY: yes
-
-      - name: store-artifact cc-shim-v2
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-cc-shim-v2.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: ubuntu-latest
-    needs: [build-asset, build-asset-cc-shim-v2]
-    steps:
-      - uses: actions/checkout@v3
-      - name: get-artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-artifacts
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-static-tarball
-          path: kata-static.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  kata-payload:
-    needs: create-kata-tarball
-    runs-on: ubuntu-latest
-    steps:
-      - name: Login to Confidential Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
-
-      - uses: actions/checkout@v3
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball
-
-      - name: build-and-push-kata-payload
-        id: build-and-push-kata-payload
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz "quay.io/confidential-containers/runtime-payload-ci" \
-          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/cc-payload-after-push-s390x.yaml

@@ -1,164 +0,0 @@
-name: CI | Publish CC runtime payload for s390x
-on:
-  workflow_call:
-    inputs:
-      target-arch:
-        required: true
-        type: string
-
-jobs:
-  build-asset:
-    runs-on: s390x
-    strategy:
-      matrix:
-        asset:
-          - cc-kernel
-          - cc-qemu
-          - cc-rootfs-image
-          - cc-rootfs-initrd
-          - cc-se-image
-          - cc-virtiofsd
-    steps:
-      - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # This is needed in order to keep the commit ids history
-
-      - name: Place a host key document
-        run: |
-          mkdir -p "host-key-document"
-          cp "${CI_HKD_PATH}" "host-key-document"
-        env:
-          CI_HKD_PATH: ${{ secrets.CI_HKD_PATH }}
-
-      - name: Build ${{ matrix.asset }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-          sudo chown -R $(id -u):$(id -g) "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-          PUSH_TO_REGISTRY: yes
-          HKD_PATH: "host-key-document"
-
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts-s390x
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-      - name: store-artifact root_hash_vanilla.txt 
-        uses: actions/upload-artifact@v3
-        with:
-          name: root_hash_vanilla.txt-s390x
-          path: tools/osbuilder/root_hash_vanilla.txt
-          retention-days: 1
-          if-no-files-found: ignore
-
-  build-asset-cc-shim-v2:
-    runs-on: s390x
-    needs: build-asset
-    steps:
-      - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-
-      - name: Get root_hash_vanilla.txt
-        uses: actions/download-artifact@v3
-        with:
-          name: root_hash_vanilla.txt-s390x
-          path: tools/osbuilder/
-
-      - name: Build cc-shim-v2
-        run: |
-          make cc-shim-v2-tarball
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          PUSH_TO_REGISTRY: yes
-
-      - name: store-artifact cc-shim-v2
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts-s390x
-          path: kata-build/kata-static-cc-shim-v2.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: s390x
-    needs: [build-asset, build-asset-cc-shim-v2]
-    steps:
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-      - name: get-artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-artifacts-s390x
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-static-tarball-s390x
-          path: kata-static.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  kata-payload:
-    needs: create-kata-tarball
-    runs-on: s390x
-    steps:
-      - name: Login to Confidential Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
-
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-s390x
-
-      - name: build-and-push-kata-payload
-        id: build-and-push-kata-payload
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz "quay.io/confidential-containers/runtime-payload-ci" \
-          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/cc-payload-after-push.yaml

@@ -1,46 +0,0 @@
-name: CI | Publish Kata Containers payload for Confidential Containers
-on:
-  push:
-    branches:
-      - CCv0
-
-jobs:
-  build-assets-amd64:
-    uses: ./.github/workflows/cc-payload-after-push-amd64.yaml
-    with:
-      target-arch: amd64
-    secrets: inherit
-
-  build-assets-s390x:
-    uses: ./.github/workflows/cc-payload-after-push-s390x.yaml
-    with:
-      target-arch: s390x
-    secrets: inherit
-
-  publish:
-    runs-on: ubuntu-latest
-    needs: [build-assets-amd64, build-assets-s390x]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Login to Confidential Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
-
-      - name: Push commit multi-arch manifest
-        run: |
-          docker manifest create quay.io/confidential-containers/runtime-payload-ci:kata-containers-${GITHUB_SHA} \
-          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-${GITHUB_SHA}-amd64 \
-          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-${GITHUB_SHA}-s390x
-          docker manifest push quay.io/confidential-containers/runtime-payload-ci:kata-containers-${GITHUB_SHA}
-
-      - name: Push latest multi-arch manifest
-        run: |
-          docker manifest create quay.io/confidential-containers/runtime-payload-ci:kata-containers-latest \
-          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-amd64 \
-          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-s390x
-          docker manifest push quay.io/confidential-containers/runtime-payload-ci:kata-containers-latest

.github/workflows/cc-payload-amd64.yaml

@@ -1,144 +0,0 @@
-name: Publish Kata Containers payload for Confidential Containers (amd64)
-on:
-  workflow_call:
-    inputs:
-      target-arch:
-        required: true
-        type: string
-
-jobs:
-  build-asset:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        asset:
-          - cc-cloud-hypervisor
-          - cc-kernel
-          - cc-qemu
-          - cc-rootfs-image
-          - cc-virtiofsd
-          - cc-sev-kernel
-          - cc-sev-ovmf
-          - cc-x86_64-ovmf
-          - cc-snp-qemu
-          - cc-sev-rootfs-initrd
-          - cc-tdx-kernel
-          - cc-tdx-rootfs-image
-          - cc-tdx-qemu
-          - cc-tdx-td-shim
-          - cc-tdx-tdvf
-    steps:
-      - uses: actions/checkout@v3
-      - name: Build ${{ matrix.asset }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-      - name: store-artifact root_hash_tdx.txt 
-        uses: actions/upload-artifact@v3
-        with:
-          name: root_hash_tdx.txt
-          path: tools/osbuilder/root_hash_tdx.txt
-          retention-days: 1
-          if-no-files-found: ignore
-
-      - name: store-artifact root_hash_vanilla.txt 
-        uses: actions/upload-artifact@v3
-        with:
-          name: root_hash_vanilla.txt
-          path: tools/osbuilder/root_hash_vanilla.txt
-          retention-days: 1
-          if-no-files-found: ignore
-
-  build-asset-cc-shim-v2:
-    runs-on: ubuntu-latest
-    needs: build-asset
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Get root_hash_tdx.txt
-        uses: actions/download-artifact@v3
-        with:
-          name: root_hash_tdx.txt
-          path: tools/osbuilder/
-
-      - name: Get root_hash_vanilla.txt
-        uses: actions/download-artifact@v3
-        with:
-          name: root_hash_vanilla.txt
-          path: tools/osbuilder/
-
-      - name: Build cc-shim-v2
-        run: |
-          make cc-shim-v2-tarball
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-
-      - name: store-artifact cc-shim-v2
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-cc-shim-v2.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: ubuntu-latest
-    needs: [build-asset, build-asset-cc-shim-v2]
-    steps:
-      - uses: actions/checkout@v3
-      - name: get-artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-artifacts
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-static-tarball
-          path: kata-static.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  kata-payload:
-    needs: create-kata-tarball
-    runs-on: ubuntu-latest
-    steps:
-      - name: Login to quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
-
-      - uses: actions/checkout@v3
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball
-
-      - name: build-and-push-kata-payload
-        id: build-and-push-kata-payload
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz \
-          "quay.io/confidential-containers/runtime-payload" \
-          "kata-containers-${{ inputs.target-arch }}"
-

.github/workflows/cc-payload-s390x.yaml

@@ -1,134 +0,0 @@
-name: Publish Kata Containers payload for Confidential Containers (s390x)
-on:
-  workflow_call:
-    inputs:
-      target-arch:
-        required: true
-        type: string
-
-jobs:
-  build-asset:
-    runs-on: s390x
-    strategy:
-      matrix:
-        asset:
-          - cc-kernel
-          - cc-qemu
-          - cc-rootfs-image
-          - cc-virtiofsd
-    steps:
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-      - name: Build ${{ matrix.asset }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts-s390x
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-      - name: store-artifact root_hash_vanilla.txt 
-        uses: actions/upload-artifact@v3
-        with:
-          name: root_hash_vanilla.txt-s390x
-          path: tools/osbuilder/root_hash_vanilla.txt
-          retention-days: 1
-          if-no-files-found: ignore
-
-  build-asset-cc-shim-v2:
-    runs-on: s390x
-    needs: build-asset
-    steps:
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-
-      - name: Get root_hash_vanilla.txt
-        uses: actions/download-artifact@v3
-        with:
-          name: root_hash_vanilla.txt-s390x
-          path: tools/osbuilder/
-
-      - name: Build cc-shim-v2
-        run: |
-          make cc-shim-v2-tarball
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-
-      - name: store-artifact cc-shim-v2
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts-s390x
-          path: kata-build/kata-static-cc-shim-v2.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: s390x
-    needs: [build-asset, build-asset-cc-shim-v2]
-    steps:
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-      - name: get-artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-artifacts-s390x
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-static-tarball-s390x
-          path: kata-static.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  kata-payload:
-    needs: create-kata-tarball
-    runs-on: s390x
-    steps:
-      - name: Login to quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
-
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-s390x
-
-      - name: build-and-push-kata-payload
-        id: build-and-push-kata-payload
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz \
-          "quay.io/confidential-containers/runtime-payload" \
-          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/cc-payload.yaml

@@ -1,46 +0,0 @@
-name: Publish Kata Containers payload for Confidential Containers
-on:
-  push:
-    tags:
-      - 'CC\-[0-9]+.[0-9]+.[0-9]+'
-
-jobs:
-  build-assets-amd64:
-    uses: ./.github/workflows/cc-payload-amd64.yaml
-    with:
-      target-arch: amd64
-    secrets: inherit
-
-  build-assets-s390x:
-    uses: ./.github/workflows/cc-payload-s390x.yaml
-    with:
-      target-arch: s390x
-    secrets: inherit
-
-  publish:
-    runs-on: ubuntu-latest
-    needs: [build-assets-amd64, build-assets-s390x]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Login to Confidential Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
-
-      - name: Push commit multi-arch manifest
-        run: |
-          docker manifest create quay.io/confidential-containers/runtime-payload:kata-containers-${GITHUB_SHA} \
-          --amend quay.io/confidential-containers/runtime-payload:kata-containers-${GITHUB_SHA}-amd64 \
-          --amend quay.io/confidential-containers/runtime-payload:kata-containers-${GITHUB_SHA}-s390x
-          docker manifest push quay.io/confidential-containers/runtime-payload:kata-containers-${GITHUB_SHA}
-
-      - name: Push latest multi-arch manifest
-        run: |
-          docker manifest create quay.io/confidential-containers/runtime-payload:kata-containers-latest \
-          --amend quay.io/confidential-containers/runtime-payload:kata-containers-amd64 \
-          --amend quay.io/confidential-containers/runtime-payload:kata-containers-s390x
-          docker manifest push quay.io/confidential-containers/runtime-payload:kata-containers-latest

.github/workflows/ci-nightly.yaml

@@ -0,0 +1,18 @@
+name: Kata Containers Nightly CI
+on:
+  schedule:
+    - cron: '0 0 * * *'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  kata-containers-ci-on-push:
+    uses: ./.github/workflows/ci.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      pr-number: "nightly"
+      tag: ${{ github.sha }}-nightly
+    secrets: inherit

.github/workflows/ci-on-push.yaml

@@ -3,52 +3,29 @@ on:
   pull_request_target:
     branches:
       - 'main'
+      - 'stable-*'
+    types:
+      # Adding 'labeled' to the list of activity types that trigger this event
+      # (default: opened, synchronize, reopened) so that we can run this
+      # workflow when the 'ok-to-test' label is added.
+      # Reference: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target
+      - opened
+      - synchronize
+      - reopened
+      - labeled
+    paths-ignore:
+      - 'docs/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 jobs:
-  build-kata-static-tarball-amd64:
-    uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
+  kata-containers-ci-on-push:
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'ok-to-test') }}
+    uses: ./.github/workflows/ci.yaml
     with:
-      tarball-suffix: -${{ github.event.pull_request.number}}-${{ github.event.pull_request.head.sha }}
-
-  publish-kata-deploy-payload-amd64:
-    needs: build-kata-static-tarball-amd64
-    uses: ./.github/workflows/publish-kata-deploy-payload-amd64.yaml
-    with:
-      tarball-suffix: -${{ github.event.pull_request.number}}-${{ github.event.pull_request.head.sha }}
-      registry: ghcr.io
-      repo: ${{ github.repository_owner }}/kata-deploy-ci
-      tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64
+      commit-hash: ${{ github.event.pull_request.head.sha }}
+      pr-number: ${{ github.event.pull_request.number }}
+      tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
     secrets: inherit
-
-  run-k8s-tests-on-aks:
-    needs: publish-kata-deploy-payload-amd64
-    uses: ./.github/workflows/run-k8s-tests-on-aks.yaml
-    with:
-      registry: ghcr.io
-      repo: ${{ github.repository_owner }}/kata-deploy-ci
-      tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64
-    secrets: inherit
-
-  run-k8s-tests-on-sev:
-    needs: publish-kata-deploy-payload-amd64
-    uses: ./.github/workflows/run-k8s-tests-on-sev.yaml
-    with:
-      registry: ghcr.io
-      repo: ${{ github.repository_owner }}/kata-deploy-ci
-      tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64
-
-  run-k8s-tests-on-snp:
-    needs: publish-kata-deploy-payload-amd64
-    uses: ./.github/workflows/run-k8s-tests-on-snp.yaml
-    with:
-      registry: ghcr.io
-      repo: ${{ github.repository_owner }}/kata-deploy-ci
-      tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64
-
-  run-k8s-tests-on-tdx:
-    needs: publish-kata-deploy-payload-amd64
-    uses: ./.github/workflows/run-k8s-tests-on-tdx.yaml
-    with:
-      registry: ghcr.io
-      repo: ${{ github.repository_owner }}/kata-deploy-ci
-      tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64

.github/workflows/ci.yaml

@@ -0,0 +1,97 @@
+name: Run the Kata Containers CI
+on:
+  workflow_call:
+    inputs:
+      commit-hash:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+
+jobs:
+  build-kata-static-tarball-amd64:
+    uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+
+  publish-kata-deploy-payload-amd64:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/publish-kata-deploy-payload-amd64.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+    secrets: inherit
+
+  run-k8s-tests-on-aks:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-k8s-tests-on-aks.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+    secrets: inherit
+
+  run-k8s-tests-on-sev:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-k8s-tests-on-sev.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+
+  run-k8s-tests-on-snp:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-k8s-tests-on-snp.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+
+  run-k8s-tests-on-tdx:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-k8s-tests-on-tdx.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+
+  run-metrics-tests:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/run-metrics.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+
+  run-cri-containerd-tests:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/run-cri-containerd-tests.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+
+  run-nydus-tests:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/run-nydus-tests.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+
+  run-vfio-tests:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/run-vfio-tests.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}

.github/workflows/commit-message-check.yaml

@@ -6,6 +6,10 @@ on:
       - reopened
       - synchronize
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   error_msg: |+
     See the document below for help on formatting commits for the project.
@@ -47,7 +51,7 @@ jobs:
       uses: tim-actions/commit-message-checker-with-regex@v0.3.1
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
-        pattern: '^.{0,75}(\n.*)*$|^Merge pull request (?:kata-containers)?#[\d]+ from.*'
+        pattern: '^.{0,75}(\n.*)*$'
         error: 'Subject too long (max 75)'
         post_error: ${{ env.error_msg }}
 
@@ -98,6 +102,6 @@ jobs:
       uses: tim-actions/commit-message-checker-with-regex@v0.3.1
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
-        pattern: '^[\s\t]*[^:\s\t]+[\s\t]*:|^Merge pull request (?:kata-containers)?#[\d]+ from.*'
+        pattern: '^[\s\t]*[^:\s\t]+[\s\t]*:'
         error: 'Failed to find subsystem in subject'
         post_error: ${{ env.error_msg }}

.github/workflows/darwin-tests.yaml

@@ -6,6 +6,11 @@ on:
       - reopened
       - synchronize
     paths-ignore: [ '**.md', '**.png', '**.jpg', '**.jpeg', '**.svg', '/docs/**' ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 name: Darwin tests
 jobs:
   test:

.github/workflows/deploy-ccv0-demo.yaml

@@ -1,124 +0,0 @@
-on:
-  issue_comment:
-    types: [created, edited]
-
-name: deploy-ccv0-demo
-
-jobs:
-  check-comment-and-membership:
-    runs-on: ubuntu-latest
-    if: |
-      github.event.issue.pull_request
-      && github.event_name == 'issue_comment'
-      && github.event.action == 'created'
-      && startsWith(github.event.comment.body, '/deploy-ccv0-demo')
-    steps:
-      - name: Check membership
-        uses: kata-containers/is-organization-member@1.0.1
-        id: is_organization_member
-        with:
-          organization: kata-containers
-          username: ${{ github.event.comment.user.login }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Fail if not member
-        run: |
-          result=${{ steps.is_organization_member.outputs.result }}
-          if [ $result == false ]; then
-              user=${{ github.event.comment.user.login }}
-              echo Either ${user} is not part of the kata-containers organization
-              echo or ${user} has its Organization Visibility set to Private at
-              echo https://github.com/orgs/kata-containers/people?query=${user}
-              echo 
-              echo Ensure you change your Organization Visibility to Public and
-              echo trigger the test again.
-              exit 1
-          fi
-
-  build-asset:
-    runs-on: ubuntu-latest
-    needs: check-comment-and-membership
-    strategy:
-      matrix:
-        asset:
-          - cloud-hypervisor
-          - firecracker
-          - kernel
-          - qemu
-          - rootfs-image
-          - rootfs-initrd
-          - shim-v2
-    steps:
-      - uses: actions/checkout@v2
-      - name: Prepare confidential container rootfs
-        if: ${{ matrix.asset == 'rootfs-initrd' }}
-        run: |
-          pushd include_rootfs/etc
-          curl -LO https://raw.githubusercontent.com/confidential-containers/documentation/main/demos/ssh-demo/aa-offline_fs_kbc-keys.json
-          mkdir kata-containers
-          envsubst < docs/how-to/data/confidential-agent-config.toml.in > kata-containers/agent.toml
-          popd
-        env:
-          AA_KBC_PARAMS: offline_fs_kbc::null
-
-      - name: Build ${{ matrix.asset }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          AA_KBC: offline_fs_kbc
-          INCLUDE_ROOTFS: include_rootfs
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: ubuntu-latest
-    needs: build-asset
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifacts
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-static-tarball
-          path: kata-static.tar.xz
-
-  kata-deploy:
-    needs: create-kata-tarball
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-static-tarball
-      - name: build-and-push-kata-deploy-ci
-        id: build-and-push-kata-deploy-ci
-        run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          pushd $GITHUB_WORKSPACE
-          git checkout $tag
-          pkg_sha=$(git rev-parse HEAD)
-          popd
-          mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
-          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t quay.io/confidential-containers/runtime-payload:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
-          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
-          docker push quay.io/confidential-containers/runtime-payload:$pkg_sha
-          mkdir -p packaging/kata-deploy
-          ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
-          echo "::set-output name=PKG_SHA::${pkg_sha}"

.github/workflows/kata-runtime-classes-sync.yaml

@@ -0,0 +1,36 @@
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  kata-deploy-runtime-classes-check:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+    - name: Ensure the split out runtime classes match the all-in-one file
+      run: |
+        pushd tools/packaging/kata-deploy/runtimeclasses/
+        echo "::group::Combine runtime classes"
+        for runtimeClass in `find . -type f \( -name "*.yaml" -and -not -name "kata-runtimeClasses.yaml" \) | sort`; do
+            echo "Adding ${runtimeClass} to the resultingRuntimeClasses.yaml"
+            cat ${runtimeClass} >> resultingRuntimeClasses.yaml;
+        done
+        echo "::endgroup::"
+        echo "::group::Displaying the content of resultingRuntimeClasses.yaml"
+        cat resultingRuntimeClasses.yaml
+        echo "::endgroup::"
+        echo ""
+        echo "::group::Displaying the content of kata-runtimeClasses.yaml"
+        cat kata-runtimeClasses.yaml
+        echo "::endgroup::"
+        echo ""
+        diff resultingRuntimeClasses.yaml kata-runtimeClasses.yaml

.github/workflows/payload-after-push.yaml

@@ -5,22 +5,29 @@ on:
       - main
       - stable-*
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build-assets-amd64:
     uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
     with:
+      commit-hash: ${{ github.sha }}
       push-to-registry: yes
     secrets: inherit
 
   build-assets-arm64:
     uses: ./.github/workflows/build-kata-static-tarball-arm64.yaml
     with:
+      commit-hash: ${{ github.sha }}
       push-to-registry: yes
     secrets: inherit
 
   build-assets-s390x:
     uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml
     with:
+      commit-hash: ${{ github.sha }}
       push-to-registry: yes
     secrets: inherit
 
@@ -28,6 +35,7 @@ jobs:
     needs: build-assets-amd64
     uses: ./.github/workflows/publish-kata-deploy-payload-amd64.yaml
     with:
+      commit-hash: ${{ github.sha }}
       registry: quay.io
       repo: kata-containers/kata-deploy-ci
       tag: kata-containers-amd64
@@ -37,6 +45,7 @@ jobs:
     needs: build-assets-arm64
     uses: ./.github/workflows/publish-kata-deploy-payload-arm64.yaml
     with:
+      commit-hash: ${{ github.sha }}
       registry: quay.io
       repo: kata-containers/kata-deploy-ci
       tag: kata-containers-arm64
@@ -46,6 +55,7 @@ jobs:
     needs: build-assets-s390x
     uses: ./.github/workflows/publish-kata-deploy-payload-s390x.yaml
     with:
+      commit-hash: ${{ github.sha }}
       registry: quay.io
       repo: kata-containers/kata-deploy-ci
       tag: kata-containers-s390x

.github/workflows/publish-kata-deploy-payload-amd64.yaml

@@ -14,6 +14,9 @@ on:
       tag:
         required: true
         type: string
+      commit-hash:
+        required: false
+        type: string
 
 jobs:
   kata-payload:
@@ -21,7 +24,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ inputs.commit-hash }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@v3

.github/workflows/publish-kata-deploy-payload-arm64.yaml

@@ -14,6 +14,9 @@ on:
       tag:
         required: true
         type: string
+      commit-hash:
+        required: false
+        type: string
 
 jobs:
   kata-payload:
@@ -25,7 +28,7 @@ jobs:
 
       - uses: actions/checkout@v3
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ inputs.commit-hash }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@v3

.github/workflows/publish-kata-deploy-payload-s390x.yaml

@@ -14,6 +14,9 @@ on:
       tag:
         required: true
         type: string
+      commit-hash:
+        required: false
+        type: string
 
 jobs:
   kata-payload:
@@ -25,7 +28,7 @@ jobs:
 
       - uses: actions/checkout@v3
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ inputs.commit-hash }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@v3

.github/workflows/release-amd64.yaml

@@ -9,6 +9,8 @@ on:
 jobs:
   build-kata-static-tarball-amd64:
     uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
+    with:
+      stage: release
 
   kata-deploy:
     needs: build-kata-static-tarball-amd64

.github/workflows/release-arm64.yaml

@@ -9,6 +9,8 @@ on:
 jobs:
   build-kata-static-tarball-arm64:
     uses: ./.github/workflows/build-kata-static-tarball-arm64.yaml
+    with:
+      stage: release
 
   kata-deploy:
     needs: build-kata-static-tarball-arm64

.github/workflows/release-s390x.yaml

@@ -9,6 +9,8 @@ on:
 jobs:
   build-kata-static-tarball-s390x:
     uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml
+    with:
+      stage: release
 
   kata-deploy:
     needs: build-kata-static-tarball-s390x

.github/workflows/release.yaml

@@ -4,6 +4,10 @@ on:
     tags:
       - '[0-9]+.[0-9]+.[0-9]+*'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build-and-push-assets-amd64:
     uses: ./.github/workflows/release-amd64.yaml
@@ -72,8 +76,7 @@ jobs:
       - uses: actions/checkout@v3
       - name: install hub
         run: |
-          HUB_VER=$(curl -s "https://api.github.com/repos/github/hub/releases/latest" | jq -r .tag_name | sed 's/^v//')
-          wget -q -O- https://github.com/github/hub/releases/download/v$HUB_VER/hub-linux-amd64-$HUB_VER.tgz | \
+          wget -q -O- https://github.com/mislav/hub/releases/download/v2.14.2/hub-linux-amd64-2.14.2.tgz | \
           tar xz --strip-components=2 --wildcards '*/bin/hub' && sudo mv hub /usr/local/bin/hub
 
       - name: download-artifacts-amd64
@@ -83,7 +86,7 @@ jobs:
       - name: push amd64 static tarball to github
         run: |
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          tarball="kata-static-$tag-x86_64.tar.xz"
+          tarball="kata-static-$tag-amd64.tar.xz"
           mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
           pushd $GITHUB_WORKSPACE
           echo "uploading asset '${tarball}' for tag: ${tag}"
@@ -97,7 +100,7 @@ jobs:
       - name: push arm64 static tarball to github
         run: |
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          tarball="kata-static-$tag-aarch64.tar.xz"
+          tarball="kata-static-$tag-arm64.tar.xz"
           mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
           pushd $GITHUB_WORKSPACE
           echo "uploading asset '${tarball}' for tag: ${tag}"
@@ -118,6 +121,21 @@ jobs:
           GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}"
           popd
 
+  upload-versions-yaml:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: upload versions.yaml
+        env:
+          GITHUB_TOKEN: ${{ secrets.GIT_UPLOAD_TOKEN }}
+        run: |
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          pushd $GITHUB_WORKSPACE
+          versions_file="kata-containers-$tag-versions.yaml"
+          cp versions.yaml ${versions_file}
+          hub release edit -m "" -a "${versions_file}" "${tag}"
+          popd
+
   upload-cargo-vendored-tarball:
     needs: upload-multi-arch-static-tarball
     runs-on: ubuntu-latest

.github/workflows/require-pr-porting-labels.yaml

@@ -15,6 +15,10 @@ on:
     branches:
       - main
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   check-pr-porting-labels:
     runs-on: ubuntu-latest

.github/workflows/run-cri-containerd-tests.yaml

@@ -0,0 +1,42 @@
+name: CI | Run cri-containerd tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+
+jobs:
+  run-cri-containerd:
+    strategy:
+      fail-fast: true
+      matrix:
+        containerd_version: ['lts', 'active']
+        vmm: ['clh', 'qemu']
+    runs-on: garm-ubuntu-2204
+    env:
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.commit-hash }}
+
+      - name: Install dependencies
+        run: bash tests/integration/cri-containerd/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/cri-containerd/gha-run.sh install-kata kata-artifacts
+
+      - name: Run cri-containerd tests
+        run: bash tests/integration/cri-containerd/gha-run.sh run

.github/workflows/run-k8s-tests-on-aks.yaml

@@ -11,85 +11,72 @@ on:
       tag:
         required: true
         type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
 
 jobs:
   run-k8s-tests:
     strategy:
       fail-fast: false
       matrix:
+        host_os:
+          - ubuntu
         vmm:
           - clh
           - dragonball
           - qemu
+        include:
+          - host_os: cbl-mariner
+            vmm: clh
     runs-on: ubuntu-latest
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HOST_OS: ${{ matrix.host_os }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      USING_NFD: "false"
     steps:
       - uses: actions/checkout@v3
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ inputs.commit-hash }}
 
       - name: Download Azure CLI
-        run: |
-          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+        run: bash tests/integration/kubernetes/gha-run.sh install-azure-cli
 
       - name: Log into the Azure account
-        run: |
-          az login \
-            --service-principal \
-            -u "${{ secrets.AZ_APPID }}" \
-            -p "${{ secrets.AZ_PASSWORD }}" \
-            --tenant "${{ secrets.AZ_TENANT_ID }}"
+        run: bash tests/integration/kubernetes/gha-run.sh login-azure
+        env:
+          AZ_APPID: ${{ secrets.AZ_APPID }}
+          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
+          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
 
       - name: Create AKS cluster
-        run: |
-          az aks create \
-            -g "kataCI" \
-            -n "${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-amd64" \
-            -s "Standard_D4s_v5" \
-            --node-count 1 \
-            --generate-ssh-keys
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh create-cluster
 
       - name: Install `bats`
-        run: |
-          sudo apt-get update
-          sudo apt-get -y install bats
+        run: bash tests/integration/kubernetes/gha-run.sh install-bats
 
       - name: Install `kubectl`
-        run: |
-          sudo az aks install-cli
+        run: bash tests/integration/kubernetes/gha-run.sh install-kubectl
 
       - name: Download credentials for the Kubernetes CLI to use them
-        run: |
-          az aks get-credentials -g "kataCI" -n ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-amd64
+        run: bash tests/integration/kubernetes/gha-run.sh get-cluster-credentials
 
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-aks
+      
       - name: Run tests
-        timeout-minutes: 35
-        run: |
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-
-          kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
-          kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
-
-          # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
-          # which may cause issues like not having the node properly labeled or the artefacts
-          # properly deployed when the tests actually start running.
-          sleep 150s
-
-          pushd tests/integration/kubernetes
-          sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
-          bash run_kubernetes_tests.sh
-          popd
-        env:
-          KATA_HYPERVISOR: ${{ matrix.vmm }}
+        timeout-minutes: 60
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
 
       - name: Delete AKS cluster
         if: always()
-        run: |
-          az aks delete \
-            -g "kataCI" \
-            -n "${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-amd64" \
-            --yes \
-            --no-wait
+        run: bash tests/integration/kubernetes/gha-run.sh delete-cluster

.github/workflows/run-k8s-tests-on-sev.yaml

@@ -11,6 +11,9 @@ on:
       tag:
         required: true
         type: string
+      commit-hash:
+        required: false
+        type: string
 
 jobs:
   run-k8s-tests:
@@ -21,48 +24,25 @@ jobs:
           - qemu-sev
     runs-on: sev
     env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
       KUBECONFIG: /home/kata/.kube/config
+      USING_NFD: "false"
     steps:
       - uses: actions/checkout@v3
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ inputs.commit-hash }}
 
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-sev
+  
       - name: Run tests
         timeout-minutes: 30
-        run: |
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-
-          kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
-          kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
-
-          # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
-          # which may cause issues like not having the node properly labeled or the artefacts
-          # properly deployed when the tests actually start running.
-          sleep 60s
-
-          pushd tests/integration/kubernetes
-          sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
-          bash run_kubernetes_tests.sh
-          popd
-        env:
-          KATA_HYPERVISOR: ${{ matrix.vmm }}
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
 
       - name: Delete kata-deploy
         if: always()
-        run: |
-          kubectl delete -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
-
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-          kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          sleep 180s
-
-          kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-sev

.github/workflows/run-k8s-tests-on-snp.yaml

@@ -11,6 +11,9 @@ on:
       tag:
         required: true
         type: string
+      commit-hash:
+        required: false
+        type: string
 
 jobs:
   run-k8s-tests:
@@ -21,48 +24,25 @@ jobs:
           - qemu-snp
     runs-on: sev-snp
     env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
       KUBECONFIG: /home/kata/.kube/config
+      USING_NFD: "false"
     steps:
       - uses: actions/checkout@v3
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ inputs.commit-hash }}
 
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-snp
+  
       - name: Run tests
         timeout-minutes: 30
-        run: |
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-
-          kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
-          kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
-
-          # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
-          # which may cause issues like not having the node properly labeled or the artefacts
-          # properly deployed when the tests actually start running.
-          sleep 60s
-
-          pushd tests/integration/kubernetes
-          sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
-          bash run_kubernetes_tests.sh
-          popd
-        env:
-          KATA_HYPERVISOR: ${{ matrix.vmm }}
-
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+      
       - name: Delete kata-deploy
         if: always()
-        run: |
-          kubectl delete -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
-
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-          kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          sleep 180s
-
-          kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-snp

.github/workflows/run-k8s-tests-on-tdx.yaml

@@ -11,6 +11,9 @@ on:
       tag:
         required: true
         type: string
+      commit-hash:
+        required: false
+        type: string
 
 jobs:
   run-k8s-tests:
@@ -21,48 +24,24 @@ jobs:
           - qemu-tdx
     runs-on: tdx
     env:
-      KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      USING_NFD: "true"
     steps:
       - uses: actions/checkout@v3
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ inputs.commit-hash }}
 
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-tdx
+  
       - name: Run tests
         timeout-minutes: 30
-        run: |
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-
-          kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl apply -k tools/packaging/kata-deploy/kata-deploy/overlays/k3s
-          kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
-          kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
-
-          # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
-          # which may cause issues like not having the node properly labeled or the artefacts
-          # properly deployed when the tests actually start running.
-          sleep 60s
-
-          pushd tests/integration/kubernetes
-          sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
-          bash run_kubernetes_tests.sh
-          popd
-        env:
-          KATA_HYPERVISOR: ${{ matrix.vmm }}
-
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+  
       - name: Delete kata-deploy
         if: always()
-        run: |
-          kubectl delete -k tools/packaging/kata-deploy/kata-deploy/overlays/k3s
-          kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
-
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-          kubectl apply -k tools/packaging/kata-deploy/kata-cleanup/overlays/k3s
-          sleep 180s
-
-          kubectl delete -k tools/packaging/kata-deploy/kata-cleanup/overlays/k3s
-          kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-tdx

.github/workflows/run-metrics.yaml

@@ -0,0 +1,64 @@
+name: CI | Run test metrics
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+
+jobs:
+  run-metrics:
+    strategy:
+      fail-fast: true
+      matrix:
+        vmm: ['clh', 'qemu']
+      max-parallel: 1
+    runs-on: metrics
+    env:
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.commit-hash }}
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/metrics/gha-run.sh install-kata kata-artifacts
+
+      - name: run launch times test
+        run: bash tests/metrics/gha-run.sh run-test-launchtimes
+
+      - name: run memory foot print test
+        run:  bash tests/metrics/gha-run.sh run-test-memory-usage
+
+      - name: run memory usage inside container test
+        run:  bash tests/metrics/gha-run.sh run-test-memory-usage-inside-container
+
+      - name: run blogbench test
+        run:  bash tests/metrics/gha-run.sh run-test-blogbench
+
+      - name: run tensorflow test
+        run:  bash tests/metrics/gha-run.sh run-test-tensorflow
+
+      - name: run fio test
+        run:  bash tests/metrics/gha-run.sh run-test-fio
+
+      - name: make metrics tarball ${{ matrix.vmm }}
+        run: bash tests/metrics/gha-run.sh make-tarball-results
+
+      - name: archive metrics results ${{ matrix.vmm }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: metrics-artifacts-${{ matrix.vmm }}
+          path: results-${{ matrix.vmm }}.tar.gz
+          retention-days: 1
+          if-no-files-found: error

.github/workflows/run-nydus-tests.yaml

@@ -0,0 +1,42 @@
+name: CI | Run nydus tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+
+jobs:
+  run-nydus:
+    strategy:
+      fail-fast: true
+      matrix:
+        containerd_version: ['lts', 'active']
+        vmm: ['clh', 'qemu', 'dragonball']
+    runs-on: garm-ubuntu-2204
+    env:
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.commit-hash }}
+
+      - name: Install dependencies
+        run: bash tests/integration/nydus/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/nydus/gha-run.sh install-kata kata-artifacts
+
+      - name: Run nydus tests
+        run: bash tests/integration/nydus/gha-run.sh run

.github/workflows/run-vfio-tests.yaml

@@ -0,0 +1,37 @@
+name: CI | Run vfio tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+
+jobs:
+  run-vfio:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm: ['clh', 'qemu']
+    runs-on: garm-ubuntu-2204
+    env:
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.commit-hash }}
+
+      - name: Install dependencies
+        run: bash tests/functional/vfio/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Run vfio tests
+        run: bash tests/functional/vfio/gha-run.sh run

.github/workflows/snap-release.yaml

@@ -1,53 +0,0 @@
-name: Release Kata in snapcraft store
-on:
-  push:
-    tags:
-      - '[0-9]+.[0-9]+.[0-9]+*'
-
-env:
-  SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.snapcraft_token }}
-
-jobs:
-  release-snap:
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Check out Git repository
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Install Snapcraft
-        run: |
-          # Required to avoid snapcraft install failure
-          sudo chown root:root /
-
-          # "--classic" is needed for the GitHub action runner
-          # environment.
-          sudo snap install snapcraft --classic
-
-          # Allow other parts to access snap binaries
-          echo /snap/bin >> "$GITHUB_PATH"
-
-      - name: Build snap
-        run: |
-          # Removing man-db, workflow kept failing, fixes: #4480
-          sudo apt -y remove --purge man-db
-          sudo apt-get update
-          sudo apt-get install -y git git-extras
-          kata_url="https://github.com/kata-containers/kata-containers"
-          latest_version=$(git ls-remote --tags ${kata_url}  | egrep -o "refs.*" | egrep -v "\-alpha|\-rc|{}" | egrep -o "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+" | sort -V -r | head -1)
-          current_version="$(echo ${GITHUB_REF} | cut -d/ -f3)"
-          # Check semantic versioning format (x.y.z) and if the current tag is the latest tag
-          if echo "${current_version}" | grep -q "^[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+$" && echo -e "$latest_version\n$current_version" | sort -C -V; then
-            # Current version is the latest version, build it
-            snapcraft snap --debug --destructive-mode
-          fi
-
-      - name: Upload snap
-        run: |
-          snap_version="$(echo ${GITHUB_REF} | cut -d/ -f3)"
-          snap_file="kata-containers_${snap_version}_amd64.snap"
-          # Upload the snap if it exists
-          if [ -f ${snap_file} ]; then
-            snapcraft upload --release=stable ${snap_file}
-          fi

.github/workflows/snap.yaml

@@ -1,37 +0,0 @@
-name: snap CI
-on:
-  pull_request:
-    types:
-      - opened
-      - synchronize
-      - reopened
-      - edited
-    paths-ignore: [ '**.md', '**.png', '**.jpg', '**.jpeg', '**.svg', '/docs/**' ]
-
-jobs:
-  test:
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Check out
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Install Snapcraft
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          # Required to avoid snapcraft install failure
-          sudo chown root:root /
-
-          # "--classic" is needed for the GitHub action runner
-          # environment.
-          sudo snap install snapcraft --classic
-
-          # Allow other parts to access snap binaries
-          echo /snap/bin >> "$GITHUB_PATH"
-
-      - name: Build snap
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          snapcraft snap --debug --destructive-mode

.github/workflows/static-checks-dragonball.yaml

@@ -7,10 +7,14 @@ on:
       - synchronize
     paths-ignore: [ '**.md', '**.png', '**.jpg', '**.jpeg', '**.svg', '/docs/**' ]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 name: Static checks dragonball
 jobs:
   test-dragonball:
-    runs-on: self-hosted
+    runs-on: dragonball
     env:
       RUST_BACKTRACE: "1"
     steps:
@@ -23,7 +27,7 @@ jobs:
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         run: |
           ./ci/install_rust.sh
-          PATH=$PATH:"$HOME/.cargo/bin"
+          echo PATH="$HOME/.cargo/bin:$PATH" >> $GITHUB_ENV
       - name: Run Unit Test
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         run: |

.github/workflows/static-checks.yaml

@@ -6,6 +6,10 @@ on:
       - reopened
       - synchronize
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 name: Static checks
 jobs:
   static-checks:
@@ -41,8 +45,8 @@ jobs:
         cd "${{ github.workspace }}/src/github.com/${{ github.repository }}"
         kernel_dir="tools/packaging/kernel/"
         kernel_version_file="${kernel_dir}kata_config_version"
-        modified_files=$(git diff --name-only origin/CCv0..HEAD)
-        if git diff --name-only origin/CCv0..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then
+        modified_files=$(git diff --name-only origin/main..HEAD)
+        if git diff --name-only origin/main..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then
           echo "Kernel directory has changed, checking if $kernel_version_file has been updated"
           if echo "$modified_files" | grep -v "README.md" | grep "${kernel_dir}" >>"/dev/null"; then
             echo "$modified_files" | grep "$kernel_version_file" >>/dev/null || ( echo "Please bump version in $kernel_version_file" && exit 1)

Makefile

@@ -24,6 +24,10 @@ TOOLS += trace-forwarder
 
 STANDARD_TARGETS = build check clean install static-checks-build test vendor
 
+# Variables for the build-and-publish-kata-debug target
+KATA_DEBUG_REGISTRY ?= ""
+KATA_DEBUG_TAG ?= ""
+
 default: all
 
 include utils.mk
@@ -44,6 +48,9 @@ static-checks: static-checks-build
 docs-url-alive-check:
 	bash ci/docs-url-alive-check.sh
 
+build-and-publish-kata-debug:
+	bash tools/packaging/kata-debug/kata-debug-build-and-upload-payload.sh ${KATA_DEBUG_REGISTRY} ${KATA_DEBUG_TAG} 
+
 .PHONY: \
 	all \
 	kata-tarball \

README.md

@@ -1,4 +1,6 @@
-<img src="https://www.openstack.org/assets/kata/kata-vertical-on-white.png" width="150">
+<img src="https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/kata/SVG/kata-1.svg" width="900">
+
+[![CI | Publish Kata Containers payload](https://github.com/kata-containers/kata-containers/actions/workflows/payload-after-push.yaml/badge.svg)](https://github.com/kata-containers/kata-containers/actions/workflows/payload-after-push.yaml) [![Kata Containers Nightly CI](https://github.com/kata-containers/kata-containers/actions/workflows/ci-nightly.yaml/badge.svg)](https://github.com/kata-containers/kata-containers/actions/workflows/ci-nightly.yaml)
 
 # Kata Containers
 
@@ -132,6 +134,7 @@ The table below lists the remaining parts of the project:
 | [packaging](tools/packaging) | infrastructure | Scripts and metadata for producing packaged binaries<br/>(components, hypervisors, kernel and rootfs). |
 | [kernel](https://www.kernel.org) | kernel | Linux kernel used by the hypervisor to boot the guest image. Patches are stored [here](tools/packaging/kernel). |
 | [osbuilder](tools/osbuilder) | infrastructure | Tool to create "mini O/S" rootfs and initrd images and kernel for the hypervisor. |
+| [kata-debug](tools/packaging/kata-debug/README.md) | infrastructure | Utility tool to gather Kata Containers debug information from Kubernetes clusters. |
 | [`agent-ctl`](src/tools/agent-ctl) | utility | Tool that provides low-level access for testing the agent. |
 | [`kata-ctl`](src/tools/kata-ctl) | utility | Tool that provides advanced commands and debug facilities. |
 | [`log-parser-rs`](src/tools/log-parser-rs) | utility | Tool that aid in analyzing logs from the kata runtime. |
@@ -144,8 +147,10 @@ The table below lists the remaining parts of the project:
 
 Kata Containers is now
 [available natively for most distributions](docs/install/README.md#packaged-installation-methods).
-However, packaging scripts and metadata are still used to generate [snap](snap/local) and GitHub releases. See
-the [components](#components) section for further details.
+
+## Metrics tests
+
+See the [metrics documentation](tests/metrics/README.md).
 
 ## Glossary of Terms
 

VERSION

@@ -1 +1 @@
-3.2.0-alpha3
+3.2.0-rc0

ci/install_libseccomp.sh

@@ -72,8 +72,7 @@ build_and_install_gperf() {
     curl -sLO "${gperf_tarball_url}"
     tar -xf "${gperf_tarball}"
     pushd "gperf-${gperf_version}"
-    # gperf is a build time dependency of libseccomp and not to be used in the target.
-    # Unset $CC since that might point to a cross compiler.
+    # Unset $CC for configure, we will always use native for gperf
     CC= ./configure --prefix="${gperf_install_dir}"
     make
     make install
@@ -88,7 +87,8 @@ build_and_install_libseccomp() {
     curl -sLO "${libseccomp_tarball_url}"
     tar -xf "${libseccomp_tarball}"
     pushd "libseccomp-${libseccomp_version}"
-    ./configure --prefix="${libseccomp_install_dir}" CFLAGS="${cflags}" --enable-static --host="${arch}"
+    [ "${arch}" == $(uname -m) ] && cc_name="" || cc_name="${arch}-linux-gnu-gcc"
+    CC=${cc_name} ./configure --prefix="${libseccomp_install_dir}" CFLAGS="${cflags}" --enable-static --host="${arch}"
     make
     make install
     popd

ci/lib.sh

@@ -64,86 +64,3 @@ run_get_pr_changed_file_details()
 	source "$tests_repo_dir/.ci/lib.sh"
 	get_pr_changed_file_details
 }
-
-# Check if the 1st argument version is greater than and equal to 2nd one
-# Version format: [0-9]+ separated by period (e.g. 2.4.6, 1.11.3 and etc.)
-#
-# Parameters:
-#	$1	- a version to be tested
-#	$2	- a target version
-#
-# Return:
-# 	0 if $1 is greater than and equal to $2
-#	1 otherwise
-version_greater_than_equal() {
-	local current_version=$1
-	local target_version=$2
-	smaller_version=$(echo -e "$current_version\n$target_version" | sort -V | head -1)
-	if [ "${smaller_version}" = "${target_version}" ]; then
-		return 0
-	else
-		return 1
-	fi
-}
-
-# Build a IBM zSystem secure execution (SE) image
-#
-# Parameters:
-#	$1	- kernel_parameters
-#	$2	- a source directory where kernel and initrd are located
-#	$3	- a destination directory where a SE image is built
-#
-# Return:
-# 	0 if the image is successfully built
-#	1 otherwise
-build_secure_image() {
-	kernel_params="${1:-}"
-	install_src_dir="${2:-}"
-	install_dest_dir="${3:-}"
-
-	if [ ! -f "${install_src_dir}/vmlinuz.container" ] ||
-		[ ! -f "${install_src_dir}/kata-containers-initrd.img" ]; then
-		cat << EOF >&2
-Either kernel or initrd does not exist or is mistakenly named
-A file name for kernel must be vmlinuz.container (raw binary)
-A file name for initrd must be kata-containers-initrd.img
-EOF
-		return 1
-	fi
-
-	cmdline="${kernel_params} panic=1 scsi_mod.scan=none swiotlb=262144"
-	parmfile="$(mktemp --suffix=-cmdline)"
-	echo "${cmdline}" > "${parmfile}"
-	chmod 600 "${parmfile}"
-
-	[ -n "${HKD_PATH:-}" ] || (echo >&2 "No host key document specified." && return 1)
-	cert_list=($(ls -1 $HKD_PATH))
-	declare hkd_options
-	eval "for cert in ${cert_list[*]}; do
-		hkd_options+=\"--host-key-document=\\\"\$HKD_PATH/\$cert\\\" \"
-	done"
-
-	command -v genprotimg > /dev/null 2>&1 || { apt update; apt install -y s390-tools; }
-	extra_arguments=""
-	genprotimg_version=$(genprotimg --version | grep -Po '(?<=version )[^-]+')
-	if ! version_greater_than_equal "${genprotimg_version}" "2.17.0"; then
-		extra_arguments="--x-pcf '0xe0'"
-	fi
-
-	eval genprotimg \
-		"${extra_arguments}" \
-		"${hkd_options}" \
-		--output="${install_dest_dir}/kata-containers-secure.img" \
-		--image="${install_src_dir}/vmlinuz.container" \
-		--ramdisk="${install_src_dir}/kata-containers-initrd.img" \
-		--parmfile="${parmfile}" \
-		--no-verify # no verification for CI testing purposes
-
-	build_result=$?
-	rm -f "${parmfile}"
-	if [ $build_result -eq 0 ]; then
-		return 0
-	else
-		return 1
-	fi
-}

docs/Developer-Guide.md

@@ -587,10 +587,15 @@ $ sudo kata-monitor
 
 #### Connect to debug console
 
-Command `kata-runtime exec` is used to connect to the debug console.
+You need to start a container for example:
+```bash
+$ sudo ctr run  --runtime io.containerd.kata.v2 -d docker.io/library/ubuntu:latest testdebug
+```
+
+Then, you can use the command `kata-runtime exec <sandbox id>` to connect to the debug console.
 
 ```
-$ kata-runtime exec 1a9ab65be63b8b03dfd0c75036d27f0ed09eab38abb45337fea83acd3cd7bacd
+$ kata-runtime exec testdebug
 bash-4.2# id
 uid=0(root) gid=0(root) groups=0(root)
 bash-4.2# pwd

docs/Limitations.md

@@ -147,7 +147,8 @@ these commands is potentially challenging.
 See issue https://github.com/clearcontainers/runtime/issues/341 and [the constraints challenge](#the-constraints-challenge) for more information.
 
 For CPUs resource management see
-[CPU constraints](design/vcpu-handling.md).
+[CPU constraints(in runtime-go)](design/vcpu-handling-runtime-go.md).
+[CPU constraints(in runtime-rs)](design/vcpu-handling-runtime-rs.md).
 
 # Architectural limitations
 

docs/design/README.md

@@ -6,16 +6,19 @@ Kata Containers design documents:
 - [API Design of Kata Containers](kata-api-design.md)
 - [Design requirements for Kata Containers](kata-design-requirements.md)
 - [VSocks](VSocks.md)
-- [VCPU handling](vcpu-handling.md)
+- [VCPU handling(in runtime-go)](vcpu-handling-runtime-go.md)
+- [VCPU handling(in runtime-rs)](vcpu-handling-runtime-rs.md)
 - [VCPU threads pinning](vcpu-threads-pinning.md)
 - [Host cgroups](host-cgroups.md)
 - [Agent systemd cgroup](agent-systemd-cgroup.md)
 - [`Inotify` support](inotify.md)
 - [`Hooks` support](hooks-handling.md)
 - [Metrics(Kata 2.0)](kata-2-0-metrics.md)
+- [Metrics in Rust Runtime(runtime-rs)](kata-metrics-in-runtime-rs.md)
 - [Design for Kata Containers `Lazyload` ability with `nydus`](kata-nydus-design.md)
 - [Design for direct-assigned volume](direct-blk-device-assignment.md)
 - [Design for core-scheduling](core-scheduling.md)
+- [Virtualization Reference Architecture](kata-vra.md)
 ---
 
 - [Design proposals](proposals)

docs/design/VSocks.md

@@ -78,4 +78,4 @@ with the containers is if the VM itself or the `containerd-shim-kata-v2` dies, i
 the containers are removed automatically.
 
 [1]: https://wiki.qemu.org/Features/VirtioVsock
-[2]: ./vcpu-handling.md#virtual-cpus-and-kubernetes-pods
+[2]: ./vcpu-handling-runtime-go.md#virtual-cpus-and-kubernetes-pods

docs/design/architecture/kubernetes.md

@@ -3,11 +3,11 @@
 [Kubernetes](https://github.com/kubernetes/kubernetes/), or K8s, is a popular open source
 container orchestration engine. In Kubernetes, a set of containers sharing resources
 such as networking, storage, mount, PID, etc. is called a
-[pod](https://kubernetes.io/docs/user-guide/pods/).
+[pod](https://kubernetes.io/docs/concepts/workloads/pods/).
 
 A node can have multiple pods, but at a minimum, a node within a Kubernetes cluster
 only needs to run a container runtime and a container agent (called a
-[Kubelet](https://kubernetes.io/docs/admin/kubelet/)).
+[Kubelet](https://kubernetes.io/docs/concepts/overview/components/#kubelet)).
 
 Kata Containers represents a Kubelet pod as a VM.
 

docs/design/kata-metrics-in-runtime-rs.md

@@ -0,0 +1,50 @@
+# Kata Metrics in Rust Runtime(runtime-rs)
+
+Rust Runtime(runtime-rs) is responsible for:
+
+- Gather metrics about `shim`.
+- Gather metrics from `hypervisor` (through `channel`).
+- Get metrics from `agent` (through `ttrpc`).
+
+---
+
+Here are listed all the metrics gathered by `runtime-rs`.
+
+> * Current status of each entry is marked as:
+>  * ✅：DONE
+>   * 🚧：TODO
+
+### Kata Shim
+
+| STATUS | Metric name                                                  | Type        | Units          | Labels                                                       |
+| ------ | ------------------------------------------------------------ | ----------- | -------------- | ------------------------------------------------------------ |
+| 🚧      | `kata_shim_agent_rpc_durations_histogram_milliseconds`: <br> RPC latency distributions. | `HISTOGRAM` | `milliseconds` | <ul><li>`action` (RPC actions of Kata agent)<ul><li>`grpc.CheckRequest`</li><li>`grpc.CloseStdinRequest`</li><li>`grpc.CopyFileRequest`</li><li>`grpc.CreateContainerRequest`</li><li>`grpc.CreateSandboxRequest`</li><li>`grpc.DestroySandboxRequest`</li><li>`grpc.ExecProcessRequest`</li><li>`grpc.GetMetricsRequest`</li><li>`grpc.GuestDetailsRequest`</li><li>`grpc.ListInterfacesRequest`</li><li>`grpc.ListProcessesRequest`</li><li>`grpc.ListRoutesRequest`</li><li>`grpc.MemHotplugByProbeRequest`</li><li>`grpc.OnlineCPUMemRequest`</li><li>`grpc.PauseContainerRequest`</li><li>`grpc.RemoveContainerRequest`</li><li>`grpc.ReseedRandomDevRequest`</li><li>`grpc.ResumeContainerRequest`</li><li>`grpc.SetGuestDateTimeRequest`</li><li>`grpc.SignalProcessRequest`</li><li>`grpc.StartContainerRequest`</li><li>`grpc.StatsContainerRequest`</li><li>`grpc.TtyWinResizeRequest`</li><li>`grpc.UpdateContainerRequest`</li><li>`grpc.UpdateInterfaceRequest`</li><li>`grpc.UpdateRoutesRequest`</li><li>`grpc.WaitProcessRequest`</li><li>`grpc.WriteStreamRequest`</li></ul></li><li>`sandbox_id`</li></ul> |
+| ✅      | `kata_shim_fds`: <br> Kata containerd shim v2 open FDs.      | `GAUGE`     |                | <ul><li>`sandbox_id`</li></ul>                               |
+| ✅      | `kata_shim_io_stat`: <br> Kata containerd shim v2 process IO statistics. | `GAUGE`     |                | <ul><li>`item` (see `/proc/<pid>/io`)<ul><li>`cancelledwritebytes`</li><li>`rchar`</li><li>`readbytes`</li><li>`syscr`</li><li>`syscw`</li><li>`wchar`</li><li>`writebytes`</li></ul></li><li>`sandbox_id`</li></ul> |
+| ✅      | `kata_shim_netdev`: <br> Kata containerd shim v2 network devices statistics. | `GAUGE`     |                | <ul><li>`interface` (network device name)</li><li>`item` (see `/proc/net/dev`)<ul><li>`recv_bytes`</li><li>`recv_compressed`</li><li>`recv_drop`</li><li>`recv_errs`</li><li>`recv_fifo`</li><li>`recv_frame`</li><li>`recv_multicast`</li><li>`recv_packets`</li><li>`sent_bytes`</li><li>`sent_carrier`</li><li>`sent_colls`</li><li>`sent_compressed`</li><li>`sent_drop`</li><li>`sent_errs`</li><li>`sent_fifo`</li><li>`sent_packets`</li></ul></li><li>`sandbox_id`</li></ul> |
+| 🚧      | `kata_shim_pod_overhead_cpu`: <br> Kata Pod overhead for CPU resources(percent). | `GAUGE`     | percent        | <ul><li>`sandbox_id`</li></ul>                               |
+| 🚧      | `kata_shim_pod_overhead_memory_in_bytes`: <br> Kata Pod overhead for memory resources(bytes). | `GAUGE`     | `bytes`        | <ul><li>`sandbox_id`</li></ul>                               |
+| ✅      | `kata_shim_proc_stat`: <br> Kata containerd shim v2 process statistics. | `GAUGE`     |                | <ul><li>`item` (see `/proc/<pid>/stat`)<ul><li>`cstime`</li><li>`cutime`</li><li>`stime`</li><li>`utime`</li></ul></li><li>`sandbox_id`</li></ul> |
+| ✅      | `kata_shim_proc_status`: <br> Kata containerd shim v2 process status. | `GAUGE`     |                | <ul><li>`item` (see `/proc/<pid>/status`)<ul><li>`hugetlbpages`</li><li>`nonvoluntary_ctxt_switches`</li><li>`rssanon`</li><li>`rssfile`</li><li>`rssshmem`</li><li>`vmdata`</li><li>`vmexe`</li><li>`vmhwm`</li><li>`vmlck`</li><li>`vmlib`</li><li>`vmpeak`</li><li>`vmpin`</li><li>`vmpmd`</li><li>`vmpte`</li><li>`vmrss`</li><li>`vmsize`</li><li>`vmstk`</li><li>`vmswap`</li><li>`voluntary_ctxt_switches`</li></ul></li><li>`sandbox_id`</li></ul> |
+| 🚧      | `kata_shim_process_cpu_seconds_total`: <br> Total user and system CPU time spent in seconds. | `COUNTER`   | `seconds`      | <ul><li>`sandbox_id`</li></ul>                               |
+| 🚧      | `kata_shim_process_max_fds`: <br> Maximum number of open file descriptors. | `GAUGE`     |                | <ul><li>`sandbox_id`</li></ul>                               |
+| 🚧      | `kata_shim_process_open_fds`: <br> Number of open file descriptors. | `GAUGE`     |                | <ul><li>`sandbox_id`</li></ul>                               |
+| 🚧      | `kata_shim_process_resident_memory_bytes`: <br> Resident memory size in bytes. | `GAUGE`     | `bytes`        | <ul><li>`sandbox_id`</li></ul>                               |
+| 🚧      | `kata_shim_process_start_time_seconds`: <br> Start time of the process since `unix` epoch in seconds. | `GAUGE`     | `seconds`      | <ul><li>`sandbox_id`</li></ul>                               |
+| 🚧      | `kata_shim_process_virtual_memory_bytes`: <br> Virtual memory size in bytes. | `GAUGE`     | `bytes`        | <ul><li>`sandbox_id`</li></ul>                               |
+| 🚧      | `kata_shim_process_virtual_memory_max_bytes`: <br> Maximum amount of virtual memory available in bytes. | `GAUGE`     | `bytes`        | <ul><li>`sandbox_id`</li></ul>                               |
+| 🚧      | `kata_shim_rpc_durations_histogram_milliseconds`: <br> RPC latency distributions. | `HISTOGRAM` | `milliseconds` | <ul><li>`action` (Kata shim v2 actions)<ul><li>`checkpoint`</li><li>`close_io`</li><li>`connect`</li><li>`create`</li><li>`delete`</li><li>`exec`</li><li>`kill`</li><li>`pause`</li><li>`pids`</li><li>`resize_pty`</li><li>`resume`</li><li>`shutdown`</li><li>`start`</li><li>`state`</li><li>`stats`</li><li>`update`</li><li>`wait`</li></ul></li><li>`sandbox_id`</li></ul> |
+| ✅      | `kata_shim_threads`: <br> Kata containerd shim v2 process threads. | `GAUGE`     |                | <ul><li>`sandbox_id`</li></ul>                               |
+
+### Kata Hypervisor
+
+Different from golang runtime, hypervisor and shim in runtime-rs belong to the **same process**, so all previous metrics for hypervisor and shim only need to be gathered once. Thus, we currently only collect previous metrics in kata shim.
+
+At the same time, we added the interface(`VmmAction::GetHypervisorMetrics`) to gather hypervisor metrics, in case we design tailor-made metrics for hypervisor in the future. Here're metrics exposed from [src/dragonball/src/metric.rs](https://github.com/kata-containers/kata-containers/blob/main/src/dragonball/src/metric.rs).
+
+| Metric name                                                  | Type       | Units | Labels                                                       |
+| ------------------------------------------------------------ | ---------- | ----- | ------------------------------------------------------------ |
+| `kata_hypervisor_scrape_count`: <br> Metrics scrape count    | `COUNTER`  |       | <ul><li>`sandbox_id`</li></ul>                               |
+| `kata_hypervisor_vcpu`: <br>Hypervisor metrics specific to VCPUs' mode of functioning. | `IntGauge` |       | <ul><li>`item`<ul><li>`exit_io_in`</li><li>`exit_io_out`</li><li>`exit_mmio_read`</li><li>`exit_mmio_write`</li><li>`failures`</li><li>`filter_cpuid`</li></ul></li><li>`sandbox_id`</li></ul> |
+| `kata_hypervisor_seccomp`: <br> Hypervisor metrics for the seccomp filtering. | `IntGauge` |       | <ul><li>`item`<ul><li>`num_faults`</li></ul></li><li>`sandbox_id`</li></ul> |
+| `kata_hypervisor_seccomp`: <br> Hypervisor metrics for the seccomp filtering. | `IntGauge` |       | <ul><li>`item`<ul><li>`sigbus`</li><li>`sigsegv`</li></ul></li><li>`sandbox_id`</li></ul> |

docs/design/kata-vra.md

@@ -0,0 +1,434 @@
+# Virtualization Reference Architecture
+
+## Subject to Change | © 2022 by NVIDIA Corporation. All rights reserved. | For test and development only_
+
+Before digging deeper into the virtualization reference architecture, let's
+first look at the various GPUDirect use cases in the following table. We’re
+distinguishing between two top-tier use cases where the devices are (1)
+passthrough and (2) virtualized, where a VM gets assigned a virtual function
+(VF) and not the physical function (PF). A combination of PF and VF would also
+be possible.
+
+| Device #1  (passthrough)  | Device #2 (passthrough) | P2P Compatibility and Mode                   |
+| ------------------------- | ----------------------- | -------------------------------------------- |
+| GPU PF                    | GPU PF                  | GPUDirect P2P                                |
+| GPU PF                    | NIC PF                  | GPUDirect RDMA                               |
+| MIG-slice                 | MIG-slice               | _No GPUDirect P2P_                           |
+| MIG-slice                 | NIC PF                  | GPUDirect RDMA                               |
+| **PDevice #1  (virtualized)** | **Device #2 (virtualized)** | **P2P Compatibility and   Mode**     |
+| Time-slice vGPU VF        | Time-slice vGPU VF      | _No GPUDirect P2P  but NVLINK P2P available_ |
+| Time-slice vGPU VF        | NIC VF                  | GPUDirect RDMA                               |
+| MIG-slice vGPU            | MIG-slice vGPU          | _No GPUDirect P2P_                           |
+| MIG-slice vGPU            | NIC VF                  | GPUDirect RDMA                               |
+
+In a virtualized environment we have several distinct features that may prevent
+Peer-to-peer (P2P) communication of two endpoints in a PCI Express topology. The
+IOMMU translates IO virtual addresses (IOVA) to physical addresses (PA). Each
+device behind an IOMMU has its own IOVA memory space, usually, no two devices
+share the same IOVA memory space but it’s up to the hypervisor or OS how it
+chooses to map devices to IOVA spaces.  Any PCI Express DMA transactions will
+use IOVAs, which the IOMMU must translate. By default, all the traffic is routed
+to the root complex and not issued directly to the peer device.
+
+An IOMMU can be used to isolate and protect devices even if virtualization is
+not used; since devices can only access memory regions that are mapped for it, a
+DMA from one device to another is not possible. DPDK uses the IOMMU to have
+better isolation between devices, another benefit is that IOVA space can be
+represented as a contiguous memory even if the PA space is heavily scattered.
+
+In the case of virtualization, the IOMMU is responsible for isolating the device
+and memory between VMs for safe device assignment without compromising the host
+and other guest OSes. Without an IOMMU, any device can access the entire system
+and perform DMA transactions _anywhere_.
+
+The second feature is ACS (Access Control Services), which controls which
+devices are allowed to communicate with one another and thus avoids improper
+routing of packets irrespectively of whether IOMMU is enabled or not.
+
+When IOMMU is enabled, ACS is normally configured to force all PCI Express DMA
+to go through the root complex so IOMMU can translate it, impacting performance
+between peers with higher latency and reduced bandwidth.
+
+A way to avoid the performance hit is to enable Address Translation Services
+(ATS). ATS-capable endpoints can prefetch IOVA -> PA translations from the IOMMU
+and then perform DMA transactions directly to another endpoint. Hypervisors
+enable this by enabling ATS in such endpoints, configuring ACS to enable Direct
+Translated P2P, and configuring the IOMMU to allow Address Translation requests.
+
+Another important factor is that the NVIDIA driver stack will use the PCI
+Express topology of the system it is running on to determine whether the
+hardware is capable of supporting P2P. The driver stack qualifies specific
+chipsets, and PCI Express switches for use with GPUDirect P2P. In virtual
+environments, the PCI Express topology is flattened and obfuscated to present a
+uniform environment to the software inside the VM, which breaks the GPUDirect
+P2P use case.
+
+On a bare metal machine, the driver stack groups GPUs into cliques that can
+perform GPUDirect P2P communication, excluding peer mappings where P2P
+communication is not possible, prominently if GPUs are attached to multiple CPU
+sockets.  
+
+CPUs and local memory banks are referred to as NUMA nodes. In a two-socket
+server, each of the CPUs has a local memory bank for a total of two NUMA nodes.
+Some servers provide the ability to configure additional NUMA nodes per CPU,
+which means a CPU socket can have two NUMA nodes  (some servers support four
+NUMA nodes per socket) with local memory banks and L3 NUMA domains for improved
+performance.
+
+One of the current solutions is that the hypervisor provides additional topology
+information that the driver stack can pick up and enable GPUDirect P2P between
+GPUs, even if the virtualized environment does not directly expose it. The PCI
+Express virtual P2P approval capability structure in the PCI configuration space
+is entirely emulated by the hypervisor of passthrough GPU devices.
+
+A clique ID is provided where GPUs with the same clique ID belong to a group of
+GPUs capable of P2P communication
+
+On vSphere, Azure, and other CPSs,  the hypervisor lays down a `topologies.xml`
+which NCCL can pick up and deduce the right P2P level[^1]. NCCL is leveraging
+Infiniband (IB) and/or Unified Communication X (UCX) for communication, and
+GPUDirect P2P and GPUDirect RDMA should just work in this case. The only culprit
+is that software or applications that do not use the XML file to deduce the
+topology will fail and not enable GPUDirect ( [`nccl-p2p-level`](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/env.html#nccl-p2p-level) )
+
+## Hypervisor PCI Express Topology
+
+To enable every part of the accelerator stack, we propose a virtualized
+reference architecture to enable GPUDirect P2P and GPUDirect RDMA for any
+hypervisor. The idea is split into two parts to enable the right PCI Express
+topology. The first part builds upon extending the PCI Express virtual P2P
+approval capability structure to every device that wants to do P2P in some way
+and groups devices by clique ID. The other part involves replicating a subset of
+the host topology so that applications running in the VM do not need to read
+additional information and enable the P2P capability like in the bare-metal use
+case described above. The driver stack can then deduce automatically if the
+topology presented in the VM is capable of P2P communication.
+
+We will work with the following host topology for the following sections. It is
+a system with two converged DPUs, each having an `A100X` GPU and two `ConnectX-6`
+network ports connected to the downstream ports of a PCI Express switch.
+
+```sh
++-00.0-[d8-df]----00.0-[d9-df]--+-00.0-[da-db]--+-00.0  Mellanox Tech MT42822 BlueField-2 integrated ConnectX-6 Dx network
+                                |               +-00.1  Mellanox Tech MT42822 BlueField-2 integrated ConnectX-6 Dx network
+                                |               \-00.2  Mellanox Tech MT42822 BlueField-2 SoC Management Interface
+                                 \-01.0-[dc-df]----00.0-[dd-df]----08.0-[de-df]----00.0  NVIDIA Corporation GA100 [A100X]
+
++-00.0-[3b-42]----00.0-[3c-42]--+-00.0-[3d-3e]--+-00.0  Mellanox Tech MT42822 BlueField-2 integrated ConnectX-6 Dx network
+                                |               +-00.1  Mellanox Tech MT42822 BlueField-2 integrated ConnectX-6 Dx network
+                                |               \-00.2  Mellanox Tech MT42822 BlueField-2 SoC Management Interface
+                                 \-01.0-[3f-42]----00.0-[40-42]----08.0-[41-42]----00.0  NVIDIA Corporation GA100 [A100X]
+```
+
+The green path highlighted above is the optimal and preferred path for
+efficient P2P communication.
+
+## PCI Express Virtual P2P Approval Capability
+
+Most of the time, the PCI Express topology is flattened and obfuscated to ensure
+easy migration of the VM image between different physical hardware topologies.
+In Kata, we can configure the hypervisor to use PCI Express root ports to
+hotplug the VFIO  devices one is passing through. A user can select how many PCI
+Express root ports to allocate depending on how many devices are passed through.
+A recent addition to Kata will detect the right amount of PCI Express devices
+that need hotplugging and bail out if the number of root ports is insufficient.
+In Kata, we do not automatically increase the number of root ports, we want the
+user to be in full control of the topology.
+
+```toml
+# /etc/kata-containers/configuration.toml
+
+# VFIO devices are hotplugged on a bridge by default.
+# Enable hot-plugging on the root bus. This may be required for devices with
+# a large PCI bar, as this is a current limitation with hot-plugging on
+# a bridge.
+# Default “bridge-port”
+hotplug_vfio = "root-port"
+
+# Before hot plugging a PCIe device, you need to add a pcie_root_port device.
+# Use this parameter when using some large PCI bar devices, such as NVIDIA GPU
+# The value means the number of pcie_root_port
+# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35"
+# Default 0
+pcie_root_port = 8
+```
+
+VFIO devices are hotplugged on a PCIe-PCI bridge by default. Hotplug of PCI
+Express devices is only supported on PCI Express root or downstream ports. With
+this configuration set, if we start up a Kata container, we can inspect our
+topology and see the allocated PCI Express root ports and the hotplugged
+devices.
+
+```sh
+$ lspci -tv
+ -[0000:00]-+-00.0  Intel Corporation 82G33/G31/P35/P31 Express DRAM Controller
+           +-01.0  Red Hat, Inc. Virtio console
+           +-02.0  Red Hat, Inc. Virtio SCSI
+           +-03.0  Red Hat, Inc. Virtio RNG
+           +-04.0-[01]----00.0  Mellanox Technologies MT42822 BlueField-2 integrated ConnectX-6
+           +-05.0-[02]----00.0  Mellanox Technologies MT42822 BlueField-2 integrated ConnectX-6
+           +-06.0-[03]----00.0  NVIDIA Corporation Device 20b8
+           +-07.0-[04]----00.0  NVIDIA Corporation Device 20b8
+           +-08.0-[05]--
+           +-09.0-[06]--
+           +-0a.0-[07]--
+           +-0b.0-[08]--
+           +-0c.0  Red Hat, Inc. Virtio socket
+           +-0d.0  Red Hat, Inc. Virtio file system
+           +-1f.0  Intel Corporation 82801IB (ICH9) LPC Interface Controller
+           +-1f.2  Intel Corporation 82801IR/IO/IH (ICH9R/DO/DH) 6 port SATA Controller
+           \-1f.3  Intel Corporation 82801I (ICH9 Family) SMBus Controller
+```
+
+For devices with huge BARs (Base Address Registers) like the GPU (we need to
+configure the PCI Express root port properly and allocate enough memory for
+mapping), we have added a heuristic to Kata to deduce the right settings. Hence,
+the BARs can be mapped correctly. This functionality is added to
+[`nvidia/go-nvlib1](https://gitlab.com/nvidia/cloud-native/go-nvlib) which is part
+of Kata now.
+
+```sh
+$ sudo dmesg | grep BAR
+[    0.179960] pci 0000:00:04.0: BAR 7: assigned [io  0x1000-0x1fff]
+[    0.179962] pci 0000:00:05.0: BAR 7: assigned [io  0x2000-0x2fff]
+[    0.179963] pci 0000:00:06.0: BAR 7: assigned [io  0x3000-0x3fff]
+[    0.179964] pci 0000:00:07.0: BAR 7: assigned [io  0x4000-0x4fff]
+[    0.179966] pci 0000:00:08.0: BAR 7: assigned [io  0x5000-0x5fff]
+[    0.179967] pci 0000:00:09.0: BAR 7: assigned [io  0x6000-0x6fff]
+[    0.179968] pci 0000:00:0a.0: BAR 7: assigned [io  0x7000-0x7fff]
+[    0.179969] pci 0000:00:0b.0: BAR 7: assigned [io  0x8000-0x8fff]
+[    2.115912] pci 0000:01:00.0: BAR 0: assigned [mem 0x13000000000-0x13001ffffff 64bit pref]
+[    2.116203] pci 0000:01:00.0: BAR 2: assigned [mem 0x13002000000-0x130027fffff 64bit pref]
+[    2.683132] pci 0000:02:00.0: BAR 0: assigned [mem 0x12000000000-0x12001ffffff 64bit pref]
+[    2.683419] pci 0000:02:00.0: BAR 2: assigned [mem 0x12002000000-0x120027fffff 64bit pref]
+[    2.959155] pci 0000:03:00.0: BAR 1: assigned [mem 0x11000000000-0x117ffffffff 64bit pref]
+[    2.959345] pci 0000:03:00.0: BAR 3: assigned [mem 0x11800000000-0x11801ffffff 64bit pref]
+[    2.959523] pci 0000:03:00.0: BAR 0: assigned [mem 0xf9000000-0xf9ffffff]
+[    2.966119] pci 0000:04:00.0: BAR 1: assigned [mem 0x10000000000-0x107ffffffff 64bit pref]
+[    2.966295] pci 0000:04:00.0: BAR 3: assigned [mem 0x10800000000-0x10801ffffff 64bit pref]
+[    2.966472] pci 0000:04:00.0: BAR 0: assigned [mem 0xf7000000-0xf7ffffff]
+```
+
+The NVIDIA driver stack in this case would refuse to do P2P communication since
+(1) the topology is not what it expects, (2)  we do not have a qualified
+chipset. Since our P2P devices are not connected to a PCI Express switch port,
+we need to provide additional information to support the P2P functionality. One
+way of providing such meta information would be to annotate the container; most
+of the settings in Kata's configuration file can be overridden via annotations,
+but this limits the flexibility, and a user would need to update all the
+containers that he wants to run with Kata. The goal is to make such things as
+transparent as possible, so we also introduced
+[CDI](https://github.com/container-orchestrated-devices/container-device-interface)
+(Container Device Interface) to Kata. CDI is a[
+specification](https://github.com/container-orchestrated-devices/container-device-interface/blob/master/SPEC.md)
+for container runtimes to support third-party devices.
+
+As written before, we can provide a clique ID for the devices that belong
+together and are capable of doing P2P. This information is provided to the
+hypervisor, which will set up things in the VM accordingly. Let's suppose the
+user wanted to do GPUDirect RDMA with the first GPU and the NIC that reside on
+the same DPU, one could provide the specification telling the hypervisor that
+they belong to the same clique.
+
+```yaml
+# /etc/cdi/nvidia.yaml
+cdiVersion: 0.4.0
+kind: nvidia.com/gpu
+devices:
+- name: gpu0
+  annotations:
+    bdf: “41:00.0”
+    clique-id: “0”
+  containerEdits:
+    deviceNodes:
+    - path: “/dev/vfio/71"
+
+# /etc/cdi/mellanox.yaml
+cdiVersion: 0.4.0
+kind: mellanox.com/nic
+devices:
+- name: nic0
+  annotations:
+    bdf: “3d:00.0”
+    clique-id: “0”
+    attach-pci: “true”
+  containerEdits:
+    deviceNodes:
+    - path: "/dev/vfio/66"
+```
+
+Since this setting is bound to the device and not the container we do not need
+to alter the container just allocate the right resource and GPUDirect RDMA would
+be set up correctly. Rather than exposing them separately, an idea would be to
+expose a GPUDirect RDMA device via NFD (Node Feature Discovery) that combines
+both of them; this way, we could make sure that the right pair is allocated and
+used more on  Kubernetes deployment in the next section.
+
+The GPU driver stack is leveraging the PCI Express virtual P2P approval
+capability, but the NIC stack does not use this now. One of the action items is
+to enable MOFED to read the P2P approval capability and enable ATS and ACS
+settings as described above.
+
+This way, we could enable GPUDirect P2P and GPUDirect RDMA on any topology
+presented to the VM application. It is the responsibility of the administrator
+or infrastructure engineer to provide the right information either via
+annotations or a CDI specification.
+
+## Host Topology Replication
+
+The other way to represent the PCI Express topology in the VM is to replicate a
+subset of the topology needed to support the P2P use case inside the VM. Similar
+to the configuration for the root ports, we can easily configure the usage of
+PCI Express switch ports to hotplug the devices.
+
+```toml
+# /etc/kata-containers/configuration.toml
+
+# VFIO devices are hotplugged on a bridge by default.
+# Enable hot plugging on the root bus. This may be required for devices with
+# a large PCI bar, as this is a current limitation with hot plugging on
+# a bridge.
+# Default “bridge-port”
+hotplug_vfio = "switch-port"
+
+# Before hot plugging a PCIe device, you need to add a pcie_root_port device.
+# Use this parameter when using some large PCI bar devices, such as Nvidia GPU
+# The value means the number of pcie_root_port
+# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35"
+# Default 0
+pcie_switch_port = 8
+```
+
+Each device that is passed through is attached to a PCI Express downstream port
+as illustrated below. We can even replicate the host’s two DPUs topologies with
+added metadata through the CDI. Most of the time, a container only needs one
+pair of GPU and NIC for GPUDirect RDMA. This is more of a showcase of what we
+can do with the power of Kata and CDI. One could even think of adding groups of
+devices that support P2P, even from different CPU sockets or NUMA nodes, into
+one container; indeed, the first group is NUMA node 0 (red), and the second
+group is NUMA node 1 (green). Since they are grouped correctly, P2P would be
+enabled naturally inside a group, aka clique ID.
+
+```sh
+$ lspci -tv
+ -[0000:00]-+-00.0  Intel Corporation 82G33/G31/P35/P31 Express DRAM Controller
+            +-01.0  Red Hat, Inc. Virtio console
+            +-02.0  Red Hat, Inc. Virtio SCSI
+            +-03.0  Red Hat, Inc. Virtio RNG
+            +-04.0-[01-04]----00.0-[02-04]--+-00.0-[03]----00.0  NVIDIA Corporation Device 20b8
+            |                               \-01.0-[04]----00.0  Mellanox Tech MT42822 BlueField-2 integrated ConnectX-6 Dx
+            +-05.0-[05-08]----00.0-[06-08]--+-00.0-[07]----00.0  Mellanox Tech MT42822 BlueField-2 integrated ConnectX-6 Dx
+            |                               \-01.0-[08]----00.0  NVIDIA Corporation Device 20b8
+            +-06.0  Red Hat, Inc. Virtio socket
+            +-07.0  Red Hat, Inc. Virtio file system
+            +-1f.0  Intel Corporation 82801IB (ICH9) LPC Interface Controller
+            +-1f.2  Intel Corporation 82801IR/IO/IH (ICH9R/DO/DH) 6 port SATA Controller [AHCI mode]
+            \-1f.3  Intel Corporation 82801I (ICH9 Family) SMBus Controller
+            \-1f.3  Intel Corporation 82801I (ICH9 Family) SMBus Controller
+```
+
+The configuration of using either the root port or switch port can be applied on
+a per Container or Pod basis, meaning we can switch PCI Express topologies on
+each run of an application.
+
+## Hypervisor Resource Limits
+
+Every hypervisor will have resource limits in terms of how many PCI Express root
+ports, switch ports, or bridge ports can be created, especially with devices
+that need to reserve a 4K IO range per PCI specification. Each instance of root
+or switch port will consume 4K IO of very limited capacity, 64k is the maximum.
+
+Simple math brings us to the conclusion that we can have a maximum of 16 PCI
+Express root ports or 16 PCI Express switch ports in QEMU if devices with IO
+BARs are used in the PCI Express hierarchy.
+
+Additionally, one can have 32 slots on the PCI root bus and a maximum of 256
+slots for the complete PCI(e) topology.
+
+Per default, QEMU will attach a multi-function device in the last slot on the
+PCI root bus,
+
+```sh
+ +-1f.0  Intel Corporation 82801IB (ICH9) LPC Interface Controller
+ +-1f.2  Intel Corporation 82801IR/IO/IH (ICH9R/DO/DH) 6 port SATA Controller [AHCI mode]
+ \-1f.3  Intel Corporation 82801I (ICH9 Family) SMBus Controller
+```
+
+Kata will additionally add `virtio-xxx-pci` devices consuming (5 slots) plus a
+PCIe-PCI-bridge (1 slot) and a DRAM controller (1 slot), meaning per default, we
+have already eight slots used. This leaves us 24 slots for adding other devices
+to the root bus.
+
+The problem that arises here is one use-case from a customer that uses recent
+RTX GPUs with Kata. The user wanted to pass through eight of these GPUs into one
+container and ran into issues. The problem is that those cards often consist of
+four individual device nodes: GPU, Audio, and two USB controller devices (some
+cards have a USB-C output).
+
+These devices are grouped into one IOMMU group. Since one needs to pass through
+the complete IOMMU group into the VM, we need to allocate 32 PCI Express root
+ports or 32 PCI Express switch ports, which is technically impossible due to the
+resource limits outlined above. Since all the devices appear as PCI Express
+devices, we need to hotplug those into a root or switch port.
+
+The solution to this problem is leveraging CDI. For each device, add the
+information if it is going to be hotplugged as a PCI Express or PCI device,
+which results in either using a PCI Express root/switch port or an ordinary PCI
+bridge. PCI bridges are not affected by the limited IO range. This way, the GPU
+is attached as a PCI Express device to a root/switch port and the other three
+PCI devices to a PCI bridge, leaving enough resources to create the needed PCI
+Express root/switch ports.  For example, we’re going to attach the GPUs to a PCI
+Express root port and the NICs to a PCI bridge.
+
+```jsonld
+# /etc/cdi/mellanox.json
+cdiVersion: 0.4.0
+kind: mellanox.com/nic
+devices:
+- name: nic0
+  annotations:
+    bdf: “3d:00.0”
+    clique-id: “0”
+    attach-pci: “true”
+  containerEdits:
+    deviceNodes:
+    - path: "/dev/vfio/66"
+- name: nic1
+  annotations:
+    bdf: “3d:00.1”
+    clique-id: “1”
+    attach-pci: “true”
+  containerEdits:
+    deviceNodes:
+    - path: "/dev/vfio/67”
+```
+
+The configuration is set to use eight root ports for the GPUs and attach the
+NICs to a PCI bridge which is connected to a PCI Express-PCI bridge which is the
+preferred way of introducing a PCI topology in a PCI Express machine.
+
+```sh
+$ lspci -tv
+-[0000:00]-+-00.0  Intel Corporation 82G33/G31/P35/P31 Express DRAM Controller
+           +-01.0  Red Hat, Inc. Virtio console
+           +-02.0  Red Hat, Inc. Virtio SCSI
+           +-03.0  Red Hat, Inc. Virtio RNG
+           +-04.0-[01]----00.0  NVIDIA Corporation Device 20b8
+           +-05.0-[02]----00.0  NVIDIA Corporation Device 20b8
+           +-06.0-[03]--
+           +-07.0-[04]--
+           +-08.0-[05]--
+           +-09.0-[06]--
+           +-0a.0-[07]--
+           +-0b.0-[08]--
+           +-0c.0-[09-0a]----00.0-[0a]--+-00.0  Mellanox Tech MT42822 BlueField-2 ConnectX-6
+           |                             \-01.0  Mellanox Tech MT42822 BlueField-2 ConnectX-6
+           +-0d.0  Red Hat, Inc. Virtio socket
+           +-0e.0  Red Hat, Inc. Virtio file system
+           +-1f.0  Intel Corporation 82801IB (ICH9) LPC Interface Controller
+           +-1f.2  Intel Corporation 82801IR/IO/IH (ICH9R/DO/DH) 6 port SATA Controller
+           \-1f.3  Intel Corporation 82801I (ICH9 Family) SMBus Controller
+```
+
+The PCI devices will consume a slot of which we have 256 in the PCI(e) topology
+and leave scarce resources for the needed PCI Express devices.

docs/design/vcpu-handling-runtime-rs.md

@@ -0,0 +1,51 @@
+# Virtual machine vCPU sizing in Kata Containers 3.0
+
+> Preview: 
+> [Kubernetes(since 1.23)][1] and [Containerd(since 1.6.0-beta4)][2] will help calculate `Sandbox Size` info and pass it to Kata Containers through annotations.
+> In order to adapt to this beneficial change and be compatible with the past, we have implemented the new vCPUs handling way in `runtime-rs`, which is slightly different from the original `runtime-go`'s design.
+
+## When do we need to handle vCPUs size?
+vCPUs sizing should be determined by the container workloads. So throughout the life cycle of Kata Containers, there are several points in time when we need to think about how many vCPUs should be at the time. Mainly including the time points of `CreateVM`, `CreateContainer`, `UpdateContainer`, and `DeleteContainer`.
+* `CreateVM`: When creating a sandbox, we need to know how many vCPUs to start the VM with.
+* `CreateContainer`: When creating a new container in the VM, we may need to hot-plug the vCPUs according to the requirements in container's spec.
+* `UpdateContainer`: When receiving the `UpdateContainer` request, we may need to update the vCPU resources according to the new requirements of the container.
+* `DeleteContainer`: When a container is removed from the VM, we may need to hot-unplug the vCPUs to reclaim the vCPU resources introduced by the container.
+
+## On what basis do we calculate the number of vCPUs?
+When Kata calculate the number of vCPUs, We have three data sources, the `default_vcpus` and `default_maxvcpus` specified in the configuration file (named `TomlConfig` later in the doc), the `io.kubernetes.cri.sandbox-cpu-quota` and `io.kubernetes.cri.sandbox-cpu-period` annotations passed by the upper layer runtime, and the corresponding CPU resource part in the container's spec for the container when `CreateContainer`/`UpdateContainer`/`DeleteContainer` is requested.
+
+Our understanding and priority of these resources are as follows, which will affect how we calculate the number of vCPUs later.
+
+* From `TomlConfig`:
+  * `default_vcpus`: default number of vCPUs when starting a VM.
+  * `default_maxvcpus`: maximum number of vCPUs.
+* From `Annotation`:
+  * `InitialSize`: we call the size of the resource passed from the annotations as `InitialSize`. Kubernetes will calculate the sandbox size according to the Pod's statement, which is the `InitialSize` here. This size should be the size we want to prioritize. 
+* From `Container Spec`:
+  * The amount of CPU resources that the Container wants to use will be declared through the spec. Including the aforementioned annotations, we mainly consider `cpu quota` and `cpuset` when calculating the number of vCPUs.
+  * `cpu quota`: `cpu quota` is the most common way to declare the amount of CPU resources. The number of vCPUs introduced by `cpu quota` declared in a container's spec is: `vCPUs = ceiling( quota / period )`.
+  * `cpuset`: `cpuset` is often used to bind the CPUs that tasks can run on. The number of vCPUs may introduced by `cpuset` declared in a container's spec is the number of CPUs specified in the set that do not overlap with other containers.
+
+
+## How to calculate and adjust the vCPUs size:
+There are two types of vCPUs that we need to consider, one is the number of vCPUs when starting the VM (named `Boot Size` in the doc). The second is the number of vCPUs when `CreateContainer`/`UpdateContainer`/`DeleteContainer` request is received (`Real-time Size` in the doc).
+
+### `Boot Size`
+The main considerations are `InitialSize` and `default_vcpus`. There are the following principles:
+`InitialSize` has priority over `default_vcpus` declared in `TomlConfig`.
+1. When there is such an annotation statement, the originally `default_vcpus` will be modified to the number of vCPUs in the `InitialSize` as the `Boot Size`. (Because not all runtimes support this annotation for the time being, we still keep the `default_cpus` in `TomlConfig`.)
+2. When the specs of all containers are aggregated for sandbox size calculation, the method is consistent with the calculation method of `InitialSize` here.
+
+### `Real-time Size`
+When we receive an OCI request, it may be for a single container. But what we have to consider is the number of vCPUs for the entire VM. So we will maintain a list. Every time there is a demand for adjustment, the entire list will be traversed to calculate a value for the number of vCPUs. In addition, there are the following principles:
+1. Do not cut computing power and try to keep the number of vCPUs specified by `InitialSize`.
+   * So the number of vCPUs after will not be less than the `Boot Size`.
+2. `cpu quota` takes precedence over `cpuset` and the setting history are took into account.
+   * We think quota describes the CPU time slice that a cgroup can use, and `cpuset` describes the actual CPU number that a cgroup can use. Quota can better describe the size of the CPU time slice that a cgroup actually wants to use. The `cpuset` only describes which CPUs the cgroup can use, but the cgroup can use the specified CPU but consumes a smaller time slice, so the quota takes precedence over the `cpuset`.
+   * On the one hand, when both `cpu quota` and `cpuset` are specified, we will calculate the number of vCPUs based on `cpu quota` and ignore `cpuset`. On the other hand, if `cpu quota` was used to control the number of vCPUs in the past, and only `cpuset` was updated during `UpdateContainer`, we will not adjust the number of vCPUs at this time.
+3. `StaticSandboxResourceMgmt` controls hotplug.
+   * Some VMMs and kernels of some architectures do not support hotplugging. We can accommodate this situation through `StaticSandboxResourceMgmt`. When `StaticSandboxResourceMgmt = true` is set, we don't make any further attempts to update the number of vCPUs after booting.
+
+
+[1]: https://github.com/kubernetes/kubernetes/pull/104886
+[2]: https://github.com/containerd/containerd/pull/6155

docs/how-to/README.md

@@ -45,6 +45,4 @@
 - [How to run Kata Containers with `nydus`](how-to-use-virtio-fs-nydus-with-kata.md)
 - [How to run Kata Containers with AMD SEV-SNP](how-to-run-kata-containers-with-SNP-VMs.md)
 - [How to use EROFS to build rootfs in Kata Containers](how-to-use-erofs-build-rootfs.md)
-## Confidential Containers
-- [How to use build and test the Confidential Containers `CCv0` proof of concept](how-to-build-and-test-ccv0.md)
-- [How to generate a Kata Containers payload for the Confidential Containers Operator](how-to-generate-a-kata-containers-payload-for-the-confidential-containers-operator.md)
+- [How to run Kata Containers with kinds of Block Volumes](how-to-run-kata-containers-with-kinds-of-Block-Volumes.md)

docs/how-to/ccv0.sh

@@ -1,635 +0,0 @@
-#!/bin/bash -e
-#
-# Copyright (c) 2021, 2023 IBM Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-# Disclaimer: This script is work in progress for supporting the CCv0 prototype
-# It shouldn't be considered supported by the Kata Containers community, or anyone else
-
-# Based on https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md,
-# but with elements of the tests/.ci scripts used
-
-readonly script_name="$(basename "${BASH_SOURCE[0]}")"
-
-# By default in Golang >= 1.16 GO111MODULE is set to "on", but not all modules support it, so overwrite to "auto"
-export GO111MODULE="auto"
-
-# Setup kata containers environments if not set - we default to use containerd
-export CRI_CONTAINERD=${CRI_CONTAINERD:-"yes"}
-export CRI_RUNTIME=${CRI_RUNTIME:-"containerd"}
-export CRIO=${CRIO:-"no"}
-export KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}"
-export KUBERNETES=${KUBERNETES:-"no"}
-export AGENT_INIT="${AGENT_INIT:-${TEST_INITRD:-no}}"
-export AA_KBC="${AA_KBC:-offline_fs_kbc}"
-export KATA_BUILD_CC=${KATA_BUILD_CC:-"yes"}
-export TEE_TYPE=${TEE_TYPE:-}
-export PREFIX="${PREFIX:-/opt/confidential-containers}"
-export RUNTIME_CONFIG_PATH="${RUNTIME_CONFIG_PATH:-${PREFIX}/share/defaults/kata-containers/configuration.toml}"
-
-# Allow the user to overwrite the default repo and branch names if they want to build from a fork
-export katacontainers_repo="${katacontainers_repo:-github.com/kata-containers/kata-containers}"
-export katacontainers_branch="${katacontainers_branch:-CCv0}" 
-export kata_default_branch=${katacontainers_branch}
-export tests_repo="${tests_repo:-github.com/kata-containers/tests}"
-export tests_branch="${tests_branch:-CCv0}"
-export target_branch=${tests_branch} # kata-containers/ci/lib.sh uses target branch var to check out tests repo
-
-# if .bash_profile exists then use it, otherwise fall back to .profile
-export PROFILE="${HOME}/.profile"
-if [ -r "${HOME}/.bash_profile" ]; then
-    export PROFILE="${HOME}/.bash_profile"
-fi
-# Stop PS1: unbound variable error happening
-export PS1=${PS1:-}
-
-# Create a bunch of common, derived values up front so we don't need to create them in all the different functions
-. ${PROFILE}
-if [ -z ${GOPATH} ]; then
-    export GOPATH=${HOME}/go
-fi
-export tests_repo_dir="${GOPATH}/src/${tests_repo}"
-export katacontainers_repo_dir="${GOPATH}/src/${katacontainers_repo}"
-export ROOTFS_DIR="${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder/rootfs"
-export PULL_IMAGE="${PULL_IMAGE:-quay.io/kata-containers/confidential-containers:signed}" # Doesn't need authentication
-export CONTAINER_ID="${CONTAINER_ID:-0123456789}"
-source /etc/os-release || source /usr/lib/os-release
-grep -Eq "\<fedora\>" /etc/os-release 2> /dev/null && export USE_PODMAN=true
-
-
-# If we've already checked out the test repo then source the confidential scripts
-if [ "${KUBERNETES}" == "yes" ]; then
-    export BATS_TEST_DIRNAME="${tests_repo_dir}/integration/kubernetes/confidential"
-    [ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/lib.sh"
-else
-    export BATS_TEST_DIRNAME="${tests_repo_dir}/integration/containerd/confidential"
-    [ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/lib.sh"
-fi
-
-[ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
-
-usage() {
-    exit_code="$1"
-    cat <<EOF
-Overview:
-    Build and test kata containers from source
-    Optionally set kata-containers and tests repo and branch as exported variables before running
-    e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && ~/${script_name} build_and_install_all
-Usage:
-    ${script_name} [options] <command>
-Commands:
-- agent_create_container:           Run CreateContainer command against the agent with agent-ctl
-- agent_pull_image:                 Run PullImage command against the agent with agent-ctl
-- all:                              Build and install everything, test kata with containerd and capture the logs
-- build_and_add_agent_to_rootfs:    Builds the kata-agent and adds it to the rootfs
-- build_and_install_all:            Build and install everything
-- build_and_install_rootfs:         Builds and installs the rootfs image
-- build_kata_runtime:               Build and install the kata runtime
-- build_cloud_hypervisor            Checkout, patch, build and install Cloud Hypervisor
-- build_qemu:                       Checkout, patch, build and install QEMU
-- configure:                        Configure Kata to use rootfs and enable debug
-- connect_to_ssh_demo_pod:          Ssh into the ssh demo pod, showing that the decryption succeeded
-- copy_signature_files_to_guest     Copies signature verification files to guest
-- create_rootfs:                    Create a local rootfs
-- crictl_create_cc_container        Use crictl to create a new busybox container in the kata cc pod
-- crictl_create_cc_pod              Use crictl to create a new kata cc pod
-- crictl_delete_cc                  Use crictl to delete the kata cc pod sandbox and container in it
-- help:                             Display this help
-- init_kubernetes:                  initialize a Kubernetes cluster on this system
-- initialize:                       Install dependencies and check out kata-containers source
-- install_guest_kernel:             Setup, build and install the guest kernel
-- kubernetes_create_cc_pod:         Create a Kata CC runtime busybox-based pod in Kubernetes
-- kubernetes_create_ssh_demo_pod:   Create a Kata CC runtime pod based on the ssh demo
-- kubernetes_delete_cc_pod:         Delete the Kata CC runtime busybox-based pod in Kubernetes
-- kubernetes_delete_ssh_demo_pod:   Delete the Kata CC runtime pod based on the ssh demo
-- open_kata_shell:                  Open a shell into the kata runtime
-- rebuild_and_install_kata:         Rebuild the kata runtime and agent and build and install the image
-- shim_pull_image:                  Run PullImage command against the shim with ctr
-- test_capture_logs:                Test using kata with containerd and capture the logs in the user's home directory
-- test:                             Test using kata with containerd
-
-Options:
-    -d: Enable debug
-    -h: Display this help
-EOF
-    # if script sourced don't exit as this will exit the main shell, just return instead
-    [[ $_ != $0 ]] && return "$exit_code" || exit "$exit_code"
-}
-
-build_and_install_all() {
-    initialize
-    build_and_install_kata_runtime
-    configure
-    create_a_local_rootfs
-    build_and_install_rootfs
-    install_guest_kernel_image
-    case "$KATA_HYPERVISOR" in
-        "qemu") 
-            build_qemu
-            ;;
-        "cloud-hypervisor") 
-            build_cloud_hypervisor
-            ;;
-        *)
-            echo "Invalid option: $KATA_HYPERVISOR is not supported." >&2
-            ;;
-    esac
-
-    check_kata_runtime
-    if [ "${KUBERNETES}" == "yes" ]; then
-        init_kubernetes
-    fi
-}
-
-rebuild_and_install_kata() {
-    checkout_tests_repo
-    checkout_kata_containers_repo
-    build_and_install_kata_runtime
-    build_and_add_agent_to_rootfs
-    build_and_install_rootfs
-    check_kata_runtime
-}
-
-# Based on the jenkins_job_build.sh script in kata-containers/tests/.ci - checks out source code and installs dependencies
-initialize() {
-    # We need git to checkout and bootstrap the ci scripts and some other packages used in testing
-    sudo apt-get update && sudo apt-get install -y curl git qemu-utils
-    
-    grep -qxF "export GOPATH=\${HOME}/go" "${PROFILE}" || echo "export GOPATH=\${HOME}/go" >> "${PROFILE}"
-    grep -qxF "export GOROOT=/usr/local/go" "${PROFILE}" || echo "export GOROOT=/usr/local/go" >> "${PROFILE}"
-    grep -qxF "export PATH=\${GOPATH}/bin:/usr/local/go/bin:\${PATH}" "${PROFILE}" || echo "export PATH=\${GOPATH}/bin:/usr/local/go/bin:\${PATH}" >> "${PROFILE}"
-    
-    # Load the new go and PATH parameters from the profile
-    . ${PROFILE}
-    mkdir -p "${GOPATH}"
-
-    checkout_tests_repo
-
-    pushd "${tests_repo_dir}"
-    local ci_dir_name=".ci"
-    sudo -E PATH=$PATH -s "${ci_dir_name}/install_go.sh" -p -f
-    sudo -E PATH=$PATH -s "${ci_dir_name}/install_rust.sh"
-    # Need to change ownership of rustup so later process can create temp files there
-    sudo chown -R ${USER}:${USER} "${HOME}/.rustup"
-
-    checkout_kata_containers_repo
-
-    # Run setup, but don't install kata as we will build it ourselves in locations matching the developer guide
-    export INSTALL_KATA="no"
-    sudo -E PATH=$PATH -s ${ci_dir_name}/setup.sh
-    # Reload the profile to pick up installed dependencies
-    . ${PROFILE}
-    popd
-}
-
-checkout_tests_repo() {
-    echo "Creating repo: ${tests_repo} and branch ${tests_branch} into ${tests_repo_dir}..."
-    # Due to git https://github.blog/2022-04-12-git-security-vulnerability-announced/ the tests repo needs
-    # to be owned by root as it is re-checked out in rootfs.sh
-    mkdir -p $(dirname "${tests_repo_dir}")
-    [ -d "${tests_repo_dir}" ] || sudo -E git clone "https://${tests_repo}.git" "${tests_repo_dir}"
-    sudo -E chown -R root:root "${tests_repo_dir}"
-    pushd "${tests_repo_dir}"
-    sudo -E git fetch
-    if [ -n "${tests_branch}" ]; then
-        sudo -E git checkout ${tests_branch}
-    fi
-    sudo -E git reset --hard origin/${tests_branch}
-    popd
-
-    source "${BATS_TEST_DIRNAME}/lib.sh"
-    source "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
-}
-
-# Note: clone_katacontainers_repo using go, so that needs to be installed first
-checkout_kata_containers_repo() {
-    source "${tests_repo_dir}/.ci/lib.sh"
-    echo "Creating repo: ${katacontainers_repo} and branch ${kata_default_branch} into ${katacontainers_repo_dir}..."
-    clone_katacontainers_repo
-    sudo -E chown -R ${USER}:${USER} "${katacontainers_repo_dir}"
-}
-
-build_and_install_kata_runtime() {
-    export DEFAULT_HYPERVISOR=${KATA_HYPERVISOR}
-    ${tests_repo_dir}/.ci/install_runtime.sh
-}
-
-configure() {
-    # configure kata to use rootfs, not initrd
-    sudo sed -i 's/^\(initrd =.*\)/# \1/g' ${RUNTIME_CONFIG_PATH}
-
-    enable_full_debug
-    enable_agent_console
-
-    # Switch image offload to true in kata config
-    switch_image_service_offload "on"
-
-    configure_cc_containerd
-    # From crictl v1.24.1 the default timoout leads to the pod creation failing, so update it
-    sudo crictl config --set timeout=10
-
-    # Verity checks aren't working locally, as we aren't re-genning the hash maybe? so remove it from the kernel parameters
-    remove_kernel_param "cc_rootfs_verity.scheme"
-}
-
-build_and_add_agent_to_rootfs() {
-    build_a_custom_kata_agent
-    add_custom_agent_to_rootfs
-}
-
-build_a_custom_kata_agent() {
-    # Install libseccomp for static linking
-    sudo -E PATH=$PATH GOPATH=$GOPATH ${katacontainers_repo_dir}/ci/install_libseccomp.sh /tmp/kata-libseccomp /tmp/kata-gperf
-    export LIBSECCOMP_LINK_TYPE=static
-    export LIBSECCOMP_LIB_PATH=/tmp/kata-libseccomp/lib
-
-    . "$HOME/.cargo/env"
-    pushd ${katacontainers_repo_dir}/src/agent
-    sudo -E PATH=$PATH make
-
-    ARCH=$(uname -m)
-    [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
-    [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
-
-    # Run a make install into the rootfs directory in order to create the kata-agent.service file which is required when we add to the rootfs
-    sudo -E PATH=$PATH make install DESTDIR="${ROOTFS_DIR}"
-    popd
-}
-
-create_a_local_rootfs() {
-    sudo rm -rf "${ROOTFS_DIR}"
-    pushd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder
-    export distro="ubuntu"
-    [[ -z "${USE_PODMAN:-}" ]] && use_docker="${use_docker:-1}"
-    sudo -E OS_VERSION="${OS_VERSION:-}" GOPATH=$GOPATH EXTRA_PKGS="vim iputils-ping net-tools" DEBUG="${DEBUG:-}" USE_DOCKER="${use_docker:-}" SKOPEO=${SKOPEO:-} AA_KBC=${AA_KBC:-} UMOCI=yes SECCOMP=yes ./rootfs.sh -r ${ROOTFS_DIR} ${distro}
-
-     # Install_rust.sh during rootfs.sh switches us to the main branch of the tests repo, so switch back now
-    pushd "${tests_repo_dir}"
-    sudo -E git checkout ${tests_branch}
-    popd
-    # During the ./rootfs.sh call the kata agent is built as root, so we need to update the permissions, so we can rebuild it
-    sudo chown -R ${USER}:${USER} "${katacontainers_repo_dir}/src/agent/"
-
-    popd
-}
-
-add_custom_agent_to_rootfs() {
-    pushd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder
-
-    ARCH=$(uname -m)
-    [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
-    [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
-
-    sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/usr/bin ${katacontainers_repo_dir}/src/agent/target/${ARCH}-unknown-linux-${LIBC}/release/kata-agent
-    sudo install -o root -g root -m 0440 ../../../src/agent/kata-agent.service ${ROOTFS_DIR}/usr/lib/systemd/system/
-    sudo install -o root -g root -m 0440 ../../../src/agent/kata-containers.target ${ROOTFS_DIR}/usr/lib/systemd/system/
-    popd
-}
-
-build_and_install_rootfs() {
-    build_rootfs_image
-    install_rootfs_image
-}
-
-build_rootfs_image() {
-    pushd ${katacontainers_repo_dir}/tools/osbuilder/image-builder
-    # Logic from install_kata_image.sh - if we aren't using podman (ie on a fedora like), then use docker
-    [[ -z "${USE_PODMAN:-}" ]] && use_docker="${use_docker:-1}"
-    sudo -E USE_DOCKER="${use_docker:-}" ./image_builder.sh ${ROOTFS_DIR}
-    popd
-}
-
-install_rootfs_image() {
-    pushd ${katacontainers_repo_dir}/tools/osbuilder/image-builder
-    local commit=$(git log --format=%h -1 HEAD)
-    local date=$(date +%Y-%m-%d-%T.%N%z)
-    local image="kata-containers-${date}-${commit}"
-    sudo install -o root -g root -m 0640 -D kata-containers.img "${PREFIX}/share/kata-containers/${image}"
-    (cd ${PREFIX}/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
-    echo "Built Rootfs from ${ROOTFS_DIR} to ${PREFIX}/share/kata-containers/${image}"
-    ls -al ${PREFIX}/share/kata-containers
-    popd
-}
-
-install_guest_kernel_image() {
-   ${tests_repo_dir}/.ci/install_kata_kernel.sh
-}
-
-build_qemu() {
-    ${tests_repo_dir}/.ci/install_virtiofsd.sh
-    ${tests_repo_dir}/.ci/install_qemu.sh
-}
-
-build_cloud_hypervisor() {
-    ${tests_repo_dir}/.ci/install_virtiofsd.sh
-    ${tests_repo_dir}/.ci/install_cloud_hypervisor.sh
-}
-
-check_kata_runtime() {
-    sudo kata-runtime check
-}
-
-k8s_pod_file="${HOME}/busybox-cc.yaml"
-init_kubernetes() {
-    # Check that kubeadm was installed and install it otherwise
-    if ! [ -x "$(command -v kubeadm)" ]; then
-        pushd "${tests_repo_dir}/.ci"
-        sudo -E PATH=$PATH -s install_kubernetes.sh
-        if [ "${CRI_CONTAINERD}" == "yes" ]; then
-            sudo -E PATH=$PATH -s "configure_containerd_for_kubernetes.sh"
-        fi
-        popd
-    fi
-
-    # If kubernetes init has previously run we need to clean it by removing the image and resetting k8s
-    local cid=$(sudo docker ps -a -q -f name=^/kata-registry$)
-    if [ -n "${cid}" ]; then
-        sudo docker stop ${cid} && sudo docker rm ${cid}
-    fi
-    local k8s_nodes=$(kubectl get nodes -o name 2>/dev/null || true)
-    if [ -n "${k8s_nodes}" ]; then
-        sudo kubeadm reset -f
-    fi
-
-    export CI="true" && sudo -E PATH=$PATH -s ${tests_repo_dir}/integration/kubernetes/init.sh
-    sudo chown ${USER}:$(id -g -n ${USER}) "$HOME/.kube/config"
-    cat << EOF > ${k8s_pod_file}
-apiVersion: v1
-kind: Pod
-metadata:
-  name: busybox-cc
-spec:
-  runtimeClassName: kata
-  containers:
-  - name: nginx
-    image: quay.io/kata-containers/confidential-containers:signed
-    imagePullPolicy: Always  
-EOF
-}
-
-call_kubernetes_create_cc_pod() {
-    kubernetes_create_cc_pod ${k8s_pod_file}
-}
-
-call_kubernetes_delete_cc_pod() {
-    pod_name=$(kubectl get pods -o jsonpath='{.items..metadata.name}')
-    kubernetes_delete_cc_pod $pod_name
-}
-
-call_kubernetes_create_ssh_demo_pod() {
-    setup_decryption_files_in_guest
-    kubernetes_create_ssh_demo_pod
-}
-
-call_connect_to_ssh_demo_pod() {
-    connect_to_ssh_demo_pod
-}
-
-call_kubernetes_delete_ssh_demo_pod() {
-    pod=$(kubectl get pods -o jsonpath='{.items..metadata.name}')
-    kubernetes_delete_ssh_demo_pod $pod
-}
-
-crictl_sandbox_name=kata-cc-busybox-sandbox
-call_crictl_create_cc_pod() {
-    # Update iptables to allow forwarding to the cni0 bridge avoiding issues caused by the docker0 bridge
-    sudo iptables -P FORWARD ACCEPT
-    
-    # get_pod_config in tests_common exports `pod_config` that points to the prepared pod config yaml 
-    get_pod_config
-
-    crictl_delete_cc_pod_if_exists "${crictl_sandbox_name}"
-    crictl_create_cc_pod "${pod_config}"
-    sudo crictl pods
-}
-
-call_crictl_create_cc_container() {
-    # Create container configuration yaml based on our test copy of busybox
-    # get_pod_config in tests_common exports `pod_config` that points to the prepared pod config yaml 
-    get_pod_config
-
-    local container_config="${FIXTURES_DIR}/${CONTAINER_CONFIG_FILE:-container-config.yaml}"
-    local pod_name=${crictl_sandbox_name}
-    crictl_create_cc_container ${pod_name} ${pod_config} ${container_config}
-    sudo crictl ps -a
-}
-
-crictl_delete_cc() {
-    crictl_delete_cc_pod ${crictl_sandbox_name}
-}
-
-test_kata_runtime() {
-    echo "Running ctr with the kata runtime..."
-    local test_image="quay.io/kata-containers/confidential-containers:signed"
-    if [ -z $(sudo ctr images ls -q name=="${test_image}") ]; then
-        sudo ctr image pull "${test_image}"
-    fi
-    sudo ctr run --runtime "io.containerd.kata.v2" --rm -t "${test_image}" test-kata uname -a
-}
-
-run_kata_and_capture_logs() {
-    echo "Clearing systemd journal..."
-    sudo systemctl stop systemd-journald
-    sudo rm -f /var/log/journal/*/* /run/log/journal/*/*
-    sudo systemctl start systemd-journald
-    test_kata_runtime
-    echo "Collecting logs..."
-    sudo journalctl -q -o cat -a -t kata-runtime > ${HOME}/kata-runtime.log
-    sudo journalctl -q -o cat -a -t kata > ${HOME}/shimv2.log
-    echo "Logs output to ${HOME}/kata-runtime.log and ${HOME}/shimv2.log"
-}
-
-get_ids() {
-    guest_cid=$(sudo ss -H --vsock | awk '{print $6}' | cut -d: -f1)
-    sandbox_id=$(ps -ef | grep containerd-shim-kata-v2 | egrep -o "id [^,][^,].* " | awk '{print $2}')
-}
-
-open_kata_shell() {
-    get_ids
-    sudo -E "PATH=$PATH" kata-runtime exec ${sandbox_id}
-}
-
-build_bundle_dir_if_necessary() {
-    bundle_dir="/tmp/bundle"
-    if [ ! -d "${bundle_dir}" ]; then
-        rootfs_dir="$bundle_dir/rootfs"
-        image="quay.io/kata-containers/confidential-containers:signed"
-        mkdir -p "$rootfs_dir" && (cd "$bundle_dir" && runc spec)
-        sudo docker export $(sudo docker create "$image") | tar -C "$rootfs_dir" -xvf -
-    fi
-    # There were errors in create container agent-ctl command due to /bin/ seemingly not being on the path, so hardcode it
-    sudo sed -i -e 's%^\(\t*\)"sh"$%\1"/bin/sh"%g' "${bundle_dir}/config.json"
-}
-
-build_agent_ctl() {
-    cd ${GOPATH}/src/${katacontainers_repo}/src/tools/agent-ctl/
-    if [ -e "${HOME}/.cargo/registry" ]; then
-        sudo chown -R ${USER}:${USER} "${HOME}/.cargo/registry"
-    fi
-    sudo -E PATH=$PATH -s  make
-    ARCH=$(uname -m)
-    [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
-    [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
-    cd "./target/${ARCH}-unknown-linux-${LIBC}/release/"
-}
-
-run_agent_ctl_command() {
-    get_ids
-    build_bundle_dir_if_necessary
-    command=$1
-    # If kata-agent-ctl pre-built in this directory, use it directly, otherwise build it first and switch to release
-    if [ ! -x kata-agent-ctl ]; then
-        build_agent_ctl
-    fi 
-     ./kata-agent-ctl -l debug connect --bundle-dir "${bundle_dir}" --server-address "vsock://${guest_cid}:1024" -c "${command}"
-}
-
-agent_pull_image() {
-    run_agent_ctl_command "PullImage image=${PULL_IMAGE} cid=${CONTAINER_ID} source_creds=${SOURCE_CREDS}"
-}
-
-agent_create_container() {
-    run_agent_ctl_command "CreateContainer cid=${CONTAINER_ID}"
-}
-
-shim_pull_image() {
-    get_ids
-    local ctr_shim_command="sudo ctr --namespace k8s.io shim --id ${sandbox_id} pull-image ${PULL_IMAGE} ${CONTAINER_ID}"
-    echo "Issuing command '${ctr_shim_command}'"
-    ${ctr_shim_command}
-}
-
-call_copy_signature_files_to_guest() {
-    # TODO #5173 - remove this once the kernel_params aren't ignored by the agent config
-    export DEBUG_CONSOLE="true"
-    
-    if [ "${SKOPEO:-}" = "yes" ]; then
-        add_kernel_params "agent.container_policy_file=/etc/containers/quay_verification/quay_policy.json"
-        setup_skopeo_signature_files_in_guest
-    else
-        # TODO #4888 - set config to specifically enable signature verification to be on in ImageClient
-        setup_offline_fs_kbc_signature_files_in_guest
-    fi
-}
-
-main() {
-    while getopts "dh" opt; do
-        case "$opt" in
-            d) 
-                export DEBUG="-d"
-                set -x
-                ;;
-            h) 
-                usage 0
-                ;;
-            \?)
-                echo "Invalid option: -$OPTARG" >&2
-                usage 1
-                ;;
-        esac
-    done
-
-    shift $((OPTIND - 1))
-
-    subcmd="${1:-}"
-
-    [ -z "${subcmd}" ] && usage 1
-
-    case "${subcmd}" in
-        all)
-            build_and_install_all
-            run_kata_and_capture_logs
-            ;;
-        build_and_install_all)
-            build_and_install_all
-            ;;
-        rebuild_and_install_kata)
-            rebuild_and_install_kata
-            ;;
-        initialize)
-            initialize
-            ;;
-        build_kata_runtime)
-            build_and_install_kata_runtime
-            ;;
-        configure)
-            configure
-            ;;
-        create_rootfs)
-            create_a_local_rootfs
-            ;;
-        build_and_add_agent_to_rootfs)
-            build_and_add_agent_to_rootfs
-            ;;
-        build_and_install_rootfs)
-            build_and_install_rootfs
-            ;;
-        install_guest_kernel)
-            install_guest_kernel_image
-            ;;
-        build_cloud_hypervisor)
-            build_cloud_hypervisor
-            ;;
-        build_qemu)
-            build_qemu
-            ;;
-        init_kubernetes)
-            init_kubernetes
-            ;;
-        crictl_create_cc_pod)
-            call_crictl_create_cc_pod
-            ;;
-        crictl_create_cc_container)
-            call_crictl_create_cc_container
-            ;;
-        crictl_delete_cc)
-            crictl_delete_cc
-            ;;
-        kubernetes_create_cc_pod)
-            call_kubernetes_create_cc_pod
-            ;;
-        kubernetes_delete_cc_pod)
-            call_kubernetes_delete_cc_pod
-            ;;
-        kubernetes_create_ssh_demo_pod)
-            call_kubernetes_create_ssh_demo_pod
-            ;;
-        connect_to_ssh_demo_pod)
-            call_connect_to_ssh_demo_pod
-            ;;
-        kubernetes_delete_ssh_demo_pod)
-            call_kubernetes_delete_ssh_demo_pod
-            ;;
-        test)
-            test_kata_runtime
-            ;;
-        test_capture_logs)
-            run_kata_and_capture_logs
-            ;;
-        open_kata_console)
-            open_kata_console
-            ;;
-        open_kata_shell)
-            open_kata_shell
-            ;;
-        agent_pull_image)
-            agent_pull_image
-            ;;
-        shim_pull_image)
-            shim_pull_image
-            ;;
-        agent_create_container)
-            agent_create_container
-            ;;
-        copy_signature_files_to_guest)
-            call_copy_signature_files_to_guest
-            ;;
-        *)
-            usage 1
-            ;;
-    esac
-}
-
-main $@

docs/how-to/data/confidential-agent-config.toml.in

@@ -1,45 +0,0 @@
-# Copyright (c) 2021 IBM Corp.
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-aa_kbc_params = "$AA_KBC_PARAMS"
-https_proxy = "$HTTPS_PROXY"
-[endpoints]
-allowed = [
-"AddARPNeighborsRequest",
-"AddSwapRequest",
-"CloseStdinRequest",
-"CopyFileRequest",
-"CreateContainerRequest",
-"CreateSandboxRequest",
-"DestroySandboxRequest",
-#"ExecProcessRequest",
-"GetMetricsRequest",
-"GetOOMEventRequest",
-"GuestDetailsRequest",
-"ListInterfacesRequest",
-"ListRoutesRequest",
-"MemHotplugByProbeRequest",
-"OnlineCPUMemRequest",
-"PauseContainerRequest",
-"PullImageRequest",
-"ReadStreamRequest",
-"RemoveContainerRequest",
-#"ReseedRandomDevRequest",
-"ResizeVolumeRequest",
-"ResumeContainerRequest",
-"SetGuestDateTimeRequest",
-"SignalProcessRequest",
-"StartContainerRequest",
-"StartTracingRequest",
-"StatsContainerRequest",
-"StopTracingRequest",
-"TtyWinResizeRequest",
-"UpdateContainerRequest",
-"UpdateInterfaceRequest",
-"UpdateRoutesRequest",
-"VolumeStatsRequest",
-"WaitProcessRequest",
-"WriteStreamRequest"
-]

docs/how-to/how-to-build-and-test-ccv0.md

@@ -1,475 +0,0 @@
-# How to build, run and test Kata CCv0
-
-## Introduction and Background
-
-In order to try and make building (locally) and demoing the Kata Containers `CCv0` code base as simple as possible I've
-shared a script [`ccv0.sh`](./ccv0.sh). This script was originally my attempt to automate the steps of the 
-[Developer Guide](https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md) so that I could do
-different sections of them repeatedly and reliably as I was playing around with make changes to different parts of the 
-Kata code base. I then tried to weave in some of the [`tests/.ci`](https://github.com/kata-containers/tests/tree/main/.ci) 
-scripts in order to have less duplicated code.
-As we're progress on the confidential containers journey I hope to add more features to demonstrate the functionality
-we have working.
-
-*Disclaimer: This script has mostly just been used and tested by me ([@stevenhorsman](https://github.com/stevenhorsman)),*
-*so there might be issues with it. I'm happy to try and help solve these if possible, but this shouldn't be considered a*
-*fully supported process by the Kata Containers community.*
-
-### Basic script set-up and optional environment variables
-
-In order to build, configure and demo the CCv0 functionality, these are the set-up steps I take:
-- Provision a new VM
-    - *I choose a Ubuntu 20.04 8GB VM for this as I had one available. There are some dependences on apt-get installed*
-    *packages, so these will need re-working to be compatible with other platforms.*
-- Copy the script over to your VM *(I put it in the home directory)* and ensure it has execute permission by running 
-```bash
-$ chmod u+x ccv0.sh
-```
-- Optionally set up some environment variables
-    - By default the script checks out the `CCv0` branches of the `kata-containers/kata-containers` and 
-      `kata-containers/tests` repositories, but it is designed to be used to test of personal forks and branches as well. 
-      If you want to build and run these you can export the `katacontainers_repo`, `katacontainers_branch`, `tests_repo`
-      and `tests_branch` variables e.g.
-      ```bash
-      $ export katacontainers_repo=github.com/stevenhorsman/kata-containers
-      $ export katacontainers_branch=stevenh/agent-pull-image-endpoint
-      $ export tests_repo=github.com/stevenhorsman/tests
-      $ export tests_branch=stevenh/add-ccv0-changes-to-build
-      ```
-      before running the script.
-    - By default the build and configuration are using `QEMU` as the hypervisor. In order to use `Cloud Hypervisor` instead
-      set:
-      ```
-      $ export KATA_HYPERVISOR="cloud-hypervisor"
-      ```
-      before running the build.
-
-- At this point you can provision a Kata confidential containers pod and container with either
-  [`crictl`](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image),
-  or [Kubernetes](#using-kubernetes-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image)
-  and then test and use it. 
-
-### Using crictl for end-to-end provisioning of a Kata confidential containers pod with an unencrypted image
-
-- Run the full build process with Kubernetes turned off, so its configuration doesn't interfere with `crictl` using:
-  ```bash
-  $ export KUBERNETES="no"
-  $ export KATA_HYPERVISOR="qemu"
-  $ ~/ccv0.sh -d build_and_install_all
-  ```
-    > **Note**: Much of this script has to be run as `sudo`, so you are likely to get prompted for your password.
-    - *I run this script sourced just so that the required installed components are accessible on the `PATH` to the rest*
-      *of the process without having to reload the session.*
-    - The steps that `build_and_install_all` takes is:
-        - Checkout the git repos for the `tests` and `kata-containers` repos as specified by the environment variables
-        (default to `CCv0` branches if they are not supplied)
-        - Use the `tests/.ci` scripts to install the build dependencies
-        - Build and install the Kata runtime
-        - Configure Kata to use containerd and for debug and confidential containers features to be enabled (including
-          enabling console access to the Kata guest shell, which should only be done in development)
-        - Create, build and install a rootfs for the Kata hypervisor to use. For 'CCv0' this is currently based on Ubuntu
-        20.04.
-        - Build the Kata guest kernel
-        - Install the hypervisor (in order to select which hypervisor will be used, the `KATA_HYPERVISOR` environment
-        variable can be used to select between `qemu` or `cloud-hypervisor`)
-    > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
-    during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
-    this, login into Docker Hub and pull the images used with:
-  >  ```bash
-  >  $ sudo docker login
-  >  $ sudo docker pull ubuntu
-  >  ```
-  >  then re-run the command.
-    - The first time this runs it may take a while, but subsequent runs will be quicker as more things are already
-      installed and they can be further cut down by not running all the above steps 
-      [see "Additional script usage" below](#additional-script-usage)
-   
-- Create a new Kata sandbox pod using `crictl` with:
-  ```bash
-  $ ~/ccv0.sh crictl_create_cc_pod
-  ```
-    - This creates a pod configuration file, creates the pod from this using 
-    `sudo crictl runp -r kata ~/pod-config.yaml` and runs `sudo crictl pods` to show the pod
-- Create a new Kata confidential container with:
-  ```bash
-  $ ~/ccv0.sh crictl_create_cc_container
-  ```
-    - This creates a container (based on `busybox:1.33.1`) in the Kata cc sandbox and prints a list of containers.
-      This will have been created based on an image pulled in the Kata pod sandbox/guest, not on the host machine.
-
-As this point you should have a `crictl` pod and container that is using the Kata confidential containers runtime.
-You can [validate that the container image was pulled on the guest](#validate-that-the-container-image-was-pulled-on-the-guest)
-or [using the Kata pod sandbox for testing with `agent-ctl` or `ctr shim`](#using-a-kata-pod-sandbox-for-testing-with-agent-ctl-or-ctr-shim)
-
-#### Clean up the `crictl` pod sandbox and container
-- When the testing is complete you can delete the container and pod by running:
-  ```bash
-  $ ~/ccv0.sh crictl_delete_cc
-  ```
-### Using Kubernetes for end-to-end provisioning of a Kata confidential containers pod with an unencrypted image
-
-- Run the full build process with the Kubernetes environment variable set to `"yes"`, so the Kubernetes cluster is
-  configured and created using the VM
-  as a single node cluster: 
-  ```bash
-  $ export KUBERNETES="yes"
-  $ ~/ccv0.sh build_and_install_all
-  ```
-    > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
-    during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
-    this, login into Docker Hub and pull the images used with:
-  >  ```bash
-  >  $ sudo docker login
-  >  $ sudo docker pull registry:2
-  >  $ sudo docker pull ubuntu:20.04
-  >  ```
-  >  then re-run the command.
-- Check that your Kubernetes cluster has been correctly set-up by running : 
-  ```bash
-  $ kubectl get nodes
-  ```
-  and checking that you see a single node e.g.
-  ```text
-  NAME                             STATUS   ROLES                  AGE   VERSION
-  stevenh-ccv0-k8s1.fyre.ibm.com   Ready    control-plane,master   43s   v1.22.0
-  ```
-- Create a Kata confidential containers pod by running:
-  ```bash
-  $ ~/ccv0.sh kubernetes_create_cc_pod
-  ```
-- Wait a few seconds for pod to start then check that the pod's status is `Running` with 
-  ```bash
-  $ kubectl get pods
-  ```
-  which should show something like:
-  ```text
-  NAME         READY   STATUS    RESTARTS   AGE
-  busybox-cc   1/1     Running   0          54s
-  ```
-
-- As this point you should have a Kubernetes pod and container running, that is using the Kata 
-confidential containers runtime.
-You can [validate that the container image was pulled on the guest](#validate-that-the-container-image-was-pulled-on-the-guest)
-or [using the Kata pod sandbox for testing with `agent-ctl` or `ctr shim`](#using-a-kata-pod-sandbox-for-testing-with-agent-ctl-or-ctr-shim)
-
-#### Clean up the Kubernetes pod sandbox and container
-- When the testing is complete you can delete the container and pod by running:
-  ```bash
-  $ ~/ccv0.sh kubernetes_delete_cc_pod
-  ```
-
-### Validate that the container image was pulled on the guest
-
-There are a couple of ways we can check that the container pull image action was offloaded to the guest, by checking
-the guest's file system for the unpacked bundle and checking the host's directories to ensure it wasn't also pulled
-there.
-- To check the guest's file system:
-    - Open a shell into the Kata guest with:
-      ```bash
-      $ ~/ccv0.sh open_kata_shell
-      ```
-    - List the files in the directory that the container image bundle should have been unpacked to with:
-      ```bash
-      $ ls -ltr /run/kata-containers/confidential-containers_signed/
-      ```
-    - This should give something like
-        ```
-        total 72
-        -rw-r--r--  1 root root  2977 Jan 20 10:03 config.json
-        drwxr-xr-x 12 root root   240 Jan 20 10:03 rootfs
-        ```
-        which shows how the image has been pulled and then unbundled on the guest.
-    - Leave the Kata guest shell by running:
-      ```bash
-      $ exit
-      ```
-- To verify that the image wasn't pulled on the host system we can look at the shared sandbox on the host and we
-  should only see a single bundle for the pause container as the `busybox` based container image should have been
-  pulled on the guest:
-    - Find all the `rootfs` directories under in the pod's shared directory with:
-      ```bash
-      $ pod_id=$(ps -ef | grep containerd-shim-kata-v2 | egrep -o "id [^,][^,].* " | awk '{print $2}')
-      $ sudo find /run/kata-containers/shared/sandboxes/${pod_id}/shared -name rootfs
-      ```
-      which should only show a single `rootfs` directory if the container image was pulled on the guest, not the host
-    - Looking that `rootfs` directory with
-      ```bash
-      $ sudo ls -ltr $(sudo find /run/kata-containers/shared/sandboxes/${pod_id}/shared -name rootfs)
-      ```
-      shows something similar to
-      ```
-      total 668
-      -rwxr-xr-x 1 root root 682696 Aug 25 13:58 pause
-      drwxr-xr-x 2 root root      6 Jan 20 02:01 proc
-      drwxr-xr-x 2 root root      6 Jan 20 02:01 dev
-      drwxr-xr-x 2 root root      6 Jan 20 02:01 sys
-      drwxr-xr-x 2 root root     25 Jan 20 02:01 etc
-      ```
-      which is clearly the pause container indicating that the `busybox` based container image is not exposed to the host.
-
-### Using a Kata pod sandbox for testing with `agent-ctl` or `ctr shim`
-
-Once you have a kata pod sandbox created as described above, either using 
-[`crictl`](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image), or [Kubernetes](#using-kubernetes-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image)
-, you can use this to test specific components of the Kata confidential
-containers architecture. This can be useful for development and debugging to isolate and test features
-that aren't broadly supported end-to-end. Here are some examples:
-
-- In the first terminal run the pull image on guest command against the Kata agent, via the shim (`containerd-shim-kata-v2`).
-This can be achieved using the [containerd](https://github.com/containerd/containerd) CLI tool, `ctr`, which can be used to
-interact with the shim directly. The command takes the form
-`ctr --namespace k8s.io shim --id <sandbox-id> pull-image <image> <new-container-id>` and can been run directly, or through
-the `ccv0.sh` script to automatically fill in the variables:
-  - Optionally, set up some environment variables to set the image and credentials used:
-    - By default the shim pull image test in `ccv0.sh` will use the `busybox:1.33.1` based test image
-     `quay.io/kata-containers/confidential-containers:signed` which requires no authentication. To use a different
-      image, set the `PULL_IMAGE` environment variable e.g. 
-      ```bash
-      $ export PULL_IMAGE="docker.io/library/busybox:latest"
-      ```
-      Currently the containerd shim pull image
-      code doesn't support using a container registry that requires authentication, so if this is required, see the 
-      below steps to run the pull image command against the agent directly.
-  - Run the pull image agent endpoint with:
-    ```bash
-    $ ~/ccv0.sh shim_pull_image
-    ```
-    which we print the `ctr shim` command for reference
-- Alternatively you can issue the command directly to the `kata-agent` pull image endpoint, which also supports
-  credentials in order to pull from an authenticated registry:
-    - Optionally set up some environment variables to set the image and credentials used:
-        - Set the `PULL_IMAGE` environment variable e.g. `export PULL_IMAGE="docker.io/library/busybox:latest"`
-          if a specific container image is required.
-        - If the container registry for the image requires authentication then this can be set with an environment
-          variable `SOURCE_CREDS`. For example to use Docker Hub (`docker.io`) as an authenticated user first run 
-          `export SOURCE_CREDS="<dockerhub username>:<dockerhub api key>"`
-            > **Note**: the credentials support on the agent request is a tactical solution for the short-term
-              proof of concept to allow more images to be pulled and tested. Once we have support for getting
-              keys into the Kata guest image using the attestation-agent and/or KBS I'd expect container registry
-              credentials to be looked up using that mechanism.
-    - Run the pull image agent endpoint with
-      ```bash
-      $ ~/ccv0.sh agent_pull_image
-      ```
-      and you should see output which includes `Command PullImage (1 of 1) returned (Ok(()), false)` to indicate
-      that the `PullImage` request was successful e.g.
-      ```
-      Finished release [optimized] target(s) in 0.21s
-      {"msg":"announce","level":"INFO","ts":"2021-09-15T08:40:14.189360410-07:00","subsystem":"rpc","name":"kata-agent-ctl","pid":"830920","version":"0.1.0","source":"kata-agent-ctl","config":"Config { server_address: \"vsock://1970354082:1024\", bundle_dir: \"/tmp/bundle\", timeout_nano: 0, interactive: false, ignore_errors: false }"}
-      {"msg":"client setup complete","level":"INFO","ts":"2021-09-15T08:40:14.193639057-07:00","pid":"830920","source":"kata-agent-ctl","name":"kata-agent-ctl","subsystem":"rpc","version":"0.1.0","server-address":"vsock://1970354082:1024"}
-      {"msg":"Run command PullImage (1 of 1)","level":"INFO","ts":"2021-09-15T08:40:14.196643765-07:00","pid":"830920","source":"kata-agent-ctl","subsystem":"rpc","name":"kata-agent-ctl","version":"0.1.0"}
-      {"msg":"response received","level":"INFO","ts":"2021-09-15T08:40:43.828200633-07:00","source":"kata-agent-ctl","name":"kata-agent-ctl","subsystem":"rpc","version":"0.1.0","pid":"830920","response":""}
-      {"msg":"Command PullImage (1 of 1) returned (Ok(()), false)","level":"INFO","ts":"2021-09-15T08:40:43.828261708-07:00","subsystem":"rpc","pid":"830920","source":"kata-agent-ctl","version":"0.1.0","name":"kata-agent-ctl"}
-      ```
-      > **Note**: The first time that `~/ccv0.sh agent_pull_image` is run, the `agent-ctl` tool will be built
-      which may take a few minutes.
-- To validate that the image pull was successful, you can open a shell into the Kata guest with:
-  ```bash
-  $ ~/ccv0.sh open_kata_shell
-  ```
-- Check the `/run/kata-containers/` directory to verify that the container image bundle has been created in a directory
-  named either `01234556789` (for the container id), or the container image name, e.g.
-  ```bash
-  $ ls -ltr /run/kata-containers/confidential-containers_signed/
-  ```
-  which should show something like
-  ```
-  total 72
-  drwxr-xr-x 10 root root   200 Jan  1  1970 rootfs
-  -rw-r--r--  1 root root  2977 Jan 20 16:45 config.json
-  ```
-- Leave the Kata shell by running:
-  ```bash
-  $ exit
-  ```
-
-## Verifying signed images
-
-For this sample demo, we use local attestation to pass through the required
-configuration to do container image signature verification. Due to this, the ability to verify images is limited
-to a pre-created selection of test images in our test
-repository [`quay.io/kata-containers/confidential-containers`](https://quay.io/repository/kata-containers/confidential-containers?tab=tags).
-For pulling images not in this test repository (called an *unprotected* registry below), we fall back to the behaviour
-of not enforcing signatures. More documentation on how to customise this to match your own containers through local,
-or remote attestation will be available in future.
-
-In our test repository there are three tagged images:
-
-| Test Image | Base Image used | Signature status | GPG key status |
-| --- | --- | --- | --- |
-| `quay.io/kata-containers/confidential-containers:signed` | `busybox:1.33.1` | [signature](https://github.com/kata-containers/tests/tree/CCv0/integration/confidential/fixtures/quay_verification/x86_64/signatures.tar) embedded in kata rootfs |  [public key](https://github.com/kata-containers/tests/tree/CCv0/integration/confidential/fixtures/quay_verification/x86_64/public.gpg) embedded in kata rootfs |
-| `quay.io/kata-containers/confidential-containers:unsigned` | `busybox:1.33.1` | not signed | not signed |
-| `quay.io/kata-containers/confidential-containers:other_signed` | `nginx:1.21.3` | [signature](https://github.com/kata-containers/tests/tree/CCv0/integration/confidential/fixtures/quay_verification/x86_64/signatures.tar) embedded in kata rootfs | GPG key not kept |
-
-Using a standard unsigned `busybox` image that can be pulled from another, *unprotected*, `quay.io` repository we can
-test a few scenarios.
-
-In this sample, with local attestation, we pass in the the public GPG key and signature files, and the [`offline_fs_kbc`
-configuration](https://github.com/confidential-containers/attestation-agent/blob/main/src/kbc_modules/offline_fs_kbc/README.md)
-into the guest image which specifies that any container image from `quay.io/kata-containers`
-must be signed with the embedded GPG key and the agent configuration needs updating to enable this. 
-With this policy set a few tests of image verification can be done to test different scenarios by attempting
-to create containers from these images using `crictl`:
-
-- If you don't already have the Kata Containers CC code built and configured for `crictl`, then follow the 
-[instructions above](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image)
-up to the `~/ccv0.sh crictl_create_cc_pod` command.
-
-- In order to enable the guest image, you will need to setup the required configuration, policy and signature files
-needed by running 
-`~/ccv0.sh copy_signature_files_to_guest` and then run `~/ccv0.sh crictl_create_cc_pod` which will delete and recreate 
-your pod - adding in the new files.
- 
-- To test the fallback behaviour works using an unsigned image from an *unprotected* registry we can pull the `busybox`
-image by running:
-  ```bash
-  $ export CONTAINER_CONFIG_FILE=container-config_unsigned-unprotected.yaml
-  $ ~/ccv0.sh crictl_create_cc_container
-  ```
-  - This finishes showing the running container e.g.
-  ```text
-  CONTAINER           IMAGE                               CREATED                  STATE               NAME                        ATTEMPT             POD ID
-  98c70fefe997a       quay.io/prometheus/busybox:latest   Less than a second ago   Running             prometheus-busybox-signed   0                   70119e0539238
-  ```
-- To test that an unsigned image from our *protected* test container registry is rejected we can run:
-  ```bash
-  $ export CONTAINER_CONFIG_FILE=container-config_unsigned-protected.yaml
-  $ ~/ccv0.sh crictl_create_cc_container
-  ```
-  - This correctly results in an error message from `crictl`:
-  `PullImage from image service failed" err="rpc error: code = Internal desc = Security validate failed: Validate image failed: The signatures do not satisfied! Reject reason: [Match reference failed.]" image="quay.io/kata-containers/confidential-containers:unsigned"`
-- To test that the signed image our *protected* test container registry is accepted we can run:
-  ```bash
-  $ export CONTAINER_CONFIG_FILE=container-config.yaml
-  $ ~/ccv0.sh crictl_create_cc_container
-  ```
-  - This finishes by showing a new `kata-cc-busybox-signed` running container e.g.
-  ```text
-  CONTAINER           IMAGE                                                    CREATED                  STATE               NAME                        ATTEMPT             POD ID
-  b4d85c2132ed9       quay.io/kata-containers/confidential-containers:signed   Less than a second ago   Running             kata-cc-busybox-signed      0                   70119e0539238
-  ...
-  ```
-- Finally to check the image with a valid signature, but invalid GPG key (the real trusted piece of information we really
-want to protect with the attestation agent in future) fails we can run: 
-  ```bash
-  $ export CONTAINER_CONFIG_FILE=container-config_signed-protected-other.yaml
-  $ ~/ccv0.sh crictl_create_cc_container
-  ```
-  - Again this results in an error message from `crictl`:
-  `"PullImage from image service failed" err="rpc error: code = Internal desc = Security validate failed: Validate image failed: The signatures do not satisfied! Reject reason: [signature verify failed! There is no pubkey can verify the signature!]" image="quay.io/kata-containers/confidential-containers:other_signed"`
-
-### Using Kubernetes to create a Kata confidential containers pod from the encrypted ssh demo sample image
-
-The [ssh-demo](https://github.com/confidential-containers/documentation/tree/main/demos/ssh-demo) explains how to
-demonstrate creating a Kata confidential containers pod from an encrypted image with the runtime created by the
-[confidential-containers operator](https://github.com/confidential-containers/documentation/blob/main/demos/operator-demo).
-To be fully confidential, this should be run on a Trusted Execution Environment, but it can be tested on generic
-hardware as well.
-
-If you wish to build the Kata confidential containers runtime to do this yourself, then you can using the following
-steps:
-
-- Run the full build process with the Kubernetes environment variable set to `"yes"`, so the Kubernetes cluster is
-  configured and created using the VM as a single node cluster and with `AA_KBC` set to `offline_fs_kbc`. 
-  ```bash
-  $ export KUBERNETES="yes"
-  $ export AA_KBC=offline_fs_kbc
-  $ ~/ccv0.sh build_and_install_all
-  ```
-    - The `AA_KBC=offline_fs_kbc` mode will ensure that, when creating the rootfs of the Kata guest, the 
-      [attestation-agent](https://github.com/confidential-containers/attestation-agent) will be added along with the
-      [sample offline KBC](https://github.com/confidential-containers/documentation/blob/main/demos/ssh-demo/aa-offline_fs_kbc-keys.json)
-      and an agent configuration file
-    > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
-    during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
-    this, login into Docker Hub and pull the images used with:
-  >  ```bash
-  >  $ sudo docker login
-  >  $ sudo docker pull registry:2
-  >  $ sudo docker pull ubuntu:20.04
-  >  ```
-  >  then re-run the command.
-- Check that your Kubernetes cluster has been correctly set-up by running : 
-  ```bash
-  $ kubectl get nodes
-  ```
-  and checking that you see a single node e.g.
-  ```text
-  NAME                             STATUS   ROLES                  AGE   VERSION
-  stevenh-ccv0-k8s1.fyre.ibm.com   Ready    control-plane,master   43s   v1.22.0
-  ```
-- Create a sample Kata confidential containers ssh pod by running:
-  ```bash
-  $ ~/ccv0.sh kubernetes_create_ssh_demo_pod
-  ```
-- As this point you should have a Kubernetes pod running the Kata confidential containers runtime that has pulled
-the [sample image](https://hub.docker.com/r/katadocker/ccv0-ssh) which was encrypted by the key file that we included
-in the rootfs.
-During the pod deployment the image was pulled and then decrypted using the key file, on the Kata guest image, without
-it ever being available to the host.
-
-- To validate that the container is working you, can connect to the image via SSH by running:
-  ```bash
-  $ ~/ccv0.sh connect_to_ssh_demo_pod
-  ```
-  - During this connection the host key fingerprint is shown and should match:
-    `ED25519 key fingerprint is SHA256:wK7uOpqpYQczcgV00fGCh+X97sJL3f6G1Ku4rvlwtR0.`
-  - After you are finished connecting then run:
-    ```bash
-    $ exit
-    ```
-  
-- To delete the sample SSH demo pod run:
-  ```bash
-  $ ~/ccv0.sh kubernetes_delete_ssh_demo_pod
-  ```
-
-## Additional script usage
-
-As well as being able to use the script as above to build all of `kata-containers` from scratch it can be used to just
-re-build bits of it by running the script with different parameters. For example after the first build you will often
-not need to re-install the dependencies, the hypervisor or the Guest kernel, but just test code changes made to the
-runtime and agent. This can be done by running `~/ccv0.sh rebuild_and_install_kata`. (*Note this does a hard checkout*
-*from git, so if your changes are only made locally it is better to do the individual steps e.g.* 
-`~/ccv0.sh build_kata_runtime && ~/ccv0.sh build_and_add_agent_to_rootfs && ~/ccv0.sh build_and_install_rootfs`).
-There are commands for a lot of steps in building, setting up and testing and the full list can be seen by running
-`~/ccv0.sh help`:
-```
-$ ~/ccv0.sh help
-Overview:
-    Build and test kata containers from source
-    Optionally set kata-containers and tests repo and branch as exported variables before running
-    e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && ~/ccv0.sh build_and_install_all
-Usage:
-    ccv0.sh [options] <command>
-Commands:
-- help:                         Display this help
-- all:                          Build and install everything, test kata with containerd and capture the logs
-- build_and_install_all:        Build and install everything
-- initialize:                   Install dependencies and check out kata-containers source
-- rebuild_and_install_kata:     Rebuild the kata runtime and agent and build and install the image
-- build_kata_runtime:           Build and install the kata runtime
-- configure:                    Configure Kata to use rootfs and enable debug
-- create_rootfs:                Create a local rootfs
-- build_and_add_agent_to_rootfs:Builds the kata-agent and adds it to the rootfs
-- build_and_install_rootfs:     Builds and installs the rootfs image
-- install_guest_kernel:         Setup, build and install the guest kernel
-- build_cloud_hypervisor        Checkout, patch, build and install Cloud Hypervisor
-- build_qemu:                   Checkout, patch, build and install QEMU
-- init_kubernetes:              initialize a Kubernetes cluster on this system
-- crictl_create_cc_pod          Use crictl to create a new kata cc pod
-- crictl_create_cc_container    Use crictl to create a new busybox container in the kata cc pod
-- crictl_delete_cc              Use crictl to delete the kata cc pod sandbox and container in it
-- kubernetes_create_cc_pod:     Create a Kata CC runtime busybox-based pod in Kubernetes
-- kubernetes_delete_cc_pod:     Delete the Kata CC runtime busybox-based pod in Kubernetes
-- open_kata_shell:              Open a shell into the kata runtime
-- agent_pull_image:             Run PullImage command against the agent with agent-ctl
-- shim_pull_image:              Run PullImage command against the shim with ctr
-- agent_create_container:       Run CreateContainer command against the agent with agent-ctl
-- test:                         Test using kata with containerd
-- test_capture_logs:            Test using kata with containerd and capture the logs in the user's home directory
-
-Options:
-    -d: Enable debug
-    -h: Display this help
-```

docs/how-to/how-to-generate-a-kata-containers-payload-for-the-confidential-containers-operator.md

@@ -1,44 +0,0 @@
-# Generating a Kata Containers payload for the Confidential Containers Operator
-
-[Confidential Containers
-Operator](https://github.com/confidential-containers/operator) consumes a Kata
-Containers payload, generated from the `CCv0` branch, and here one can find all
-the necessary info on how to build such a payload.
-
-## Requirements
-
-* `make` installed in the machine
-* Docker installed in the machine
-* `sudo` access to the machine
-
-## Process
-
-* Clone [Kata Containers](https://github.com/kata-containers/kata-containers)
-  ```sh
-  git clone --branch CCv0 https://github.com/kata-containers/kata-containers
-  ```
-  * In case you've already cloned the repo, make sure to switch to the `CCv0` branch
-    ```sh
-    git checkout CCv0
-    ```
-  * Ensure your tree is clean and in sync with upstream `CCv0`
-    ```sh
-    git clean -xfd
-    git reset --hard <upstream>/CCv0
-    ```
-* Make sure you're authenticated to `quay.io`
-    ```sh
-    sudo docker login quay.io
-    ```
-* From the top repo directory, run:
-  ```sh
-  sudo make cc-payload
-  ```
-* Make sure the image was upload to the [Confidential Containers
-  runtime-payload
-registry](https://quay.io/repository/confidential-containers/runtime-payload?tab=tags)
-
-## Notes
-
-Make sure to run it on a machine that's not the one you're hacking on, prepare a
-cup of tea, and get back to it an hour later (at least).

docs/how-to/how-to-run-kata-containers-with-kinds-of-Block-Volumes.md

@@ -0,0 +1,226 @@
+# A new way for Kata Containers to use Kinds of Block Volumes
+
+> **Note:** This guide is only available for runtime-rs with default Hypervisor Dragonball.
+> Now, other hypervisors are still ongoing, and it'll be updated when they're ready.
+
+
+## Background
+
+Currently, there is no widely applicable and convenient method available for users to use some kinds of backend storages, such as File on host based block volume, SPDK based volume or VFIO device based volume for Kata Containers, so we adopt [Proposal: Direct Block Device Assignment](https://github.com/kata-containers/kata-containers/blob/main/docs/design/direct-blk-device-assignment.md) to address it.
+
+## Solution
+
+According to the proposal, it requires to use the `kata-ctl direct-volume` command to add a direct assigned block volume device to the Kata Containers runtime. 
+
+And then with the help of method [get_volume_mount_info](https://github.com/kata-containers/kata-containers/blob/099b4b0d0e3db31b9054e7240715f0d7f51f9a1c/src/libs/kata-types/src/mount.rs#L95), get information from JSON file: `(mountinfo.json)` and parse them into structure [Direct Volume Info](https://github.com/kata-containers/kata-containers/blob/099b4b0d0e3db31b9054e7240715f0d7f51f9a1c/src/libs/kata-types/src/mount.rs#L70) which is used to save device-related information. 
+
+We only fill the `mountinfo.json`, such as `device` ,`volume_type`, `fs_type`, `metadata` and `options`, which correspond to the fields in [Direct Volume Info](https://github.com/kata-containers/kata-containers/blob/099b4b0d0e3db31b9054e7240715f0d7f51f9a1c/src/libs/kata-types/src/mount.rs#L70), to describe a device. 
+
+The JSON file `mountinfo.json` placed in a sub-path `/kubelet/kata-test-vol-001/volume001` which under fixed path `/run/kata-containers/shared/direct-volumes/`. 
+And the full path looks like: `/run/kata-containers/shared/direct-volumes/kubelet/kata-test-vol-001/volume001`, But for some security reasons. it is 
+encoded as `/run/kata-containers/shared/direct-volumes/L2t1YmVsZXQva2F0YS10ZXN0LXZvbC0wMDEvdm9sdW1lMDAx`.
+
+Finally, when running a Kata Containers with `ctr run --mount type=X, src=Y, dst=Z,,options=rbind:rw`, the `type=X` should be specified a proprietary type specifically designed for some kind of volume. 
+
+Now, supported types: 
+
+- `directvol` for direct volume
+- `vfiovol` for VFIO device based volume
+- `spdkvol` for SPDK/vhost-user based volume
+
+
+## Setup Device and Run a Kata-Containers
+
+### Direct Block Device Based Volume
+
+#### create raw block based backend storage
+
+> **Tips:** raw block based backend storage MUST be formatted with `mkfs`.
+
+```bash
+$ sudo dd if=/dev/zero of=/tmp/stor/rawdisk01.20g bs=1M count=20480
+$ sudo mkfs.ext4 /tmp/stor/rawdisk01.20g
+```
+
+#### setup direct block device for kata-containers
+
+```json
+{
+  "device": "/tmp/stor/rawdisk01.20g", 
+  "volume_type": "directvol", 
+  "fs_type": "ext4", 
+  "metadata":"{}", 
+  "options": []
+}
+```
+
+```bash
+$ sudo kata-ctl direct-volume add /kubelet/kata-direct-vol-002/directvol002 "{\"device\": \"/tmp/stor/rawdisk01.20g\", \"volume_type\": \"directvol\", \"fs_type\": \"ext4\", \"metadata\":"{}", \"options\": []}"
+$# /kubelet/kata-direct-vol-002/directvol002 <==> /run/kata-containers/shared/direct-volumes/W1lMa2F0ZXQva2F0YS10a2F0DAxvbC0wMDEvdm9sdW1lMDAx
+$ cat W1lMa2F0ZXQva2F0YS10a2F0DAxvbC0wMDEvdm9sdW1lMDAx/mountInfo.json 
+{"volume_type":"directvol","device":"/tmp/stor/rawdisk01.20g","fs_type":"ext4","metadata":{},"options":[]}
+```
+
+#### Run a Kata container with direct block device volume
+
+```bash
+$ # type=disrectvol,src=/kubelet/kata-direct-vol-002/directvol002,dst=/disk002,options=rbind:rw
+$ sudo ctr run -t --rm --runtime io.containerd.kata.v2 --mount type=directvol,src=/kubelet/kata-direct-vol-002/directvol002,dst=/disk002,options=rbind:rw "$image" kata-direct-vol-xx05302045 /bin/bash
+```
+
+
+### VFIO Device Based Block Volume
+
+#### create VFIO device based backend storage
+
+> **Tip:** It only supports `vfio-pci` based PCI device passthrough mode.
+
+In this scenario, the device's host kernel driver will be replaced by `vfio-pci`, and IOMMU group ID generated.
+And either device's BDF or its VFIO IOMMU group ID in `/dev/vfio/` is fine for "device" in `mountinfo.json`.
+
+```bash
+$ lspci -nn -k -s 45:00.1 
+45:00.1 SCSI storage controller 
+...
+Kernel driver in use: vfio-pci
+...
+$ ls /dev/vfio/110
+/dev/vfio/110
+$ ls /sys/kernel/iommu_groups/110/devices/
+0000:45:00.1
+```
+
+#### setup VFIO device for kata-containers
+
+First, configure the `mountinfo.json`, as below:
+
+- (1) device with `BB:DD:F`
+
+```json
+{
+  "device": "45:00.1",
+  "volume_type": "vfiovol", 
+  "fs_type": "ext4", 
+  "metadata":"{}", 
+  "options": []
+}
+```
+
+- (2) device with `DDDD:BB:DD:F`
+
+```json
+{
+  "device": "0000:45:00.1",
+  "volume_type": "vfiovol", 
+  "fs_type": "ext4", 
+  "metadata":"{}", 
+  "options": []
+}
+```
+
+- (3) device with `/dev/vfio/X`
+
+```json
+{
+  "device": "/dev/vfio/110", 
+  "volume_type": "vfiovol", 
+  "fs_type": "ext4", 
+  "metadata":"{}", 
+  "options": []
+}
+```
+
+Second, run kata-containers with device(`/dev/vfio/110`) as an example:
+
+```bash
+$ sudo kata-ctl direct-volume add /kubelet/kata-vfio-vol-003/vfiovol003 "{\"device\": \"/dev/vfio/110\", \"volume_type\": \"vfiovol\", \"fs_type\": \"ext4\", \"metadata\":"{}", \"options\": []}"
+$ # /kubelet/kata-vfio-vol-003/directvol003 <==> /run/kata-containers/shared/direct-volumes/F0va22F0ZvaS12F0YS10a2F0DAxvbC0F0ZXvdm9sdF0Z0YSx
+$ cat F0va22F0ZvaS12F0YS10a2F0DAxvbC0F0ZXvdm9sdF0Z0YSx/mountInfo.json 
+{"volume_type":"vfiovol","device":"/dev/vfio/110","fs_type":"ext4","metadata":{},"options":[]}
+```
+
+#### Run a Kata container with VFIO block device based volume
+
+```bash
+$ # type=disrectvol,src=/kubelet/kata-vfio-vol-003/vfiovol003,dst=/disk003,options=rbind:rw
+$ sudo ctr run -t --rm --runtime io.containerd.kata.v2 --mount type=vfiovol,src=/kubelet/kata-vfio-vol-003/vfiovol003,dst=/disk003,options=rbind:rw "$image" kata-vfio-vol-xx05302245 /bin/bash
+```
+
+
+### SPDK Device Based Block Volume
+
+SPDK vhost-user devices in runtime-rs, unlike runtime (golang version), there is no need to `mknod` device node under `/dev/` any more.
+Just using the `kata-ctl direct-volume add ..` to make a mount info config is enough.
+
+#### Run SPDK vhost target and Expose vhost block device
+
+Run a SPDK vhost target and get vhost-user block controller as an example:
+
+First, run SPDK vhost target:
+
+> **Tips:** If driver `vfio-pci` supported, you can run SPDK with `DRIVER_OVERRIDE=vfio-pci`
+> Otherwise, Just run without it `sudo HUGEMEM=4096 ./scripts/setup.sh`.
+
+```bash
+$ SPDK_DEVEL=/xx/spdk
+$ VHU_UDS_PATH=/tmp/vhu-targets
+$ RAW_DISKS=/xx/rawdisks
+$ # Reset first
+$ ${SPDK_DEVEL}/scripts/setup.sh reset
+$ sudo sysctl -w vm.nr_hugepages=2048
+$ #4G Huge Memory for spdk
+$ sudo HUGEMEM=4096 DRIVER_OVERRIDE=vfio-pci ${SPDK_DEVEL}/scripts/setup.sh
+$ sudo ${SPDK_DEVEL}/build/bin/spdk_tgt -S $VHU_UDS_PATH -s 1024 -m 0x3 &
+```
+
+Second, create a vhost controller:
+
+```bash
+$ sudo dd if=/dev/zero of=${RAW_DISKS}/rawdisk01.20g bs=1M count=20480
+$ sudo ${SPDK_DEVEL}/scripts/rpc.py bdev_aio_create ${RAW_DISKS}/rawdisk01.20g vhu-rawdisk01.20g 512
+$ sudo ${SPDK_DEVEL}/scripts/rpc.py vhost_create_blk_controller vhost-blk-rawdisk01.sock vhu-rawdisk01.20g
+```
+
+Here, a vhost controller `vhost-blk-rawdisk01.sock` is created, and the controller will
+be passed to Hypervisor, such as Dragonball, Cloud-Hypervisor, Firecracker or QEMU.
+
+
+#### setup vhost-user block device for kata-containers
+
+
+First, `mkdir` a sub-path `kubelet/kata-test-vol-001/` under `/run/kata-containers/shared/direct-volumes/`.
+
+Second, fill fields in `mountinfo.json`, it looks like as below:
+```json
+{
+  "device": "/tmp/vhu-targets/vhost-blk-rawdisk01.sock",
+  "volume_type": "spdkvol",
+  "fs_type": "ext4",
+  "metadata":"{}",
+  "options": []
+}
+```
+
+Third, with the help of `kata-ctl direct-volume` to add block device to generate `mountinfo.json`, and run a kata container with `--mount`.
+
+```bash
+$ # kata-ctl direct-volume add
+$ sudo kata-ctl direct-volume add /kubelet/kata-test-vol-001/volume001 "{\"device\": \"/tmp/vhu-targets/vhost-blk-rawdisk01.sock\", \"volume_type\":\"spdkvol\", \"fs_type\": \"ext4\", \"metadata\":"{}", \"options\": []}"
+$ # /kubelet/kata-test-vol-001/volume001 <==> /run/kata-containers/shared/direct-volumes/L2t1YmVsZXQva2F0YS10ZXN0LXZvbC0wMDEvdm9sdW1lMDAx
+$ cat L2t1YmVsZXQva2F0YS10ZXN0LXZvbC0wMDEvdm9sdW1lMDAx/mountInfo.json
+$ {"volume_type":"spdkvol","device":"/tmp/vhu-targets/vhost-blk-rawdisk01.sock","fs_type":"ext4","metadata":{},"options":[]}
+```
+
+As `/run/kata-containers/shared/direct-volumes/` is a fixed path , we will be able to run a kata pod with `--mount` and set
+`src` sub-path. And the `--mount` argument looks like: `--mount type=spdkvol,src=/kubelet/kata-test-vol-001/volume001,dst=/disk001`.
+
+
+#### Run a Kata container with SPDK vhost-user block device
+
+
+In the case, `ctr run --mount type=X, src=source, dst=dest`, the X will be set `spdkvol` which is a proprietary type specifically designed for SPDK volumes.
+
+```bash
+$ # ctr run with --mount type=spdkvol,src=/kubelet/kata-test-vol-001/volume001,dst=/disk001
+$ sudo ctr run -t --rm --runtime io.containerd.kata.v2 --mount type=spdkvol,src=/kubelet/kata-test-vol-001/volume001,dst=/disk001,options=rbind:rw "$image" kata-spdk-vol-xx0530 /bin/bash
+```

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -94,16 +94,6 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.enable_guest_swap` | `boolean` | enable swap in the guest |
 | `io.katacontainers.config.hypervisor.use_legacy_serial` | `boolean` | uses legacy serial device for guest's console (QEMU) |
 
-## Confidential Computing Options
-| Key | Value Type | Comments |
-|-------| ----- | ----- |
-| `io.katacontainers.config.pre_attestation.enabled"` | `bool` |
-determines if SEV/-ES attestation is enabled |
-| `io.katacontainers.config.pre_attestation.uri"` | `string` |
-specify the location of the attestation server |
-| `io.katacontainers.config.sev.policy"` | `uint32` |
-specify the SEV guest policy |
-
 ## Container Options
 | Key | Value Type | Comments |
 |-------| ----- | ----- |

docs/how-to/how-to-setup-swap-devices-in-guest-kernel.md

@@ -27,8 +27,6 @@ $ image="quay.io/prometheus/busybox:latest"
 $ cat << EOF > "${pod_yaml}"
 metadata:
   name: busybox-sandbox1
-  uid: $(uuidgen)
-  namespace: default
 EOF
 $ cat << EOF > "${container_yaml}"
 metadata:

docs/how-to/how-to-use-virtio-fs-nydus-with-kata.md

@@ -32,7 +32,6 @@ The `nydus-sandbox.yaml` looks like below:
 metadata:
   attempt: 1
   name: nydus-sandbox
-  uid: nydus-uid
   namespace: default
 log_directory: /tmp
 linux:

docs/how-to/how-to-use-virtio-mem-with-kata.md

@@ -42,8 +42,6 @@ $ image="quay.io/prometheus/busybox:latest"
 $ cat << EOF > "${pod_yaml}"
 metadata:
   name: busybox-sandbox1
-  uid: $(uuidgen)
-  namespace: default
 EOF
 $ cat << EOF > "${container_yaml}"
 metadata:

docs/install/README.md

@@ -19,7 +19,6 @@ Packaged installation methods uses your distribution's native package format (su
 |------------------------------------------------------|----------------------------------------------------------------------------------------------|-------------------|-----------------------------------------------------------------------------------------------|
 | [Using kata-deploy](#kata-deploy-installation)       | The preferred way to deploy the Kata Containers distributed binaries on a Kubernetes cluster | **No!**           | Best way to give it a try on kata-containers on an already up and running Kubernetes cluster. | 
 | [Using official distro packages](#official-packages) | Kata packages provided by Linux distributions official repositories                          | yes               | Recommended for most users.                                                                   |
-| ~~[Using snap](#snap-installation)~~                     | ~~Easy to install~~                                                                              | ~~yes~~               | **Snap is unmaintained!** ~~Good alternative to official distro packages.~~                                                 |
 | [Automatic](#automatic-installation)                 | Run a single command to install a full system                                                | **No!**           | For those wanting the latest release quickly.                                                 |
 | [Manual](#manual-installation)                       | Follow a guide step-by-step to install a working system                                      | **No!**           | For those who want the latest release with more control.                                      |
 | [Build from source](#build-from-source-installation) | Build the software components manually                                                       | **No!**           | Power users and developers only.                                                              |
@@ -42,27 +41,6 @@ Kata packages are provided by official distribution repositories for:
 | [CentOS](centos-installation-guide.md)                   | 8                                                                              |
 | [Fedora](fedora-installation-guide.md)                   | 34                                                                             |
 
-### Snap Installation
-
-> **WARNING:**
->
-> The Snap package method is **unmaintained** and only provides an old
-> version of Kata Containers:
-> The [latest Kata Containers snap](https://snapcraft.io/kata-containers)
-> provides Kata Containers
-> [version 2.4.2](https://github.com/kata-containers/kata-containers/releases/tag/2.4.2)
-> but the latest stable Kata Containers release at the time of writing is
-> [version 3.1.0](https://github.com/kata-containers/kata-containers/releases/tag/3.1.0).
->
-> We recommend strongly that you switch to an alternative Kata Containers installation method.
->
-> See: https://github.com/kata-containers/kata-containers/issues/6769
-> for further details.
-
-~~The snap installation is available for all distributions which support `snapd`.~~
-
-~~[Use snap](snap-installation-guide.md) to install Kata Containers from https://snapcraft.io. ~~
-
 ### Automatic Installation
 
 [Use `kata-manager`](/utils/README.md) to automatically install a working Kata Containers system.

docs/install/kata-containers-3.0-rust-runtime-installation-guide.md

@@ -26,7 +26,6 @@ architectures:
 |------------------------------------------------------|----------------------------------------------------------------------------------------------|-------------------|-----------------------------------------------------------------------------------------------|----------- |
 | [Using kata-deploy](#kata-deploy-installation)       | The preferred way to deploy the Kata Containers distributed binaries on a Kubernetes cluster | **No!**           | Best way to give it a try on kata-containers on an already up and running Kubernetes cluster. | Yes |
 | [Using official distro packages](#official-packages) | Kata packages provided by Linux distributions official repositories                          | yes               | Recommended for most users. | No |                                                                   
-| [Using snap](#snap-installation)                     | Easy to install                                                                              | yes               | Good alternative to official distro packages.                                                 | No |
 | [Automatic](#automatic-installation)                 | Run a single command to install a full system                                                | **No!**           | For those wanting the latest release quickly.                                                 | No |
 | [Manual](#manual-installation)                       | Follow a guide step-by-step to install a working system                                      | **No!**           | For those who want the latest release with more control.                                      | No |
 | [Build from source](#build-from-source-installation) | Build the software components manually                                                       | **No!**           | Power users and developers only.  | Yes |              
@@ -36,8 +35,6 @@ architectures:
 Follow the [`kata-deploy`](../../tools/packaging/kata-deploy/README.md).
 ### Official packages
 `ToDo`
-### Snap Installation
-`ToDo`
 ### Automatic Installation
 `ToDo`
 ### Manual Installation

docs/install/snap-installation-guide.md

@@ -1,82 +0,0 @@
-# Kata Containers snap package
-
-> **WARNING:**
->
-> The Snap package method is **unmaintained** and only provides an old
-> version of Kata Containers:
-> The [latest Kata Containers snap](https://snapcraft.io/kata-containers)
-> provides Kata Containers
-> [version 2.4.2](https://github.com/kata-containers/kata-containers/releases/tag/2.4.2)
-> but the latest stable Kata Containers release at the time of writing is
-> [version 3.1.0](https://github.com/kata-containers/kata-containers/releases/tag/3.1.0).
->
-> We recommend strongly that you switch to an alternative Kata Containers installation method.
->
-> See: https://github.com/kata-containers/kata-containers/issues/6769
-> for further details.
-
-## Install Kata Containers
-
-Kata Containers can be installed in any Linux distribution that supports
-[snapd](https://docs.snapcraft.io/installing-snapd).
-
-Run the following command to install **Kata Containers**:
-
-> **WARNING:**
->
-> The Snap package method is **unmaintained** and only provides an old
-> version of Kata Containers:
-> The [latest Kata Containers snap](https://snapcraft.io/kata-containers)
-> provides Kata Containers
-> [version 2.4.2](https://github.com/kata-containers/kata-containers/releases/tag/2.4.2)
-> but the latest stable Kata Containers release at the time of writing is
-> [version 3.1.0](https://github.com/kata-containers/kata-containers/releases/tag/3.1.0).
->
-> We recommend strongly that you switch to an alternative Kata Containers installation method.
->
-> See: https://github.com/kata-containers/kata-containers/issues/6769
-> for further details.
-
-```sh
-$ sudo snap install kata-containers --stable --classic
-```
-
-## Configure Kata Containers
-
-By default Kata Containers snap image is mounted at `/snap/kata-containers` as a
-read-only file system, therefore default configuration file can not be edited.
-Fortunately Kata Containers supports loading a configuration file from another
-path than the default.
-
-```sh
-$ sudo mkdir -p /etc/kata-containers
-$ sudo cp /snap/kata-containers/current/usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers/
-$ $EDITOR /etc/kata-containers/configuration.toml
-```
-
-## Integration with shim v2 Container Engines
-
-The Container engine daemon (`cri-o`, `containerd`, etc) needs to be able to find the
-`containerd-shim-kata-v2` binary to allow Kata Containers to be created.
-Run the following command to create a symbolic link to the shim v2 binary.
-
-```sh
-$ sudo ln -sf /snap/kata-containers/current/usr/bin/containerd-shim-kata-v2 /usr/local/bin/containerd-shim-kata-v2
-```
-
-Once the symbolic link has been created and the engine daemon configured, `io.containerd.kata.v2`
-can be used as runtime.
-
-Read the following documents to know how to run Kata Containers 2.x with `containerd`.
-
-* [How to use Kata Containers and Containerd](../how-to/containerd-kata.md)
-* [Install Kata Containers with containerd](./container-manager/containerd/containerd-install.md)
-
-
-## Remove Kata Containers snap package
-
-Run the following command to remove the Kata Containers snap:
-
-```sh
-$ sudo snap remove kata-containers
-```

snap/local/README.md

@@ -1,101 +0,0 @@
-# Kata Containers snap image
-
-This directory contains the resources needed to build the Kata Containers
-[snap][1] image.
-
-## Initial setup
-
-Kata Containers can be installed in any Linux distribution that supports
-[snapd](https://docs.snapcraft.io/installing-snapd). For this example, we
-assume Ubuntu as your base distro.
-```sh
-$ sudo apt-get --no-install-recommends install -y apt-utils ca-certificates snapd snapcraft
-```
-
-## Install snap
-
-You can install the Kata Containers snap from the [snapcraft store][8] or by running the following command:
-
-```sh
-$ sudo snap install kata-containers --classic
-```
-
-## Build and install snap image
-
-Run the command below which will use the packaging Makefile to build the snap image:
-
-```sh
-$ make -C tools/packaging snap
-```
-
-> **Warning:**
->
-> By default, `snapcraft` will create a clean virtual machine
-> environment to build the snap in using the `multipass` tool.
->
-> However, `multipass` is silently disabled when `--destructive-mode` is
-> used.
->
-> Since building the Kata Containers package currently requires
-> `--destructive-mode`, the snap will be built using the host
-> environment. To avoid parts of the build auto-detecting additional
-> features to enable (for example for QEMU), we recommend that you
-> only run the snap build in a minimal host environment.
-
-To install the resulting snap image, snap must be put in [classic mode][3] and the
-security confinement must be disabled (`--classic`). Also since the resulting snap
-has not been signed the verification of signature must be omitted (`--dangerous`).
-
-```sh
-$ sudo snap install --classic --dangerous "kata-containers_${version}_${arch}.snap"
-```
-
-Replace `${version}` with the current version of Kata Containers and `${arch}` with
-the system architecture.
-
-## Configure Kata Containers
-
-By default Kata Containers snap image is mounted at `/snap/kata-containers` as a
-read-only file system, therefore default configuration file can not be edited.
-Fortunately [`kata-runtime`][4] supports loading a configuration file from another
-path than the default.
-
-```sh
-$ sudo mkdir -p /etc/kata-containers
-$ sudo cp /snap/kata-containers/current/usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers/
-$ $EDITOR /etc/kata-containers/configuration.toml
-```
-
-## Integration with docker and Kubernetes
-
-The path to the runtime provided by the Kata Containers snap image is
-`/snap/kata-containers/current/usr/bin/kata-runtime`. You should use it to
-run Kata Containers with [docker][9] and [Kubernetes][10].
-
-## Remove snap
-
-You can remove the Kata Containers snap by running the following command:
-
-```sh
-$ sudo snap remove kata-containers
-```
-
-## Limitations
-
-The [miniOS image][2] is not included in the snap image as it is not possible for
-QEMU to open a guest RAM backing store on a read-only filesystem. Fortunately,
-you can start Kata Containers with a Linux initial RAM disk (initrd) that is
-included in the snap image. If you want to use the miniOS image instead of initrd,
-then a new configuration file can be [created](#configure-kata-containers)
-and [configured][7].
-
-[1]: https://docs.snapcraft.io/snaps/intro
-[2]: ../../docs/design/architecture/README.md#root-filesystem-image
-[3]: https://docs.snapcraft.io/reference/confinement#classic
-[4]: https://github.com/kata-containers/kata-containers/tree/main/src/runtime#configuration
-[5]: https://docs.docker.com/engine/reference/commandline/dockerd
-[6]: ../../docs/install/docker/ubuntu-docker-install.md
-[7]: ../../docs/Developer-Guide.md#configure-to-use-initrd-or-rootfs-image
-[8]: https://snapcraft.io/kata-containers
-[9]: ../../docs/Developer-Guide.md#run-kata-containers-with-docker
-[10]: ../../docs/Developer-Guide.md#run-kata-containers-with-kubernetes

snap/local/snap-common.sh

@@ -1,114 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2022 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-
-# Description: Idempotent script to be sourced by all parts in a
-#   snapcraft config file.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-# XXX: Bash-specific code. zsh doesn't support this option and that *does*
-# matter if this script is run sourced... since it'll be using zsh! ;)
-[ -n "$BASH_VERSION" ] && set -o errtrace
-
-[ -n "${DEBUG:-}" ] && set -o xtrace
-
-die()
-{
-	echo >&2 "ERROR: $0: $*"
-}
-
-[ -n "${SNAPCRAFT_STAGE:-}" ] ||\
-	die "must be sourced from a snapcraft config file"
-
-snap_yq_version=3.4.1
-
-snap_common_install_yq()
-{
-      export yq="${SNAPCRAFT_STAGE}/bin/yq"
-
-      local yq_pkg
-      yq_pkg="github.com/mikefarah/yq"
-
-      local yq_url
-      yq_url="https://${yq_pkg}/releases/download/${snap_yq_version}/yq_${goos}_${goarch}"
-      curl -o "${yq}" -L "${yq_url}"
-      chmod +x "${yq}"
-}
-
-# Function that should be called for each snap "part" in
-# snapcraft.yaml.
-snap_common_main()
-{
-	# Architecture
-	arch="$(uname -m)"
-
-	case "${arch}" in
-		aarch64)
-			goarch="arm64"
-			qemu_arch="${arch}"
-			;;
-
-		ppc64le)
-			goarch="ppc64le"
-			qemu_arch="ppc64"
-			;;
-
-		s390x)
-			goarch="${arch}"
-			qemu_arch="${arch}"
-			;;
-
-		x86_64)
-			goarch="amd64"
-			qemu_arch="${arch}"
-			;;
-
-		*) die "unsupported architecture: ${arch}" ;;
-	esac
-
-	dpkg_arch=$(dpkg --print-architecture)
-
-	# golang
-	#
-	# We need the O/S name in golang format, but since we don't
-	# know if the godeps part has run, we don't know if golang is
-	# available yet, hence fall back to a standard system command.
-	goos="$(go env GOOS &>/dev/null || true)"
-	[ -z "$goos" ] && goos=$(uname -s|tr '[A-Z]' '[a-z]')
-
-	export GOROOT="${SNAPCRAFT_STAGE}"
-	export GOPATH="${GOROOT}/gopath"
-	export GO111MODULE="auto"
-
-	mkdir -p "${GOPATH}/bin"
-	export PATH="${GOPATH}/bin:${PATH}"
-
-	# Proxy
-	export http_proxy="${http_proxy:-}"
-	export https_proxy="${https_proxy:-}"
-
-	# Binaries
-	mkdir -p "${SNAPCRAFT_STAGE}/bin"
-
-	export PATH="$PATH:${SNAPCRAFT_STAGE}/bin"
-
-	# YAML query tool
-	export yq="${SNAPCRAFT_STAGE}/bin/yq"
-
-	# Kata paths
-	export kata_dir=$(printf "%s/src/github.com/%s/%s" \
-		"${GOPATH}" \
-		"${SNAPCRAFT_PROJECT_NAME}" \
-		"${SNAPCRAFT_PROJECT_NAME}")
-
-	export versions_file="${kata_dir}/versions.yaml"
-
-	[ -n "${yq:-}" ] && [ -x "${yq:-}" ] || snap_common_install_yq
-}
-
-snap_common_main

snap/snapcraft.yaml

@@ -1,170 +0,0 @@
-name: kata-containers
-website: https://github.com/kata-containers/kata-containers
-summary: Build lightweight VMs that seamlessly plug into the containers ecosystem
-description: |
-  Kata Containers is an open source project and community working to build a
-  standard implementation of lightweight Virtual Machines (VMs) that feel and
-  perform like containers, but provide the workload isolation and security
-  advantages of VMs
-confinement: classic
-adopt-info: metadata
-base: core20
-
-parts:
-  metadata:
-    plugin: nil
-    prime:
-      - -*
-    build-packages:
-      - git
-      - git-extras
-    override-pull: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
-      version="9999"
-
-      if echo "${GITHUB_REF:-}" | grep -q -E "^refs/tags"; then
-        version=$(echo ${GITHUB_REF:-} | cut -d/ -f3)
-        git checkout ${version}
-      fi
-
-      snapcraftctl set-grade "stable"
-      snapcraftctl set-version "${version}"
-
-      mkdir -p $(dirname ${kata_dir})
-      ln -sf $(realpath "${SNAPCRAFT_STAGE}/..") ${kata_dir}
-
-  docker:
-    after: [metadata]
-    plugin: nil
-    prime:
-      - -*
-    build-packages:
-      - ca-certificates
-      - containerd
-      - curl
-      - gnupg
-      - lsb-release
-      - runc
-    override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
-      curl -fsSL https://download.docker.com/linux/ubuntu/gpg |\
-          sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
-      distro_codename=$(lsb_release -cs)
-      echo "deb [arch=${dpkg_arch} signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu ${distro_codename} stable" |\
-          sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
-      sudo apt-get -y update
-      sudo apt-get -y install docker-ce docker-ce-cli containerd.io
-
-      echo "Unmasking docker service"
-      sudo -E systemctl unmask docker.service || true
-      sudo -E systemctl unmask docker.socket || true
-      echo "Adding $USER into docker group"
-      sudo -E gpasswd -a $USER docker
-      echo "Starting docker"
-      # docker may fail to start using "fd://" in docker.service
-      sudo sed -i 's/fd:\/\//unix:\/\//g' /lib/systemd/system/docker.service
-      sudo systemctl daemon-reload
-      sudo -E systemctl start docker || true
-
-  image:
-    after: [docker]
-    plugin: nil
-    override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
-      cd "${SNAPCRAFT_PROJECT_DIR}"
-      sudo -E NO_TTY=true make rootfs-image-tarball
-
-      tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-rootfs-image.tar.xz"
-
-      tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}"
-
-
-      sudo -E NO_TTY=true make rootfs-initrd-tarball
-
-      tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-rootfs-initrd.tar.xz"
-
-      tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}"
-
-
-  runtime:
-    after: [docker]
-    plugin: nil
-    override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
-      cd "${SNAPCRAFT_PROJECT_DIR}"
-      sudo -E NO_TTY=true make shim-v2-tarball
-
-      tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-shim-v2.tar.xz"
-
-      tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}"
-
-      mkdir -p "${SNAPCRAFT_PART_INSTALL}/usr/bin"
-      ln -sf "${SNAPCRAFT_PART_INSTALL}/opt/kata/bin/containerd-shim-kata-v2" "${SNAPCRAFT_PART_INSTALL}/usr/bin/containerd-shim-kata-v2"
-      ln -sf "${SNAPCRAFT_PART_INSTALL}/opt/kata/bin/kata-runtime" "${SNAPCRAFT_PART_INSTALL}/usr/bin/kata-runtime"
-      ln -sf "${SNAPCRAFT_PART_INSTALL}/opt/kata/bin/kata-collect-data.sh" "${SNAPCRAFT_PART_INSTALL}/usr/bin/kata-collect-data.sh"
-
-  kernel:
-    after: [docker]
-    plugin: nil
-    override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
-      cd "${SNAPCRAFT_PROJECT_DIR}"
-      sudo -E NO_TTY=true make kernel-tarball
-
-      tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-kernel.tar.xz"
-
-      tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}"
-
-  qemu:
-    plugin: make
-    after: [docker]
-    override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
-      cd "${SNAPCRAFT_PROJECT_DIR}"
-      sudo -E NO_TTY=true make qemu-tarball
-
-      tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-qemu.tar.xz"
-
-      tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}"
-
-  virtiofsd:
-    plugin: nil
-    after: [docker]
-    override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
-      cd "${SNAPCRAFT_PROJECT_DIR}"
-      sudo -E NO_TTY=true make virtiofsd-tarball
-
-      tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-virtiofsd.tar.xz"
-
-      tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}"
-
-  cloud-hypervisor:
-    plugin: nil
-    after: [docker]
-    override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
-      if [ "${arch}" == "aarch64" ] || [ "${arch}" == "x86_64" ]; then
-          cd "${SNAPCRAFT_PROJECT_DIR}"
-          sudo -E NO_TTY=true make cloud-hypervisor-tarball
-
-          tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-cloud-hypervisor.tar.xz"
-
-          tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}"
-      fi
-
-apps:
-  runtime:
-    command: usr/bin/kata-runtime
-  shim:
-    command: usr/bin/containerd-shim-kata-v2
-  collect-data:
-    command: usr/bin/kata-collect-data.sh

src/agent/Cargo.toml

@@ -23,12 +23,11 @@ regex = "1.5.6"
 serial_test = "0.5.1"
 kata-sys-util = { path = "../libs/kata-sys-util" }
 kata-types = { path = "../libs/kata-types" }
-url = "2.2.2"
 
 # Async helpers
 async-trait = "0.1.42"
 async-recursion = "0.3.2"
-futures = "0.3.28"
+futures = "0.3.17"
 
 # Async runtime
 tokio = { version = "1.28.1", features = ["full"] }
@@ -44,6 +43,7 @@ ipnetwork = "0.17.0"
 logging = { path = "../libs/logging" }
 slog = "2.5.2"
 slog-scope = "4.1.2"
+slog-term = "2.9.0"
 
 # Redirect ttrpc log calls
 slog-stdlog = "4.0.0"
@@ -67,22 +67,12 @@ serde = { version = "1.0.129", features = ["derive"] }
 toml = "0.5.8"
 clap = { version = "3.0.1", features = ["derive"] }
 
-# "vendored" feature for openssl is required by musl build
-openssl = { version = "0.10.38", features = ["vendored"] }
-
-# Image pull/decrypt
-image-rs = { git = "https://github.com/confidential-containers/image-rs", tag = "v0.6.0", default-features = false, features = ["kata-cc-native-tls"] }
-
-[patch.crates-io]
-oci-distribution = { git = "https://github.com/krustlet/oci-distribution.git", rev = "f44124c" }
-
 [dev-dependencies]
 tempfile = "3.1.0"
 test-utils = { path = "../libs/test-utils" }
 which = "4.3.0"
 
 [workspace]
-resolver = "2"
 members = [
     "rustjail",
 ]

src/agent/Makefile

@@ -26,7 +26,7 @@ export VERSION_COMMIT := $(if $(COMMIT),$(VERSION)-$(COMMIT),$(VERSION))
 EXTRA_RUSTFEATURES :=
 
 ##VAR SECCOMP=yes|no define if agent enables seccomp feature
-SECCOMP := yes
+SECCOMP ?= yes
 
 # Enable seccomp feature of rust build
 ifeq ($(SECCOMP),yes)

src/agent/rustjail/src/cgroups/fs/mod.rs

@@ -39,11 +39,9 @@ use std::path::Path;
 
 const GUEST_CPUS_PATH: &str = "/sys/devices/system/cpu/online";
 
-// Convenience macro to obtain the scope logger
-macro_rules! sl {
-    () => {
-        slog_scope::logger().new(o!("subsystem" => "cgroups"))
-    };
+// Convenience function to obtain the scope logger.
+fn sl() -> slog::Logger {
+    slog_scope::logger().new(o!("subsystem" => "cgroups"))
 }
 
 macro_rules! get_controller_or_return_singular_none {
@@ -82,7 +80,7 @@ impl CgroupManager for Manager {
 
     fn set(&self, r: &LinuxResources, update: bool) -> Result<()> {
         info!(
-            sl!(),
+            sl(),
             "cgroup manager set resources for container. Resources input {:?}", r
         );
 
@@ -120,7 +118,7 @@ impl CgroupManager for Manager {
 
         // set devices resources
         set_devices_resources(&self.cgroup, &r.devices, res);
-        info!(sl!(), "resources after processed {:?}", res);
+        info!(sl(), "resources after processed {:?}", res);
 
         // apply resources
         self.cgroup.apply(res)?;
@@ -197,7 +195,7 @@ impl CgroupManager for Manager {
         if guest_cpuset.is_empty() {
             return Ok(());
         }
-        info!(sl!(), "update_cpuset_path to: {}", guest_cpuset);
+        info!(sl(), "update_cpuset_path to: {}", guest_cpuset);
 
         let h = cgroups::hierarchies::auto();
         let root_cg = h.root_control_group();
@@ -205,12 +203,12 @@ impl CgroupManager for Manager {
         let root_cpuset_controller: &CpuSetController = root_cg.controller_of().unwrap();
         let path = root_cpuset_controller.path();
         let root_path = Path::new(path);
-        info!(sl!(), "root cpuset path: {:?}", &path);
+        info!(sl(), "root cpuset path: {:?}", &path);
 
         let container_cpuset_controller: &CpuSetController = self.cgroup.controller_of().unwrap();
         let path = container_cpuset_controller.path();
         let container_path = Path::new(path);
-        info!(sl!(), "container cpuset path: {:?}", &path);
+        info!(sl(), "container cpuset path: {:?}", &path);
 
         let mut paths = vec![];
         for ancestor in container_path.ancestors() {
@@ -219,7 +217,7 @@ impl CgroupManager for Manager {
             }
             paths.push(ancestor);
         }
-        info!(sl!(), "parent paths to update cpuset: {:?}", &paths);
+        info!(sl(), "parent paths to update cpuset: {:?}", &paths);
 
         let mut i = paths.len();
         loop {
@@ -233,7 +231,7 @@ impl CgroupManager for Manager {
                 .to_str()
                 .unwrap()
                 .trim_start_matches(root_path.to_str().unwrap());
-            info!(sl!(), "updating cpuset for parent path {:?}", &r_path);
+            info!(sl(), "updating cpuset for parent path {:?}", &r_path);
             let cg = new_cgroup(cgroups::hierarchies::auto(), r_path)?;
             let cpuset_controller: &CpuSetController = cg.controller_of().unwrap();
             cpuset_controller.set_cpus(guest_cpuset)?;
@@ -241,7 +239,7 @@ impl CgroupManager for Manager {
 
         if !container_cpuset.is_empty() {
             info!(
-                sl!(),
+                sl(),
                 "updating cpuset for container path: {:?} cpuset: {}",
                 &container_path,
                 container_cpuset
@@ -276,7 +274,7 @@ fn set_network_resources(
     network: &LinuxNetwork,
     res: &mut cgroups::Resources,
 ) {
-    info!(sl!(), "cgroup manager set network");
+    info!(sl(), "cgroup manager set network");
 
     // set classid
     // description can be found at https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v1/net_cls.html
@@ -303,7 +301,7 @@ fn set_devices_resources(
     device_resources: &[LinuxDeviceCgroup],
     res: &mut cgroups::Resources,
 ) {
-    info!(sl!(), "cgroup manager set devices");
+    info!(sl(), "cgroup manager set devices");
     let mut devices = vec![];
 
     for d in device_resources.iter() {
@@ -332,7 +330,7 @@ fn set_hugepages_resources(
     hugepage_limits: &[LinuxHugepageLimit],
     res: &mut cgroups::Resources,
 ) {
-    info!(sl!(), "cgroup manager set hugepage");
+    info!(sl(), "cgroup manager set hugepage");
     let mut limits = vec![];
     let hugetlb_controller = cg.controller_of::<HugeTlbController>();
 
@@ -346,7 +344,7 @@ fn set_hugepages_resources(
             limits.push(hr);
         } else {
             warn!(
-                sl!(),
+                sl(),
                 "{} page size support cannot be verified, dropping requested limit", l.page_size
             );
         }
@@ -359,7 +357,7 @@ fn set_block_io_resources(
     blkio: &LinuxBlockIo,
     res: &mut cgroups::Resources,
 ) {
-    info!(sl!(), "cgroup manager set block io");
+    info!(sl(), "cgroup manager set block io");
 
     res.blkio.weight = blkio.weight;
     res.blkio.leaf_weight = blkio.leaf_weight;
@@ -387,13 +385,13 @@ fn set_block_io_resources(
 }
 
 fn set_cpu_resources(cg: &cgroups::Cgroup, cpu: &LinuxCpu) -> Result<()> {
-    info!(sl!(), "cgroup manager set cpu");
+    info!(sl(), "cgroup manager set cpu");
 
     let cpuset_controller: &CpuSetController = cg.controller_of().unwrap();
 
     if !cpu.cpus.is_empty() {
         if let Err(e) = cpuset_controller.set_cpus(&cpu.cpus) {
-            warn!(sl!(), "write cpuset failed: {:?}", e);
+            warn!(sl(), "write cpuset failed: {:?}", e);
         }
     }
 
@@ -424,7 +422,7 @@ fn set_cpu_resources(cg: &cgroups::Cgroup, cpu: &LinuxCpu) -> Result<()> {
 }
 
 fn set_memory_resources(cg: &cgroups::Cgroup, memory: &LinuxMemory, update: bool) -> Result<()> {
-    info!(sl!(), "cgroup manager set memory");
+    info!(sl(), "cgroup manager set memory");
     let mem_controller: &MemController = cg.controller_of().unwrap();
 
     if !update {
@@ -493,7 +491,7 @@ fn set_memory_resources(cg: &cgroups::Cgroup, memory: &LinuxMemory, update: bool
 }
 
 fn set_pids_resources(cg: &cgroups::Cgroup, pids: &LinuxPids) -> Result<()> {
-    info!(sl!(), "cgroup manager set pids");
+    info!(sl(), "cgroup manager set pids");
     let pid_controller: &PidController = cg.controller_of().unwrap();
     let v = if pids.limit > 0 {
         MaxValue::Value(pids.limit)
@@ -543,11 +541,8 @@ fn linux_device_to_cgroup_device(d: &LinuxDevice) -> Option<DeviceResource> {
 }
 
 fn linux_device_group_to_cgroup_device(d: &LinuxDeviceCgroup) -> Option<DeviceResource> {
-    let dev_type = match &d.r#type {
-        Some(t_s) => match DeviceType::from_char(t_s.chars().next()) {
-            Some(t_c) => t_c,
-            None => return None,
-        },
+    let dev_type = match DeviceType::from_char(d.r#type.chars().next()) {
+        Some(t) => t,
         None => return None,
     };
 
@@ -604,7 +599,7 @@ lazy_static! {
             // all mknod to all char devices
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: Some("c".to_string()),
+                r#type: "c".to_string(),
                 major: Some(WILDCARD),
                 minor: Some(WILDCARD),
                 access: "m".to_string(),
@@ -613,7 +608,7 @@ lazy_static! {
             // all mknod to all block devices
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: Some("b".to_string()),
+                r#type: "b".to_string(),
                 major: Some(WILDCARD),
                 minor: Some(WILDCARD),
                 access: "m".to_string(),
@@ -622,7 +617,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/console
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: Some("c".to_string()),
+                r#type: "c".to_string(),
                 major: Some(5),
                 minor: Some(1),
                 access: "rwm".to_string(),
@@ -631,7 +626,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/pts/<N>
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: Some("c".to_string()),
+                r#type: "c".to_string(),
                 major: Some(136),
                 minor: Some(WILDCARD),
                 access: "rwm".to_string(),
@@ -640,7 +635,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/ptmx
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: Some("c".to_string()),
+                r#type: "c".to_string(),
                 major: Some(5),
                 minor: Some(2),
                 access: "rwm".to_string(),
@@ -649,7 +644,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/net/tun
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: Some("c".to_string()),
+                r#type: "c".to_string(),
                 major: Some(10),
                 minor: Some(200),
                 access: "rwm".to_string(),
@@ -965,7 +960,7 @@ pub fn get_paths() -> Result<HashMap<String, String>> {
     for l in fs::read_to_string(PATHS)?.lines() {
         let fl: Vec<&str> = l.split(':').collect();
         if fl.len() != 3 {
-            info!(sl!(), "Corrupted cgroup data!");
+            info!(sl(), "Corrupted cgroup data!");
             continue;
         }
 
@@ -986,7 +981,7 @@ pub fn get_mounts(paths: &HashMap<String, String>) -> Result<HashMap<String, Str
         let post: Vec<&str> = p[1].split(' ').collect();
 
         if post.len() != 3 {
-            warn!(sl!(), "can't parse {} line {:?}", MOUNTS, l);
+            warn!(sl(), "can't parse {} line {:?}", MOUNTS, l);
             continue;
         }
 

src/agent/rustjail/src/cgroups/notifier.rs

@@ -16,11 +16,9 @@ use inotify::{Inotify, WatchMask};
 use tokio::io::AsyncReadExt;
 use tokio::sync::mpsc::{channel, Receiver};
 
-// Convenience macro to obtain the scope logger
-macro_rules! sl {
-    () => {
-        slog_scope::logger().new(o!("subsystem" => "cgroups_notifier"))
-    };
+// Convenience function to obtain the scope logger.
+fn sl() -> slog::Logger {
+    slog_scope::logger().new(o!("subsystem" => "cgroups_notifier"))
 }
 
 pub async fn notify_oom(cid: &str, cg_dir: String) -> Result<Receiver<String>> {
@@ -38,7 +36,7 @@ pub async fn notify_oom(cid: &str, cg_dir: String) -> Result<Receiver<String>> {
 fn get_value_from_cgroup(path: &Path, key: &str) -> Result<i64> {
     let content = fs::read_to_string(path)?;
     info!(
-        sl!(),
+        sl(),
         "get_value_from_cgroup file: {:?}, content: {}", &path, &content
     );
 
@@ -67,11 +65,11 @@ async fn register_memory_event_v2(
     let event_control_path = Path::new(&cg_dir).join(memory_event_name);
     let cgroup_event_control_path = Path::new(&cg_dir).join(cgroup_event_name);
     info!(
-        sl!(),
+        sl(),
         "register_memory_event_v2 event_control_path: {:?}", &event_control_path
     );
     info!(
-        sl!(),
+        sl(),
         "register_memory_event_v2 cgroup_event_control_path: {:?}", &cgroup_event_control_path
     );
 
@@ -82,8 +80,8 @@ async fn register_memory_event_v2(
     // Because no `unix.IN_DELETE|unix.IN_DELETE_SELF` event for cgroup file system, so watching all process exited
     let cg_wd = inotify.add_watch(&cgroup_event_control_path, WatchMask::MODIFY)?;
 
-    info!(sl!(), "ev_wd: {:?}", ev_wd);
-    info!(sl!(), "cg_wd: {:?}", cg_wd);
+    info!(sl(), "ev_wd: {:?}", ev_wd);
+    info!(sl(), "cg_wd: {:?}", cg_wd);
 
     let (sender, receiver) = channel(100);
     let containere_id = containere_id.to_string();
@@ -97,17 +95,17 @@ async fn register_memory_event_v2(
         while let Some(event_or_error) = stream.next().await {
             let event = event_or_error.unwrap();
             info!(
-                sl!(),
+                sl(),
                 "container[{}] get event for container: {:?}", &containere_id, &event
             );
             // info!("is1: {}", event.wd == wd1);
-            info!(sl!(), "event.wd: {:?}", event.wd);
+            info!(sl(), "event.wd: {:?}", event.wd);
 
             if event.wd == ev_wd {
                 let oom = get_value_from_cgroup(&event_control_path, "oom_kill");
                 if oom.unwrap_or(0) > 0 {
                     let _ = sender.send(containere_id.clone()).await.map_err(|e| {
-                        error!(sl!(), "send containere_id failed, error: {:?}", e);
+                        error!(sl(), "send containere_id failed, error: {:?}", e);
                     });
                     return;
                 }
@@ -171,13 +169,13 @@ async fn register_memory_event(
             let mut buf = [0u8; 8];
             match eventfd_stream.read(&mut buf).await {
                 Err(err) => {
-                    warn!(sl!(), "failed to read from eventfd: {:?}", err);
+                    warn!(sl(), "failed to read from eventfd: {:?}", err);
                     return;
                 }
                 Ok(_) => {
                     let content = fs::read_to_string(path.clone());
                     info!(
-                        sl!(),
+                        sl(),
                         "cgroup event for container: {}, path: {:?}, content: {:?}",
                         &containere_id,
                         &path,
@@ -193,7 +191,7 @@ async fn register_memory_event(
             }
 
             let _ = sender.send(containere_id.clone()).await.map_err(|e| {
-                error!(sl!(), "send containere_id failed, error: {:?}", e);
+                error!(sl(), "send containere_id failed, error: {:?}", e);
             });
         }
     });

src/agent/rustjail/src/container.rs

@@ -1596,10 +1596,8 @@ mod tests {
     use tempfile::tempdir;
     use test_utils::skip_if_not_root;
 
-    macro_rules! sl {
-        () => {
-            slog_scope::logger()
-        };
+    fn sl() -> slog::Logger {
+        slog_scope::logger()
     }
 
     #[test]
@@ -1854,7 +1852,7 @@ mod tests {
         let _ = new_linux_container_and_then(|mut c: LinuxContainer| {
             c.processes.insert(
                 1,
-                Process::new(&sl!(), &oci::Process::default(), "123", true, 1).unwrap(),
+                Process::new(&sl(), &oci::Process::default(), "123", true, 1).unwrap(),
             );
             let p = c.get_process("123");
             assert!(p.is_ok(), "Expecting Ok, Got {:?}", p);
@@ -1881,7 +1879,7 @@ mod tests {
         let (c, _dir) = new_linux_container();
         let ret = c
             .unwrap()
-            .start(Process::new(&sl!(), &oci::Process::default(), "123", true, 1).unwrap())
+            .start(Process::new(&sl(), &oci::Process::default(), "123", true, 1).unwrap())
             .await;
         assert!(ret.is_err(), "Expecting Err, Got {:?}", ret);
     }
@@ -1891,7 +1889,7 @@ mod tests {
         let (c, _dir) = new_linux_container();
         let ret = c
             .unwrap()
-            .run(Process::new(&sl!(), &oci::Process::default(), "123", true, 1).unwrap())
+            .run(Process::new(&sl(), &oci::Process::default(), "123", true, 1).unwrap())
             .await;
         assert!(ret.is_err(), "Expecting Err, Got {:?}", ret);
     }

src/agent/rustjail/src/lib.rs

@@ -241,12 +241,6 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
     let devices = {
         let mut d = Vec::new();
         for dev in res.Devices.iter() {
-            let dev_type = if dev.Type.is_empty() {
-                None
-            } else {
-                Some(dev.Type.clone())
-            };
-
             let major = if dev.Major == -1 {
                 None
             } else {
@@ -260,7 +254,7 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
             };
             d.push(oci::LinuxDeviceCgroup {
                 allow: dev.Allow,
-                r#type: dev_type,
+                r#type: dev.Type.clone(),
                 major,
                 minor,
                 access: dev.Access.clone(),

src/agent/rustjail/src/mount.rs

@@ -1118,6 +1118,7 @@ mod tests {
     use std::fs::create_dir;
     use std::fs::create_dir_all;
     use std::fs::remove_dir_all;
+    use std::fs::remove_file;
     use std::io;
     use std::os::unix::fs;
     use std::os::unix::io::AsRawFd;
@@ -1333,14 +1334,9 @@ mod tests {
     fn test_mknod_dev() {
         skip_if_not_root!();
 
-        let tempdir = tempdir().unwrap();
-
-        let olddir = unistd::getcwd().unwrap();
-        defer!(let _ = unistd::chdir(&olddir););
-        let _ = unistd::chdir(tempdir.path());
-
+        let path = "/dev/fifo-test";
         let dev = oci::LinuxDevice {
-            path: "/fifo".to_string(),
+            path: path.to_string(),
             r#type: "c".to_string(),
             major: 0,
             minor: 0,
@@ -1348,13 +1344,16 @@ mod tests {
             uid: Some(unistd::getuid().as_raw()),
             gid: Some(unistd::getgid().as_raw()),
         };
-        let path = Path::new("fifo");
 
-        let ret = mknod_dev(&dev, path);
+        let ret = mknod_dev(&dev, Path::new(path));
         assert!(ret.is_ok(), "Should pass. Got: {:?}", ret);
 
         let ret = stat::stat(path);
         assert!(ret.is_ok(), "Should pass. Got: {:?}", ret);
+
+        // clear test device node
+        let ret = remove_file(path);
+        assert!(ret.is_ok(), "Should pass, Got: {:?}", ret);
     }
 
     #[test]

src/agent/rustjail/src/process.rs

@@ -161,7 +161,7 @@ impl Process {
 
     pub fn notify_term_close(&mut self) {
         let notify = self.term_exit_notifier.clone();
-        notify.notify_one();
+        notify.notify_waiters();
     }
 
     pub fn close_stdin(&mut self) {

src/agent/src/config.rs

@@ -11,7 +11,6 @@ use std::fs;
 use std::str::FromStr;
 use std::time;
 use tracing::instrument;
-use url::Url;
 
 use kata_types::config::default::DEFAULT_AGENT_VSOCK_PORT;
 
@@ -26,11 +25,6 @@ const LOG_VPORT_OPTION: &str = "agent.log_vport";
 const CONTAINER_PIPE_SIZE_OPTION: &str = "agent.container_pipe_size";
 const UNIFIED_CGROUP_HIERARCHY_OPTION: &str = "agent.unified_cgroup_hierarchy";
 const CONFIG_FILE: &str = "agent.config_file";
-const AA_KBC_PARAMS: &str = "agent.aa_kbc_params";
-const HTTPS_PROXY: &str = "agent.https_proxy";
-const NO_PROXY: &str = "agent.no_proxy";
-const ENABLE_DATA_INTEGRITY: &str = "agent.data_integrity";
-const ENABLE_SIGNATURE_VERIFICATION: &str = "agent.enable_signature_verification";
 
 const DEFAULT_LOG_LEVEL: slog::Level = slog::Level::Info;
 const DEFAULT_HOTPLUG_TIMEOUT: time::Duration = time::Duration::from_secs(3);
@@ -83,12 +77,6 @@ pub struct AgentConfig {
     pub tracing: bool,
     pub endpoints: AgentEndpoints,
     pub supports_seccomp: bool,
-    pub container_policy_path: String,
-    pub aa_kbc_params: String,
-    pub https_proxy: String,
-    pub no_proxy: String,
-    pub data_integrity: bool,
-    pub enable_signature_verification: bool,
 }
 
 #[derive(Debug, Deserialize)]
@@ -104,12 +92,6 @@ pub struct AgentConfigBuilder {
     pub unified_cgroup_hierarchy: Option<bool>,
     pub tracing: Option<bool>,
     pub endpoints: Option<EndpointsConfig>,
-    pub container_policy_path: Option<String>,
-    pub aa_kbc_params: Option<String>,
-    pub https_proxy: Option<String>,
-    pub no_proxy: Option<String>,
-    pub data_integrity: Option<bool>,
-    pub enable_signature_verification: Option<bool>,
 }
 
 macro_rules! config_override {
@@ -171,12 +153,6 @@ impl Default for AgentConfig {
             tracing: false,
             endpoints: Default::default(),
             supports_seccomp: rpc::have_seccomp(),
-            container_policy_path: String::from(""),
-            aa_kbc_params: String::from(""),
-            https_proxy: String::from(""),
-            no_proxy: String::from(""),
-            data_integrity: false,
-            enable_signature_verification: true,
         }
     }
 }
@@ -205,16 +181,6 @@ impl FromStr for AgentConfig {
         config_override!(agent_config_builder, agent_config, server_addr);
         config_override!(agent_config_builder, agent_config, unified_cgroup_hierarchy);
         config_override!(agent_config_builder, agent_config, tracing);
-        config_override!(agent_config_builder, agent_config, container_policy_path);
-        config_override!(agent_config_builder, agent_config, aa_kbc_params);
-        config_override!(agent_config_builder, agent_config, https_proxy);
-        config_override!(agent_config_builder, agent_config, no_proxy);
-        config_override!(agent_config_builder, agent_config, data_integrity);
-        config_override!(
-            agent_config_builder,
-            agent_config,
-            enable_signature_verification
-        );
 
         // Populate the allowed endpoints hash set, if we got any from the config file.
         if let Some(endpoints) = agent_config_builder.endpoints {
@@ -243,10 +209,6 @@ impl AgentConfig {
         let mut config: AgentConfig = Default::default();
         let cmdline = fs::read_to_string(file)?;
         let params: Vec<&str> = cmdline.split_ascii_whitespace().collect();
-
-        let mut using_config_file = false;
-        // Check if there is config file before parsing params that might
-        // override values from the config file.
         for param in params.iter() {
             // If we get a configuration file path from the command line, we
             // generate our config from it.
@@ -254,15 +216,10 @@ impl AgentConfig {
             // or if it can't be parsed properly.
             if param.starts_with(format!("{}=", CONFIG_FILE).as_str()) {
                 let config_file = get_string_value(param)?;
-                config = AgentConfig::from_config_file(&config_file)
-                    .context("AgentConfig from kernel cmdline")
-                    .unwrap();
-                using_config_file = true;
-                break;
+                return AgentConfig::from_config_file(&config_file)
+                    .context("AgentConfig from kernel cmdline");
             }
-        }
 
-        for param in params.iter() {
             // parse cmdline flags
             parse_cmdline_param!(param, DEBUG_CONSOLE_FLAG, config.debug_console);
             parse_cmdline_param!(param, DEV_MODE_FLAG, config.dev_mode);
@@ -322,23 +279,6 @@ impl AgentConfig {
                 config.unified_cgroup_hierarchy,
                 get_bool_value
             );
-
-            parse_cmdline_param!(param, AA_KBC_PARAMS, config.aa_kbc_params, get_string_value);
-            parse_cmdline_param!(param, HTTPS_PROXY, config.https_proxy, get_url_value);
-            parse_cmdline_param!(param, NO_PROXY, config.no_proxy, get_string_value);
-            parse_cmdline_param!(
-                param,
-                ENABLE_DATA_INTEGRITY,
-                config.data_integrity,
-                get_bool_value
-            );
-
-            parse_cmdline_param!(
-                param,
-                ENABLE_SIGNATURE_VERIFICATION,
-                config.enable_signature_verification,
-                get_bool_value
-            );
         }
 
         if let Ok(addr) = env::var(SERVER_ADDR_ENV_VAR) {
@@ -358,9 +298,7 @@ impl AgentConfig {
         }
 
         // We did not get a configuration file: allow all endpoints.
-        if !using_config_file {
-            config.endpoints.all_allowed = true;
-        }
+        config.endpoints.all_allowed = true;
 
         Ok(config)
     }
@@ -495,12 +433,6 @@ fn get_container_pipe_size(param: &str) -> Result<i32> {
     Ok(value)
 }
 
-#[instrument]
-fn get_url_value(param: &str) -> Result<String> {
-    let value = get_string_value(param)?;
-    Ok(Url::parse(&value)?.to_string())
-}
-
 #[cfg(test)]
 mod tests {
     use test_utils::assert_result;
@@ -519,8 +451,6 @@ mod tests {
         assert!(!config.dev_mode);
         assert_eq!(config.log_level, DEFAULT_LOG_LEVEL);
         assert_eq!(config.hotplug_timeout, DEFAULT_HOTPLUG_TIMEOUT);
-        assert_eq!(config.container_policy_path, "");
-        assert!(config.enable_signature_verification);
     }
 
     #[test]
@@ -539,12 +469,6 @@ mod tests {
             server_addr: &'a str,
             unified_cgroup_hierarchy: bool,
             tracing: bool,
-            container_policy_path: &'a str,
-            aa_kbc_params: &'a str,
-            https_proxy: &'a str,
-            no_proxy: &'a str,
-            data_integrity: bool,
-            enable_signature_verification: bool,
         }
 
         impl Default for TestData<'_> {
@@ -560,12 +484,6 @@ mod tests {
                     server_addr: TEST_SERVER_ADDR,
                     unified_cgroup_hierarchy: false,
                     tracing: false,
-                    container_policy_path: "",
-                    aa_kbc_params: "",
-                    https_proxy: "",
-                    no_proxy: "",
-                    data_integrity: false,
-                    enable_signature_verification: true,
                 }
             }
         }
@@ -935,81 +853,6 @@ mod tests {
                 tracing: true,
                 ..Default::default()
             },
-            TestData {
-                contents: "agent.aa_kbc_params=offline_fs_kbc::null",
-                aa_kbc_params: "offline_fs_kbc::null",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.aa_kbc_params=eaa_kbc::127.0.0.1:50000",
-                aa_kbc_params: "eaa_kbc::127.0.0.1:50000",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.https_proxy=http://proxy.url.com:81/",
-                https_proxy: "http://proxy.url.com:81/",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.https_proxy=http://192.168.1.100:81/",
-                https_proxy: "http://192.168.1.100:81/",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.no_proxy=*.internal.url.com",
-                no_proxy: "*.internal.url.com",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.no_proxy=192.168.1.0/24,172.16.0.0/12",
-                no_proxy: "192.168.1.0/24,172.16.0.0/12",
-                ..Default::default()
-            },
-            TestData {
-                contents: "",
-                data_integrity: false,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.data_integrity=true",
-                data_integrity: true,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.data_integrity=false",
-                data_integrity: false,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.data_integrity=1",
-                data_integrity: true,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.data_integrity=0",
-                data_integrity: false,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.enable_signature_verification=false",
-                enable_signature_verification: false,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.enable_signature_verification=0",
-                enable_signature_verification: false,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.enable_signature_verification=1",
-                enable_signature_verification: true,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.enable_signature_verification=foo",
-                enable_signature_verification: false,
-                ..Default::default()
-            },
         ];
 
         let dir = tempdir().expect("failed to create tmpdir");
@@ -1057,20 +900,6 @@ mod tests {
             assert_eq!(d.container_pipe_size, config.container_pipe_size, "{}", msg);
             assert_eq!(d.server_addr, config.server_addr, "{}", msg);
             assert_eq!(d.tracing, config.tracing, "{}", msg);
-            assert_eq!(
-                d.container_policy_path, config.container_policy_path,
-                "{}",
-                msg
-            );
-            assert_eq!(d.aa_kbc_params, config.aa_kbc_params, "{}", msg);
-            assert_eq!(d.https_proxy, config.https_proxy, "{}", msg);
-            assert_eq!(d.no_proxy, config.no_proxy, "{}", msg);
-            assert_eq!(d.data_integrity, config.data_integrity, "{}", msg);
-            assert_eq!(
-                d.enable_signature_verification, config.enable_signature_verification,
-                "{}",
-                msg
-            );
 
             for v in vars_to_unset {
                 env::remove_var(v);
@@ -1572,50 +1401,4 @@ Caused by:
         // Verify that the default values are valid
         assert_eq!(config.hotplug_timeout, DEFAULT_HOTPLUG_TIMEOUT);
     }
-
-    #[test]
-    fn test_config_from_cmdline_and_config_file() {
-        let dir = tempdir().expect("failed to create tmpdir");
-
-        let agent_config = r#"
-               dev_mode = false
-               server_addr = 'vsock://8:2048'
-
-               [endpoints]
-               allowed = ["CreateContainer", "StartContainer"]
-              "#;
-
-        let config_path = dir.path().join("agent-config.toml");
-        let config_filename = config_path.to_str().expect("failed to get config filename");
-
-        fs::write(config_filename, agent_config).expect("failed to write agen config");
-
-        let cmdline = format!("agent.devmode agent.config_file={}", config_filename);
-
-        let cmdline_path = dir.path().join("cmdline");
-        let cmdline_filename = cmdline_path
-            .to_str()
-            .expect("failed to get cmdline filename");
-
-        fs::write(cmdline_filename, cmdline).expect("failed to write agen config");
-
-        let config = AgentConfig::from_cmdline(cmdline_filename, vec![])
-            .expect("failed to parse command line");
-
-        // Should be overwritten by cmdline
-        assert!(config.dev_mode);
-
-        // Should be from agent config
-        assert_eq!(config.server_addr, "vsock://8:2048");
-
-        // Should be from agent config
-        assert_eq!(
-            config.endpoints.allowed,
-            vec!["CreateContainer".to_string(), "StartContainer".to_string()]
-                .iter()
-                .cloned()
-                .collect()
-        );
-        assert!(!config.endpoints.all_allowed);
-    }
 }

src/agent/src/device.rs

@@ -26,11 +26,9 @@ use oci::{LinuxDeviceCgroup, LinuxResources, Spec};
 use protocols::agent::Device;
 use tracing::instrument;
 
-// Convenience macro to obtain the scope logger
-macro_rules! sl {
-    () => {
-        slog_scope::logger().new(o!("subsystem" => "device"))
-    };
+// Convenience function to obtain the scope logger.
+fn sl() -> slog::Logger {
+    slog_scope::logger().new(o!("subsystem" => "device"))
 }
 
 const VM_ROOTFS: &str = "/";
@@ -78,7 +76,7 @@ where
 {
     let syspci = Path::new(&syspci);
     let drv = drv.as_ref();
-    info!(sl!(), "rebind_pci_driver: {} => {:?}", dev, drv);
+    info!(sl(), "rebind_pci_driver: {} => {:?}", dev, drv);
 
     let devpath = syspci.join("devices").join(dev.to_string());
     let overridepath = &devpath.join("driver_override");
@@ -606,7 +604,7 @@ fn update_spec_devices(spec: &mut Spec, mut updates: HashMap<&str, DevUpdate>) -
             let host_minor = specdev.minor;
 
             info!(
-                sl!(),
+                sl(),
                 "update_spec_devices() updating device";
                 "container_path" => &specdev.path,
                 "type" => &specdev.r#type,
@@ -653,15 +651,13 @@ fn update_spec_devices(spec: &mut Spec, mut updates: HashMap<&str, DevUpdate>) -
 
     if let Some(resources) = linux.resources.as_mut() {
         for r in &mut resources.devices {
-            if let (Some(host_type), Some(host_major), Some(host_minor)) =
-                (r.r#type.as_ref(), r.major, r.minor)
-            {
-                if let Some(update) = res_updates.get(&(host_type.as_str(), host_major, host_minor))
+            if let (Some(host_major), Some(host_minor)) = (r.major, r.minor) {
+                if let Some(update) = res_updates.get(&(r.r#type.as_str(), host_major, host_minor))
                 {
                     info!(
-                        sl!(),
+                        sl(),
                         "update_spec_devices() updating resource";
-                        "type" => &host_type,
+                        "type" => &r.r#type,
                         "host_major" => host_major,
                         "host_minor" => host_minor,
                         "guest_major" => update.guest_major,
@@ -923,7 +919,7 @@ pub async fn add_devices(
 #[instrument]
 async fn add_device(device: &Device, sandbox: &Arc<Mutex<Sandbox>>) -> Result<SpecUpdate> {
     // log before validation to help with debugging gRPC protocol version differences.
-    info!(sl!(), "device-id: {}, device-type: {}, device-vm-path: {}, device-container-path: {}, device-options: {:?}",
+    info!(sl(), "device-id: {}, device-type: {}, device-vm-path: {}, device-container-path: {}, device-options: {:?}",
           device.id, device.type_, device.vm_path, device.container_path, device.options);
 
     if device.type_.is_empty() {
@@ -973,7 +969,7 @@ pub fn update_device_cgroup(spec: &mut Spec) -> Result<()> {
         allow: false,
         major: Some(major),
         minor: Some(minor),
-        r#type: Some(String::from("b")),
+        r#type: String::from("b"),
         access: String::from("rw"),
     });
 
@@ -1136,13 +1132,13 @@ mod tests {
                 resources: Some(LinuxResources {
                     devices: vec![
                         oci::LinuxDeviceCgroup {
-                            r#type: Some("c".to_string()),
+                            r#type: "c".to_string(),
                             major: Some(host_major_a),
                             minor: Some(host_minor_a),
                             ..oci::LinuxDeviceCgroup::default()
                         },
                         oci::LinuxDeviceCgroup {
-                            r#type: Some("c".to_string()),
+                            r#type: "c".to_string(),
                             major: Some(host_major_b),
                             minor: Some(host_minor_b),
                             ..oci::LinuxDeviceCgroup::default()
@@ -1235,13 +1231,13 @@ mod tests {
                 resources: Some(LinuxResources {
                     devices: vec![
                         LinuxDeviceCgroup {
-                            r#type: Some("c".to_string()),
+                            r#type: "c".to_string(),
                             major: Some(host_major),
                             minor: Some(host_minor),
                             ..LinuxDeviceCgroup::default()
                         },
                         LinuxDeviceCgroup {
-                            r#type: Some("b".to_string()),
+                            r#type: "b".to_string(),
                             major: Some(host_major),
                             minor: Some(host_minor),
                             ..LinuxDeviceCgroup::default()

src/agent/src/image_rpc.rs

@@ -1,352 +0,0 @@
-// Copyright (c) 2021 Alibaba Cloud
-// Copyright (c) 2021, 2023 IBM Corporation
-// Copyright (c) 2022 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-use std::env;
-use std::fs;
-use std::path::Path;
-use std::process::Command;
-use std::sync::atomic::{AtomicBool, AtomicU16, Ordering};
-use std::sync::Arc;
-
-use anyhow::{anyhow, Result};
-use async_trait::async_trait;
-use protocols::image;
-use tokio::sync::Mutex;
-use ttrpc::{self, error::get_rpc_status as ttrpc_error};
-
-use crate::rpc::{verify_cid, CONTAINER_BASE};
-use crate::sandbox::Sandbox;
-use crate::AGENT_CONFIG;
-
-use image_rs::image::ImageClient;
-use std::io::Write;
-
-const AA_PATH: &str = "/usr/local/bin/attestation-agent";
-
-const AA_KEYPROVIDER_URI: &str =
-    "unix:///run/confidential-containers/attestation-agent/keyprovider.sock";
-const AA_GETRESOURCE_URI: &str =
-    "unix:///run/confidential-containers/attestation-agent/getresource.sock";
-
-const OCICRYPT_CONFIG_PATH: &str = "/tmp/ocicrypt_config.json";
-// kata rootfs is readonly, use tmpfs before CC storage is implemented.
-const KATA_CC_IMAGE_WORK_DIR: &str = "/run/image/";
-const KATA_CC_PAUSE_BUNDLE: &str = "/pause_bundle";
-const CONFIG_JSON: &str = "config.json";
-
-// Convenience macro to obtain the scope logger
-macro_rules! sl {
-    () => {
-        slog_scope::logger()
-    };
-}
-
-pub struct ImageService {
-    sandbox: Arc<Mutex<Sandbox>>,
-    attestation_agent_started: AtomicBool,
-    image_client: Arc<Mutex<ImageClient>>,
-    container_count: Arc<AtomicU16>,
-}
-
-impl ImageService {
-    pub fn new(sandbox: Arc<Mutex<Sandbox>>) -> Self {
-        env::set_var("CC_IMAGE_WORK_DIR", KATA_CC_IMAGE_WORK_DIR);
-        Self {
-            sandbox,
-            attestation_agent_started: AtomicBool::new(false),
-            image_client: Arc::new(Mutex::new(ImageClient::default())),
-            container_count: Arc::new(AtomicU16::new(0)),
-        }
-    }
-
-    // pause image is packaged in rootfs for CC
-    fn unpack_pause_image(cid: &str) -> Result<()> {
-        let cc_pause_bundle = Path::new(KATA_CC_PAUSE_BUNDLE);
-        if !cc_pause_bundle.exists() {
-            return Err(anyhow!("Pause image not present in rootfs"));
-        }
-
-        info!(sl!(), "use guest pause image cid {:?}", cid);
-        let pause_bundle = Path::new(CONTAINER_BASE).join(cid);
-        let pause_rootfs = pause_bundle.join("rootfs");
-        let pause_config = pause_bundle.join(CONFIG_JSON);
-        let pause_binary = pause_rootfs.join("pause");
-        fs::create_dir_all(&pause_rootfs)?;
-        if !pause_config.exists() {
-            fs::copy(
-                cc_pause_bundle.join(CONFIG_JSON),
-                pause_bundle.join(CONFIG_JSON),
-            )?;
-        }
-        if !pause_binary.exists() {
-            fs::copy(cc_pause_bundle.join("rootfs").join("pause"), pause_binary)?;
-        }
-
-        Ok(())
-    }
-
-    // If we fail to start the AA, ocicrypt won't be able to unwrap keys
-    // and container decryption will fail.
-    fn init_attestation_agent() -> Result<()> {
-        let config_path = OCICRYPT_CONFIG_PATH;
-
-        // The image will need to be encrypted using a keyprovider
-        // that has the same name (at least according to the config).
-        let ocicrypt_config = serde_json::json!({
-            "key-providers": {
-                "attestation-agent":{
-                    "ttrpc":AA_KEYPROVIDER_URI
-                }
-            }
-        });
-
-        let mut config_file = fs::File::create(config_path)?;
-        config_file.write_all(ocicrypt_config.to_string().as_bytes())?;
-
-        // The Attestation Agent will run for the duration of the guest.
-        Command::new(AA_PATH)
-            .arg("--keyprovider_sock")
-            .arg(AA_KEYPROVIDER_URI)
-            .arg("--getresource_sock")
-            .arg(AA_GETRESOURCE_URI)
-            .spawn()?;
-        Ok(())
-    }
-
-    /// Determines the container id (cid) to use for a given request.
-    ///
-    /// If the request specifies a non-empty id, use it; otherwise derive it from the image path.
-    /// In either case, verify that the chosen id is valid.
-    fn cid_from_request(&self, req: &image::PullImageRequest) -> Result<String> {
-        let req_cid = req.container_id();
-        let cid = if !req_cid.is_empty() {
-            req_cid.to_string()
-        } else if let Some(last) = req.image().rsplit('/').next() {
-            // Support multiple containers with same image
-            let index = self.container_count.fetch_add(1, Ordering::Relaxed);
-
-            // ':' not valid for container id
-            format!("{}_{}", last.replace(':', "_"), index)
-        } else {
-            return Err(anyhow!("Invalid image name. {}", req.image()));
-        };
-        verify_cid(&cid)?;
-        Ok(cid)
-    }
-
-    async fn pull_image(&self, req: &image::PullImageRequest) -> Result<String> {
-        env::set_var("OCICRYPT_KEYPROVIDER_CONFIG", OCICRYPT_CONFIG_PATH);
-
-        let https_proxy = &AGENT_CONFIG.read().await.https_proxy;
-        if !https_proxy.is_empty() {
-            env::set_var("HTTPS_PROXY", https_proxy);
-        }
-
-        let no_proxy = &AGENT_CONFIG.read().await.no_proxy;
-        if !no_proxy.is_empty() {
-            env::set_var("NO_PROXY", no_proxy);
-        }
-
-        let cid = self.cid_from_request(req)?;
-        let image = req.image();
-        if cid.starts_with("pause") {
-            Self::unpack_pause_image(&cid)?;
-
-            let mut sandbox = self.sandbox.lock().await;
-            sandbox.images.insert(String::from(image), cid);
-            return Ok(image.to_owned());
-        }
-
-        let aa_kbc_params = &AGENT_CONFIG.read().await.aa_kbc_params;
-        if !aa_kbc_params.is_empty() {
-            match self.attestation_agent_started.compare_exchange_weak(
-                false,
-                true,
-                Ordering::SeqCst,
-                Ordering::SeqCst,
-            ) {
-                Ok(_) => Self::init_attestation_agent()?,
-                Err(_) => info!(sl!(), "Attestation Agent already running"),
-            }
-        }
-        // If the attestation-agent is being used, then enable the authenticated credentials support
-        info!(
-            sl!(),
-            "image_client.config.auth set to: {}",
-            !aa_kbc_params.is_empty()
-        );
-        self.image_client.lock().await.config.auth = !aa_kbc_params.is_empty();
-
-        // Read enable signature verification from the agent config and set it in the image_client
-        let enable_signature_verification =
-            &AGENT_CONFIG.read().await.enable_signature_verification;
-        info!(
-            sl!(),
-            "enable_signature_verification set to: {}", enable_signature_verification
-        );
-        self.image_client.lock().await.config.security_validate = *enable_signature_verification;
-
-        let source_creds = (!req.source_creds().is_empty()).then(|| req.source_creds());
-
-        let bundle_path = Path::new(CONTAINER_BASE).join(&cid);
-        fs::create_dir_all(&bundle_path)?;
-
-        let decrypt_config = format!("provider:attestation-agent:{}", aa_kbc_params);
-
-        info!(sl!(), "pull image {:?}, bundle path {:?}", cid, bundle_path);
-        // Image layers will store at KATA_CC_IMAGE_WORK_DIR, generated bundles
-        // with rootfs and config.json will store under CONTAINER_BASE/cid.
-        let res = self
-            .image_client
-            .lock()
-            .await
-            .pull_image(image, &bundle_path, &source_creds, &Some(&decrypt_config))
-            .await;
-
-        match res {
-            Ok(image) => {
-                info!(
-                    sl!(),
-                    "pull and unpack image {:?}, cid: {:?}, with image-rs succeed. ", image, cid
-                );
-            }
-            Err(e) => {
-                error!(
-                    sl!(),
-                    "pull and unpack image {:?}, cid: {:?}, with image-rs failed with {:?}. ",
-                    image,
-                    cid,
-                    e.to_string()
-                );
-                return Err(e);
-            }
-        };
-
-        let mut sandbox = self.sandbox.lock().await;
-        sandbox.images.insert(String::from(image), cid);
-        Ok(image.to_owned())
-    }
-}
-
-#[async_trait]
-impl protocols::image_ttrpc_async::Image for ImageService {
-    async fn pull_image(
-        &self,
-        _ctx: &ttrpc::r#async::TtrpcContext,
-        req: image::PullImageRequest,
-    ) -> ttrpc::Result<image::PullImageResponse> {
-        match self.pull_image(&req).await {
-            Ok(r) => {
-                let mut resp = image::PullImageResponse::new();
-                resp.image_ref = r;
-                return Ok(resp);
-            }
-            Err(e) => {
-                return Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()));
-            }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::ImageService;
-    use crate::sandbox::Sandbox;
-    use protocols::image;
-    use std::sync::Arc;
-    use tokio::sync::Mutex;
-
-    #[tokio::test]
-    async fn test_cid_from_request() {
-        struct Case {
-            cid: &'static str,
-            image: &'static str,
-            result: Option<&'static str>,
-        }
-
-        let cases = [
-            Case {
-                cid: "",
-                image: "",
-                result: None,
-            },
-            Case {
-                cid: "..",
-                image: "",
-                result: None,
-            },
-            Case {
-                cid: "",
-                image: "..",
-                result: None,
-            },
-            Case {
-                cid: "",
-                image: "abc/..",
-                result: None,
-            },
-            Case {
-                cid: "",
-                image: "abc/",
-                result: None,
-            },
-            Case {
-                cid: "",
-                image: "../abc",
-                result: Some("abc_4"),
-            },
-            Case {
-                cid: "",
-                image: "../9abc",
-                result: Some("9abc_5"),
-            },
-            Case {
-                cid: "some-string.1_2",
-                image: "",
-                result: Some("some-string.1_2"),
-            },
-            Case {
-                cid: "0some-string.1_2",
-                image: "",
-                result: Some("0some-string.1_2"),
-            },
-            Case {
-                cid: "a:b",
-                image: "",
-                result: None,
-            },
-            Case {
-                cid: "",
-                image: "prefix/a:b",
-                result: Some("a_b_6"),
-            },
-            Case {
-                cid: "",
-                image: "/a/b/c/d:e",
-                result: Some("d_e_7"),
-            },
-        ];
-
-        let logger = slog::Logger::root(slog::Discard, o!());
-        let s = Sandbox::new(&logger).unwrap();
-        let image_service = ImageService::new(Arc::new(Mutex::new(s)));
-        for case in &cases {
-            let mut req = image::PullImageRequest::new();
-            req.set_image(case.image.to_string());
-            req.set_container_id(case.cid.to_string());
-            let ret = image_service.cid_from_request(&req);
-            match (case.result, ret) {
-                (Some(expected), Ok(actual)) => assert_eq!(expected, actual),
-                (None, Err(_)) => (),
-                (None, Ok(r)) => panic!("Expected an error, got {}", r),
-                (Some(expected), Err(e)) => {
-                    panic!("Expected {} but got an error ({})", expected, e)
-                }
-            }
-        }
-    }
-}

src/agent/src/linux_abi.rs

@@ -33,7 +33,7 @@ pub fn create_pci_root_bus_path() -> String {
 
     // check if there is pci bus path for acpi
     acpi_sysfs_dir.push_str(&acpi_root_bus_path);
-    if let Ok(_) = fs::metadata(&acpi_sysfs_dir) {
+    if fs::metadata(&acpi_sysfs_dir).is_ok() {
         return acpi_root_bus_path;
     }
 
@@ -81,7 +81,8 @@ cfg_if! {
 // sysfs as directories in the subtree under /sys/devices/LNXSYSTM:00
 pub const ACPI_DEV_PATH: &str = "/devices/LNXSYSTM";
 
-pub const SYSFS_CPU_ONLINE_PATH: &str = "/sys/devices/system/cpu";
+pub const SYSFS_CPU_PATH: &str = "/sys/devices/system/cpu";
+pub const SYSFS_CPU_ONLINE_PATH: &str = "/sys/devices/system/cpu/online";
 
 pub const SYSFS_MEMORY_BLOCK_SIZE_PATH: &str = "/sys/devices/system/memory/block_size_bytes";
 pub const SYSFS_MEMORY_HOTPLUG_PROBE_PATH: &str = "/sys/devices/system/memory/probe";

src/agent/src/main.rs

@@ -65,12 +65,11 @@ use tokio::{
     io::AsyncWrite,
     sync::{
         watch::{channel, Receiver},
-        Mutex, RwLock,
+        Mutex,
     },
     task::JoinHandle,
 };
 
-mod image_rpc;
 mod rpc;
 mod tracer;
 
@@ -84,12 +83,11 @@ cfg_if! {
 const NAME: &str = "kata-agent";
 
 lazy_static! {
-    static ref AGENT_CONFIG: Arc<RwLock<AgentConfig>> = Arc::new(RwLock::new(
+    static ref AGENT_CONFIG: AgentConfig =
         // Note: We can't do AgentOpts.parse() here to send through the processed arguments to AgentConfig
         // clap::Parser::parse() greedily process all command line input including cargo test parameters,
         // so should only be used inside main.
-        AgentConfig::from_cmdline("/proc/cmdline", env::args().collect()).unwrap()
-    ));
+        AgentConfig::from_cmdline("/proc/cmdline", env::args().collect()).unwrap();
 }
 
 #[derive(Parser)]
@@ -182,13 +180,13 @@ async fn real_main() -> std::result::Result<(), Box<dyn std::error::Error>> {
 
         lazy_static::initialize(&AGENT_CONFIG);
 
-        init_agent_as_init(&logger, AGENT_CONFIG.read().await.unified_cgroup_hierarchy)?;
+        init_agent_as_init(&logger, AGENT_CONFIG.unified_cgroup_hierarchy)?;
         drop(logger_async_guard);
     } else {
         lazy_static::initialize(&AGENT_CONFIG);
     }
 
-    let config = AGENT_CONFIG.read().await;
+    let config = &AGENT_CONFIG;
     let log_vport = config.log_vport as u32;
 
     let log_handle = tokio::spawn(create_logger_task(rfd, log_vport, shutdown_rx.clone()));
@@ -201,7 +199,7 @@ async fn real_main() -> std::result::Result<(), Box<dyn std::error::Error>> {
     let (logger, logger_async_guard) =
         logging::create_logger(NAME, "agent", config.log_level, writer);
 
-    announce(&logger, &config);
+    announce(&logger, config);
 
     // This variable is required as it enables the global (and crucially static) logger,
     // which is required to satisfy the the lifetime constraints of the auto-generated gRPC code.
@@ -229,7 +227,7 @@ async fn real_main() -> std::result::Result<(), Box<dyn std::error::Error>> {
     let span_guard = root_span.enter();
 
     // Start the sandbox and wait for its ttRPC server to end
-    start_sandbox(&logger, &config, init_mode, &mut tasks, shutdown_rx.clone()).await?;
+    start_sandbox(&logger, config, init_mode, &mut tasks, shutdown_rx.clone()).await?;
 
     // Install a NOP logger for the remainder of the shutdown sequence
     // to ensure any log calls made by local crates using the scope logger

src/agent/src/metrics.rs

@@ -15,11 +15,9 @@ use tracing::instrument;
 const NAMESPACE_KATA_AGENT: &str = "kata_agent";
 const NAMESPACE_KATA_GUEST: &str = "kata_guest";
 
-// Convenience macro to obtain the scope logger
-macro_rules! sl {
-    () => {
-        slog_scope::logger().new(o!("subsystem" => "metrics"))
-    };
+// Convenience function to obtain the scope logger.
+fn sl() -> slog::Logger {
+    slog_scope::logger().new(o!("subsystem" => "metrics"))
 }
 
 lazy_static! {
@@ -139,7 +137,7 @@ fn update_agent_metrics() -> Result<()> {
         Ok(p) => p,
         Err(e) => {
             // FIXME: return Ok for all errors?
-            warn!(sl!(), "failed to create process instance: {:?}", e);
+            warn!(sl(), "failed to create process instance: {:?}", e);
 
             return Ok(());
         }
@@ -160,7 +158,7 @@ fn update_agent_metrics() -> Result<()> {
     // io
     match me.io() {
         Err(err) => {
-            info!(sl!(), "failed to get process io stat: {:?}", err);
+            info!(sl(), "failed to get process io stat: {:?}", err);
         }
         Ok(io) => {
             set_gauge_vec_proc_io(&AGENT_IO_STAT, &io);
@@ -169,7 +167,7 @@ fn update_agent_metrics() -> Result<()> {
 
     match me.stat() {
         Err(err) => {
-            info!(sl!(), "failed to get process stat: {:?}", err);
+            info!(sl(), "failed to get process stat: {:?}", err);
         }
         Ok(stat) => {
             set_gauge_vec_proc_stat(&AGENT_PROC_STAT, &stat);
@@ -177,7 +175,7 @@ fn update_agent_metrics() -> Result<()> {
     }
 
     match me.status() {
-        Err(err) => error!(sl!(), "failed to get process status: {:?}", err),
+        Err(err) => error!(sl(), "failed to get process status: {:?}", err),
         Ok(status) => set_gauge_vec_proc_status(&AGENT_PROC_STATUS, &status),
     }
 
@@ -189,7 +187,7 @@ fn update_guest_metrics() {
     // try get load and task info
     match procfs::LoadAverage::new() {
         Err(err) => {
-            info!(sl!(), "failed to get guest LoadAverage: {:?}", err);
+            info!(sl(), "failed to get guest LoadAverage: {:?}", err);
         }
         Ok(load) => {
             GUEST_LOAD
@@ -209,7 +207,7 @@ fn update_guest_metrics() {
     // try to get disk stats
     match procfs::diskstats() {
         Err(err) => {
-            info!(sl!(), "failed to get guest diskstats: {:?}", err);
+            info!(sl(), "failed to get guest diskstats: {:?}", err);
         }
         Ok(diskstats) => {
             for diskstat in diskstats {
@@ -221,7 +219,7 @@ fn update_guest_metrics() {
     // try to get vm stats
     match procfs::vmstat() {
         Err(err) => {
-            info!(sl!(), "failed to get guest vmstat: {:?}", err);
+            info!(sl(), "failed to get guest vmstat: {:?}", err);
         }
         Ok(vmstat) => {
             for (k, v) in vmstat {
@@ -233,7 +231,7 @@ fn update_guest_metrics() {
     // cpu stat
     match procfs::KernelStats::new() {
         Err(err) => {
-            info!(sl!(), "failed to get guest KernelStats: {:?}", err);
+            info!(sl(), "failed to get guest KernelStats: {:?}", err);
         }
         Ok(kernel_stats) => {
             set_gauge_vec_cpu_time(&GUEST_CPU_TIME, "total", &kernel_stats.total);
@@ -246,7 +244,7 @@ fn update_guest_metrics() {
     // try to get net device stats
     match procfs::net::dev_status() {
         Err(err) => {
-            info!(sl!(), "failed to get guest net::dev_status: {:?}", err);
+            info!(sl(), "failed to get guest net::dev_status: {:?}", err);
         }
         Ok(devs) => {
             // netdev: map[string]procfs::net::DeviceStatus
@@ -259,7 +257,7 @@ fn update_guest_metrics() {
     // get statistics about memory from /proc/meminfo
     match procfs::Meminfo::new() {
         Err(err) => {
-            info!(sl!(), "failed to get guest Meminfo: {:?}", err);
+            info!(sl(), "failed to get guest Meminfo: {:?}", err);
         }
         Ok(meminfo) => {
             set_gauge_vec_meminfo(&GUEST_MEMINFO, &meminfo);

src/agent/src/mount.rs

@@ -36,6 +36,7 @@ use crate::Sandbox;
 use crate::{ccw, device::get_virtio_blk_ccw_device_name};
 use anyhow::{anyhow, Context, Result};
 use slog::Logger;
+
 use tracing::instrument;
 
 pub const TYPE_ROOTFS: &str = "rootfs";
@@ -145,6 +146,11 @@ pub const STORAGE_HANDLER_LIST: &[&str] = &[
     DRIVER_WATCHABLE_BIND_TYPE,
 ];
 
+#[instrument]
+pub fn get_mounts() -> Result<String, std::io::Error> {
+    fs::read_to_string("/proc/mounts")
+}
+
 #[instrument]
 pub fn baremount(
     source: &Path,
@@ -168,6 +174,31 @@ pub fn baremount(
         return Err(anyhow!("need mount FS type"));
     }
 
+    let destination_str = destination.to_string_lossy();
+    let mounts = get_mounts().unwrap_or_else(|_| String::new());
+    let already_mounted = mounts
+        .lines()
+        .map(|line| line.split_whitespace().collect::<Vec<&str>>())
+        .filter(|parts| parts.len() >= 3) // ensure we have at least [source}, destination, and fs_type
+        .any(|parts| {
+            // Check if source, destination and fs_type match any entry in /proc/mounts
+            // minimal check is for destination an fstype since source can have different names like:
+            // udev /dev devtmpfs
+            //  dev /dev devtmpfs
+            // depending on which entity is mounting the dev/fs/pseudo-fs
+            parts[1] == destination_str && parts[2] == fs_type
+        });
+
+    if already_mounted {
+        slog_info!(
+            logger,
+            "{:?} is already mounted at {:?}",
+            source,
+            destination
+        );
+        return Ok(());
+    }
+
     info!(
         logger,
         "baremount source={:?}, dest={:?}, fs_type={:?}, options={:?}, flags={:?}",
@@ -725,6 +756,14 @@ pub fn recursive_ownership_change(
         mask |= EXEC_MASK;
         mask |= MODE_SETGID;
     }
+
+    // We do not want to change the permission of the underlying file
+    // using symlink. Hence we skip symlinks from recursive ownership
+    // and permission changes.
+    if path.is_symlink() {
+        return Ok(());
+    }
+
     nix::unistd::chown(path, uid, gid)?;
 
     if gid.is_some() {
@@ -1102,6 +1141,7 @@ fn parse_options(option_list: Vec<String>) -> HashMap<String, String> {
 mod tests {
     use super::*;
     use protocols::agent::FSGroup;
+    use slog::Drain;
     use std::fs::File;
     use std::fs::OpenOptions;
     use std::io::Write;
@@ -1112,6 +1152,31 @@ mod tests {
         skip_if_not_root, skip_loop_by_user, skip_loop_if_not_root, skip_loop_if_root,
     };
 
+    #[test]
+    fn test_already_baremounted() {
+        let plain = slog_term::PlainSyncDecorator::new(std::io::stdout());
+        let logger = Logger::root(slog_term::FullFormat::new(plain).build().fuse(), o!());
+
+        let test_cases = [
+            ("dev", "/dev", "devtmpfs"),
+            ("udev", "/dev", "devtmpfs"),
+            ("proc", "/proc", "proc"),
+            ("sysfs", "/sys", "sysfs"),
+        ];
+
+        for &(source, destination, fs_type) in &test_cases {
+            let source = Path::new(source);
+            let destination = Path::new(destination);
+            let flags = MsFlags::MS_RDONLY;
+            let options = "mode=755";
+            println!(
+                "testing if already mounted baremount({:?} {:?} {:?})",
+                source, destination, fs_type
+            );
+            assert!(baremount(source, destination, fs_type, flags, options, &logger).is_ok());
+        }
+    }
+
     #[test]
     fn test_mount() {
         #[derive(Debug)]

src/agent/src/netlink.rs

@@ -89,6 +89,27 @@ impl Handle {
                 .await?;
         }
 
+        // we need to update the link's interface name, thus we should rename the existed link whose name
+        // is the same with the link's request name, otherwise, it would update the link failed with the
+        // name conflicted.
+        let mut new_link = None;
+        if link.name() != iface.name {
+            if let Ok(link) = self.find_link(LinkFilter::Name(iface.name.as_str())).await {
+                // update the existing interface name with a temporary name, otherwise
+                // it would failed to udpate this interface with an existing name.
+                let mut request = self.handle.link().set(link.index());
+                request.message_mut().header = link.header.clone();
+
+                request
+                    .name(format!("{}_temp", link.name()))
+                    .up()
+                    .execute()
+                    .await?;
+
+                new_link = Some(link);
+            }
+        }
+
         // Update link
         let mut request = self.handle.link().set(link.index());
         request.message_mut().header = link.header.clone();
@@ -101,6 +122,14 @@ impl Handle {
             .execute()
             .await?;
 
+        // swap the updated iface's name.
+        if let Some(nlink) = new_link {
+            let mut request = self.handle.link().set(nlink.index());
+            request.message_mut().header = nlink.header.clone();
+
+            request.name(link.name()).up().execute().await?;
+        }
+
         Ok(())
     }
 

src/agent/src/sandbox.rs

@@ -12,6 +12,7 @@ use crate::pci;
 use crate::uevent::{Uevent, UeventMatcher};
 use crate::watcher::BindWatcher;
 use anyhow::{anyhow, Context, Result};
+use kata_types::cpu::CpuSet;
 use libc::pid_t;
 use oci::{Hook, Hooks};
 use protocols::agent::OnlineCPUMemRequest;
@@ -25,6 +26,7 @@ use std::collections::HashMap;
 use std::fs;
 use std::os::unix::fs::PermissionsExt;
 use std::path::Path;
+use std::str::FromStr;
 use std::sync::Arc;
 use std::{thread, time};
 use tokio::sync::mpsc::{channel, Receiver, Sender};
@@ -60,7 +62,6 @@ pub struct Sandbox {
     pub event_tx: Option<Sender<String>>,
     pub bind_watcher: BindWatcher,
     pub pcimap: HashMap<pci::Address, pci::Address>,
-    pub images: HashMap<String, String>,
 }
 
 impl Sandbox {
@@ -94,7 +95,6 @@ impl Sandbox {
             event_tx: Some(tx),
             bind_watcher: BindWatcher::new(),
             pcimap: HashMap::new(),
-            images: HashMap::new(),
         })
     }
 
@@ -265,12 +265,12 @@ impl Sandbox {
     pub fn online_cpu_memory(&self, req: &OnlineCPUMemRequest) -> Result<()> {
         if req.nb_cpus > 0 {
             // online cpus
-            online_cpus(&self.logger, req.nb_cpus as i32)?;
+            online_cpus(&self.logger, req.nb_cpus as i32).context("online cpus")?;
         }
 
         if !req.cpu_only {
             // online memory
-            online_memory(&self.logger)?;
+            online_memory(&self.logger).context("online memory")?;
         }
 
         if req.nb_cpus == 0 {
@@ -433,27 +433,37 @@ fn online_resources(logger: &Logger, path: &str, pattern: &str, num: i32) -> Res
 }
 
 // max wait for all CPUs to online will use 50 * 100 = 5 seconds.
-const ONLINE_CPUMEM_WATI_MILLIS: u64 = 50;
-const ONLINE_CPUMEM_MAX_RETRIES: u32 = 100;
+const ONLINE_CPUMEM_WAIT_MILLIS: u64 = 50;
+const ONLINE_CPUMEM_MAX_RETRIES: i32 = 100;
 
 #[instrument]
 fn online_cpus(logger: &Logger, num: i32) -> Result<i32> {
-    let mut onlined_count: i32 = 0;
+    let mut onlined_cpu_count = onlined_cpus().context("onlined cpu count")?;
+    // for some vmms, like dragonball, they will online cpus for us
+    // so check first whether agent need to do the online operation
+    if onlined_cpu_count >= num {
+        return Ok(num);
+    }
 
     for i in 0..ONLINE_CPUMEM_MAX_RETRIES {
-        let r = online_resources(
+        // online num resources
+        online_resources(
             logger,
-            SYSFS_CPU_ONLINE_PATH,
+            SYSFS_CPU_PATH,
             r"cpu[0-9]+",
-            num - onlined_count,
-        );
+            num - onlined_cpu_count,
+        )
+        .context("online cpu resource")?;
 
-        onlined_count += r?;
-        if onlined_count == num {
-            info!(logger, "online {} CPU(s) after {} retries", num, i);
+        onlined_cpu_count = onlined_cpus().context("onlined cpu count")?;
+        if onlined_cpu_count >= num {
+            info!(
+                logger,
+                "Currently {} onlined CPU(s) after {} retries", onlined_cpu_count, i
+            );
             return Ok(num);
         }
-        thread::sleep(time::Duration::from_millis(ONLINE_CPUMEM_WATI_MILLIS));
+        thread::sleep(time::Duration::from_millis(ONLINE_CPUMEM_WAIT_MILLIS));
     }
 
     Err(anyhow!(
@@ -465,10 +475,18 @@ fn online_cpus(logger: &Logger, num: i32) -> Result<i32> {
 
 #[instrument]
 fn online_memory(logger: &Logger) -> Result<()> {
-    online_resources(logger, SYSFS_MEMORY_ONLINE_PATH, r"memory[0-9]+", -1)?;
+    online_resources(logger, SYSFS_MEMORY_ONLINE_PATH, r"memory[0-9]+", -1)
+        .context("online memory resource")?;
     Ok(())
 }
 
+fn onlined_cpus() -> Result<i32> {
+    let content =
+        fs::read_to_string(SYSFS_CPU_ONLINE_PATH).context("read sysfs cpu online file")?;
+    let online_cpu_set = CpuSet::from_str(content.trim())?;
+    Ok(online_cpu_set.len() as i32)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/agent/src/signal.rs

@@ -57,7 +57,7 @@ async fn handle_sigchild(logger: Logger, sandbox: Arc<Mutex<Sandbox>>) -> Result
                 continue;
             }
 
-            let mut p = process.unwrap();
+            let p = process.unwrap();
 
             let ret: i32 = match wait_status {
                 WaitStatus::Exited(_, c) => c,

src/agent/src/tracer.rs

@@ -69,7 +69,7 @@ macro_rules! trace_rpc_call {
             propagator.extract(&extract_carrier_from_ttrpc($ctx))
         });
 
-        info!(sl!(), "rpc call from shim to agent: {:?}", $name);
+        info!(sl(), "rpc call from shim to agent: {:?}", $name);
 
         // generate tracing span
         let rpc_span = span!(tracing::Level::INFO, $name, "mod"="rpc.rs", req=?$req);

src/agent/src/uevent.rs

@@ -19,11 +19,9 @@ use tokio::sync::watch::Receiver;
 use tokio::sync::Mutex;
 use tracing::instrument;
 
-// Convenience macro to obtain the scope logger
-macro_rules! sl {
-    () => {
-        slog_scope::logger().new(o!("subsystem" => "uevent"))
-    };
+// Convenience function to obtain the scope logger.
+fn sl() -> slog::Logger {
+    slog_scope::logger().new(o!("subsystem" => "uevent"))
 }
 
 #[derive(Debug, Default, Clone, PartialEq, Eq)]
@@ -120,11 +118,11 @@ pub async fn wait_for_uevent(
 ) -> Result<Uevent> {
     let logprefix = format!("Waiting for {:?}", &matcher);
 
-    info!(sl!(), "{}", logprefix);
+    info!(sl(), "{}", logprefix);
     let mut sb = sandbox.lock().await;
     for uev in sb.uevent_map.values() {
         if matcher.is_match(uev) {
-            info!(sl!(), "{}: found {:?} in uevent map", logprefix, &uev);
+            info!(sl(), "{}: found {:?} in uevent map", logprefix, &uev);
             return Ok(uev.clone());
         }
     }
@@ -139,9 +137,9 @@ pub async fn wait_for_uevent(
     sb.uevent_watchers.push(Some((Box::new(matcher), tx)));
     drop(sb); // unlock
 
-    info!(sl!(), "{}: waiting on channel", logprefix);
+    info!(sl(), "{}: waiting on channel", logprefix);
 
-    let hotplug_timeout = AGENT_CONFIG.read().await.hotplug_timeout;
+    let hotplug_timeout = AGENT_CONFIG.hotplug_timeout;
 
     let uev = match tokio::time::timeout(hotplug_timeout, rx).await {
         Ok(v) => v?,
@@ -157,7 +155,7 @@ pub async fn wait_for_uevent(
         }
     };
 
-    info!(sl!(), "{}: found {:?} on channel", logprefix, &uev);
+    info!(sl(), "{}: found {:?} on channel", logprefix, &uev);
     Ok(uev)
 }
 

src/dragonball/Cargo.toml

@@ -10,18 +10,19 @@ license = "Apache-2.0"
 edition = "2018"
 
 [dependencies]
+anyhow = "1.0.32"
 arc-swap = "1.5.0"
 bytes = "1.1.0"
-dbs-address-space = "0.2.0"
-dbs-allocator = "0.1.0"
-dbs-arch = "0.2.0"
-dbs-boot = "0.3.0"
-dbs-device = "0.2.0"
-dbs-interrupt = { version = "0.2.0", features = ["kvm-irq"] }
-dbs-legacy-devices = "0.1.0"
-dbs-upcall = { version = "0.2.0", optional = true }
-dbs-utils = "0.2.0"
-dbs-virtio-devices = { version = "0.2.0", optional = true, features = ["virtio-mmio"] }
+dbs-address-space =  { path = "./src/dbs_address_space" }
+dbs-allocator = { path = "./src/dbs_allocator" }
+dbs-arch = { path = "./src/dbs_arch" }
+dbs-boot = { path = "./src/dbs_boot" }
+dbs-device = { path = "./src/dbs_device" }
+dbs-interrupt = { path = "./src/dbs_interrupt", features = ["kvm-irq"] }
+dbs-legacy-devices = { path = "./src/dbs_legacy_devices" }
+dbs-upcall = { path = "./src/dbs_upcall" , optional = true }
+dbs-utils = { path = "./src/dbs_utils" }
+dbs-virtio-devices = { path = "./src/dbs_virtio_devices", optional = true, features = ["virtio-mmio"] }
 kvm-bindings = "0.6.0"
 kvm-ioctls = "0.12.0"
 lazy_static = "1.2"
@@ -29,6 +30,8 @@ libc = "0.2.39"
 linux-loader = "0.6.0"
 log = "0.4.14"
 nix = "0.24.2"
+procfs = "0.12.0"
+prometheus = { version = "0.13.0", features = ["process"] }
 seccompiler = "0.2.0"
 serde = "1.0.27"
 serde_derive = "1.0.27"
@@ -42,8 +45,8 @@ vm-memory = { version = "0.9.0", features = ["backend-mmap"] }
 crossbeam-channel = "0.5.6"
 
 [dev-dependencies]
-slog-term = "2.9.0"
 slog-async = "2.7.0"
+slog-term = "2.9.0"
 test-utils = { path = "../libs/test-utils" }
 
 [features]
@@ -55,3 +58,5 @@ virtio-blk = ["dbs-virtio-devices/virtio-blk", "virtio-queue"]
 virtio-net = ["dbs-virtio-devices/virtio-net", "virtio-queue"]
 # virtio-fs only work on atomic-guest-memory
 virtio-fs = ["dbs-virtio-devices/virtio-fs", "virtio-queue", "atomic-guest-memory"]
+virtio-mem = ["dbs-virtio-devices/virtio-mem", "virtio-queue", "atomic-guest-memory"]
+virtio-balloon = ["dbs-virtio-devices/virtio-balloon", "virtio-queue"]

src/dragonball/Makefile

@@ -39,12 +39,15 @@ clean:
 
 test:
 ifdef SUPPORT_VIRTUALIZATION
-	cargo test --all-features --target $(TRIPLE) -- --nocapture
+	RUST_BACKTRACE=1 cargo test --all-features --target $(TRIPLE) -- --nocapture --test-threads=1
 else
 	@echo "INFO: skip testing dragonball, it need virtualization support."
 	exit 0
 endif
 
+coverage:
+	RUST_BACKTRACE=1 cargo llvm-cov --all-features --target $(TRIPLE) -- --nocapture --test-threads=1
+
 endif # ifeq ($(ARCH), s390x)
 
 .DEFAULT_GOAL := default

src/dragonball/README.md

@@ -16,10 +16,22 @@ and configuration process.
 
 # Documentation
 
-Device: [Device Document](docs/device.md)
-vCPU: [vCPU Document](docs/vcpu.md)
-API: [API Document](docs/api.md)
-`Upcall`: [`Upcall` Document](docs/upcall.md)
+- Device: [Device Document](docs/device.md)
+- vCPU: [vCPU Document](docs/vcpu.md)
+- API: [API Document](docs/api.md)
+- `Upcall`: [`Upcall` Document](docs/upcall.md)
+- `dbs_acpi`: [`dbs_acpi` Document](src/dbs_acpi/README.md)
+- `dbs_address_space`: [`dbs_address_space` Document](src/dbs_address_space/README.md)
+- `dbs_allocator`: [`dbs_allocator` Document](src/dbs_allocator/README.md)
+- `dbs_arch`: [`dbs_arch` Document](src/dbs_arch/README.md)
+- `dbs_boot`: [`dbs_boot` Document](src/dbs_boot/README.md)
+- `dbs_device`: [`dbs_device` Document](src/dbs_device/README.md)
+- `dbs_interrupt`: [`dbs_interrput` Document](src/dbs_interrupt/README.md)
+- `dbs_legacy_devices`: [`dbs_legacy_devices` Document](src/dbs_legacy_devices/README.md)
+- `dbs_tdx`: [`dbs_tdx` Document](src/dbs_tdx/README.md)
+- `dbs_upcall`: [`dbs_upcall` Document](src/dbs_upcall/README.md)
+- `dbs_utils`: [`dbs_utils` Document](src/dbs_utils/README.md)
+- `dbs_virtio_devices`: [`dbs_virtio_devices` Document](src/dbs_virtio_devices/README.md)
 
 Currently, the documents are still actively adding.
 You could see the [official documentation](docs/) page for more details.

src/dragonball/src/api/v1/instance_info.rs

@@ -5,15 +5,6 @@
 
 use serde_derive::{Deserialize, Serialize};
 
-/// This struct represents the strongly typed equivalent of the json body
-/// from confidential container related requests.
-#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Serialize)]
-#[serde(deny_unknown_fields)]
-pub enum ConfidentialVmType {
-    /// Intel Trusted Domain
-    TDX = 2,
-}
-
 /// The microvm state.
 ///
 /// When Dragonball starts, the instance state is Uninitialized. Once start_microvm method is
@@ -67,12 +58,10 @@ pub struct InstanceInfo {
     pub tids: Vec<(u8, u32)>,
     /// Last instance downtime
     pub last_instance_downtime: u64,
-    /// confidential vm type
-    pub confidential_vm_type: Option<ConfidentialVmType>,
 }
 
 impl InstanceInfo {
-    /// create instance info object with given id, version, platform type and confidential vm type.
+    /// create instance info object with given id, version, and platform type
     pub fn new(id: String, vmm_version: String) -> Self {
         InstanceInfo {
             id,
@@ -83,14 +72,8 @@ impl InstanceInfo {
             async_state: AsyncState::Uninitialized,
             tids: Vec::new(),
             last_instance_downtime: 0,
-            confidential_vm_type: None,
         }
     }
-
-    /// return true if VM confidential type is TDX
-    pub fn is_tdx_enabled(&self) -> bool {
-        matches!(self.confidential_vm_type, Some(ConfidentialVmType::TDX))
-    }
 }
 
 impl Default for InstanceInfo {
@@ -104,7 +87,6 @@ impl Default for InstanceInfo {
             async_state: AsyncState::Uninitialized,
             tids: Vec::new(),
             last_instance_downtime: 0,
-            confidential_vm_type: None,
         }
     }
 }