Merge pull request #6382 from jepio/jepio/deps-v0.4.0-tag

CCv0: set dependencies to v0.4.0 tag

.github/workflows/add-backport-label.yaml

@@ -62,15 +62,15 @@ jobs:
           has_backport_needed_label=${{ contains(github.event.pull_request.labels.*.name, 'needs-backport') }}
           has_no_backport_needed_label=${{ contains(github.event.pull_request.labels.*.name, 'no-backport-needed') }}
 
-          echo "::set-output name=add_backport_label::false"
+          echo "add_backport_label=false" >> $GITHUB_OUTPUT
           if [ $has_backport_needed_label  = true ] || [ $has_bug  = true ]; then
             if [[ $has_no_backport_needed_label = false ]]; then
-              echo "::set-output name=add_backport_label::true"
+              echo "add_backport_label=true" >> $GITHUB_OUTPUT
             fi
           fi
 
           # Do not spam comment, only if auto-backport label is going to be newly added.
-          echo "::set-output name=auto_backport_added::$CONTAINS_AUTO_BACKPORT"
+          echo "auto_backport_added=$CONTAINS_AUTO_BACKPORT" >> $GITHUB_OUTPUT
 
       - name: Add comment
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && steps.add_label.outputs.add_backport_label == 'true' && steps.add_label.outputs.auto_backport_added == 'false' }}
@@ -97,4 +97,4 @@ jobs:
         uses: andymckay/labeler@e6c4322d0397f3240f0e7e30a33b5c5df2d39e90
         with:
           add-labels: "auto-backport"
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/cargo-deny-runner.yaml

@@ -1,5 +1,12 @@
 name: Cargo Crates Check Runner
-on: [pull_request]
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+    paths-ignore: [ '**.md', '**.png', '**.jpg', '**.jpeg', '**.svg', '/docs/**' ]
 jobs:
   cargo-deny-runner:
     runs-on: ubuntu-latest

.github/workflows/cc-payload-after-push-amd64.yaml

@@ -0,0 +1,159 @@
+name: CI | Publish CC runtime payload for amd64
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-asset:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        asset:
+          - cc-cloud-hypervisor
+          - cc-kernel
+          - cc-qemu
+          - cc-rootfs-image
+          - cc-virtiofsd
+          - cc-sev-kernel
+          - cc-sev-ovmf
+          - cc-sev-rootfs-initrd
+          - cc-tdx-kernel
+          - cc-tdx-rootfs-image
+          - cc-tdx-qemu
+          - cc-tdx-td-shim
+          - cc-tdx-tdvf
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: yes
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+      - name: store-artifact root_hash_tdx.txt 
+        uses: actions/upload-artifact@v3
+        with:
+          name: root_hash_tdx.txt
+          path: tools/osbuilder/root_hash_tdx.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+      - name: store-artifact root_hash_vanilla.txt 
+        uses: actions/upload-artifact@v3
+        with:
+          name: root_hash_vanilla.txt
+          path: tools/osbuilder/root_hash_vanilla.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+  build-asset-cc-shim-v2:
+    runs-on: ubuntu-latest
+    needs: build-asset
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+
+      - name: Get root_hash_tdx.txt
+        uses: actions/download-artifact@v3
+        with:
+          name: root_hash_tdx.txt
+          path: tools/osbuilder/
+
+      - name: Get root_hash_vanilla.txt
+        uses: actions/download-artifact@v3
+        with:
+          name: root_hash_vanilla.txt
+          path: tools/osbuilder/
+
+      - name: Build cc-shim-v2
+        run: |
+          make cc-shim-v2-tarball
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          PUSH_TO_REGISTRY: yes
+
+      - name: store-artifact cc-shim-v2
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-cc-shim-v2.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: [build-asset, build-asset-cc-shim-v2]
+    steps:
+      - uses: actions/checkout@v3
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  kata-payload:
+    needs: create-kata-tarball
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to Confidential Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz "quay.io/confidential-containers/runtime-payload-ci" \
+          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/cc-payload-after-push-s390x.yaml

@@ -0,0 +1,153 @@
+name: CI | Publish CC runtime payload for s390x
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-asset:
+    runs-on: s390x
+    strategy:
+      matrix:
+        asset:
+          - cc-kernel
+          - cc-qemu
+          - cc-rootfs-image
+          - cc-virtiofsd
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+          sudo chown -R $(id -u):$(id -g) "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: yes
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+      - name: store-artifact root_hash_vanilla.txt 
+        uses: actions/upload-artifact@v3
+        with:
+          name: root_hash_vanilla.txt-s390x
+          path: tools/osbuilder/root_hash_vanilla.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+  build-asset-cc-shim-v2:
+    runs-on: s390x
+    needs: build-asset
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+
+      - name: Get root_hash_vanilla.txt
+        uses: actions/download-artifact@v3
+        with:
+          name: root_hash_vanilla.txt-s390x
+          path: tools/osbuilder/
+
+      - name: Build cc-shim-v2
+        run: |
+          make cc-shim-v2-tarball
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          PUSH_TO_REGISTRY: yes
+
+      - name: store-artifact cc-shim-v2
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-build/kata-static-cc-shim-v2.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: s390x
+    needs: [build-asset, build-asset-cc-shim-v2]
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  kata-payload:
+    needs: create-kata-tarball
+    runs-on: s390x
+    steps:
+      - name: Login to Confidential Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz "quay.io/confidential-containers/runtime-payload-ci" \
+          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/cc-payload-after-push.yaml

@@ -0,0 +1,39 @@
+name: CI | Publish Kata Containers payload for Confidential Containers
+on:
+  push:
+    branches:
+      - CCv0
+
+jobs:
+  build-assets-amd64:
+    uses: ./.github/workflows/cc-payload-after-push-amd64.yaml
+    with:
+      target-arch: amd64
+    secrets: inherit
+
+  build-assets-s390x:
+    uses: ./.github/workflows/cc-payload-after-push-s390x.yaml
+    with:
+      target-arch: s390x
+    secrets: inherit
+
+  publish:
+    runs-on: ubuntu-latest
+    needs: [build-assets-amd64, build-assets-s390x]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Login to Confidential Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Push multi-arch manifest
+        run: |
+          docker manifest create quay.io/confidential-containers/runtime-payload-ci:kata-containers-latest \
+          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-amd64 \
+          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-s390x
+          docker manifest push quay.io/confidential-containers/runtime-payload-ci:kata-containers-latest

.github/workflows/cc-payload-amd64.yaml

@@ -0,0 +1,142 @@
+name: Publish Kata Containers payload for Confidential Containers (amd64)
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-asset:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        asset:
+          - cc-cloud-hypervisor
+          - cc-kernel
+          - cc-qemu
+          - cc-rootfs-image
+          - cc-virtiofsd
+          - cc-sev-kernel
+          - cc-sev-ovmf
+          - cc-sev-rootfs-initrd
+          - cc-tdx-kernel
+          - cc-tdx-rootfs-image
+          - cc-tdx-qemu
+          - cc-tdx-td-shim
+          - cc-tdx-tdvf
+    steps:
+      - uses: actions/checkout@v3
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+      - name: store-artifact root_hash_tdx.txt 
+        uses: actions/upload-artifact@v3
+        with:
+          name: root_hash_tdx.txt
+          path: tools/osbuilder/root_hash_tdx.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+      - name: store-artifact root_hash_vanilla.txt 
+        uses: actions/upload-artifact@v3
+        with:
+          name: root_hash_vanilla.txt
+          path: tools/osbuilder/root_hash_vanilla.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+  build-asset-cc-shim-v2:
+    runs-on: ubuntu-latest
+    needs: build-asset
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Get root_hash_tdx.txt
+        uses: actions/download-artifact@v3
+        with:
+          name: root_hash_tdx.txt
+          path: tools/osbuilder/
+
+      - name: Get root_hash_vanilla.txt
+        uses: actions/download-artifact@v3
+        with:
+          name: root_hash_vanilla.txt
+          path: tools/osbuilder/
+
+      - name: Build cc-shim-v2
+        run: |
+          make cc-shim-v2-tarball
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+
+      - name: store-artifact cc-shim-v2
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-cc-shim-v2.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: [build-asset, build-asset-cc-shim-v2]
+    steps:
+      - uses: actions/checkout@v3
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  kata-payload:
+    needs: create-kata-tarball
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz \
+          "quay.io/confidential-containers/runtime-payload" \
+          "kata-containers-${{ inputs.target-arch }}"
+

.github/workflows/cc-payload-s390x.yaml

@@ -0,0 +1,134 @@
+name: Publish Kata Containers payload for Confidential Containers (s390x)
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-asset:
+    runs-on: s390x
+    strategy:
+      matrix:
+        asset:
+          - cc-kernel
+          - cc-qemu
+          - cc-rootfs-image
+          - cc-virtiofsd
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+      - name: store-artifact root_hash_vanilla.txt 
+        uses: actions/upload-artifact@v3
+        with:
+          name: root_hash_vanilla.txt-s390x
+          path: tools/osbuilder/root_hash_vanilla.txt
+          retention-days: 1
+          if-no-files-found: ignore
+
+  build-asset-cc-shim-v2:
+    runs-on: s390x
+    needs: build-asset
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+
+      - name: Get root_hash_vanilla.txt
+        uses: actions/download-artifact@v3
+        with:
+          name: root_hash_vanilla.txt-s390x
+          path: tools/osbuilder/
+
+      - name: Build cc-shim-v2
+        run: |
+          make cc-shim-v2-tarball
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+
+      - name: store-artifact cc-shim-v2
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-build/kata-static-cc-shim-v2.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: s390x
+    needs: [build-asset, build-asset-cc-shim-v2]
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  kata-payload:
+    needs: create-kata-tarball
+    runs-on: s390x
+    steps:
+      - name: Login to quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz \
+          "quay.io/confidential-containers/runtime-payload" \
+          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/cc-payload.yaml

@@ -0,0 +1,39 @@
+name: Publish Kata Containers payload for Confidential Containers
+on:
+  push:
+    tags:
+      - 'CC\-[0-9]+.[0-9]+.[0-9]+'
+
+jobs:
+  build-assets-amd64:
+    uses: ./.github/workflows/cc-payload-amd64.yaml
+    with:
+      target-arch: amd64
+    secrets: inherit
+
+  build-assets-s390x:
+    uses: ./.github/workflows/cc-payload-s390x.yaml
+    with:
+      target-arch: s390x
+    secrets: inherit
+
+  publish:
+    runs-on: ubuntu-latest
+    needs: [build-assets-amd64, build-assets-s390x]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Login to Confidential Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Push multi-arch manifest
+        run: |
+          docker manifest create quay.io/confidential-containers/runtime-payload:kata-containers-latest \
+          --amend quay.io/confidential-containers/runtime-payload:kata-containers-amd64 \
+          --amend quay.io/confidential-containers/runtime-payload:kata-containers-s390x
+          docker manifest push quay.io/confidential-containers/runtime-payload:kata-containers-latest

.github/workflows/commit-message-check.yaml

@@ -47,7 +47,7 @@ jobs:
       uses: tim-actions/commit-message-checker-with-regex@v0.3.1
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
-        pattern: '^.{0,75}(\n.*)*$'
+        pattern: '^.{0,75}(\n.*)*$|^Merge pull request (?:kata-containers)?#[\d]+ from.*'
         error: 'Subject too long (max 75)'
         post_error: ${{ env.error_msg }}
 
@@ -95,6 +95,6 @@ jobs:
       uses: tim-actions/commit-message-checker-with-regex@v0.3.1
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
-        pattern: '^[\s\t]*[^:\s\t]+[\s\t]*:'
+        pattern: '^[\s\t]*[^:\s\t]+[\s\t]*:|^Merge pull request (?:kata-containers)?#[\d]+ from.*'
         error: 'Failed to find subsystem in subject'
         post_error: ${{ env.error_msg }}

.github/workflows/darwin-tests.yaml

@@ -5,7 +5,7 @@ on:
       - edited
       - reopened
       - synchronize
-
+    paths-ignore: [ '**.md', '**.png', '**.jpg', '**.jpeg', '**.svg', '/docs/**' ]
 name: Darwin tests
 jobs:
   test:
@@ -14,7 +14,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.19.2
+        go-version: 1.19.3
     - name: Checkout code
       uses: actions/checkout@v2
     - name: Build utils

.github/workflows/deploy-ccv0-demo.yaml

@@ -0,0 +1,124 @@
+on:
+  issue_comment:
+    types: [created, edited]
+
+name: deploy-ccv0-demo
+
+jobs:
+  check-comment-and-membership:
+    runs-on: ubuntu-latest
+    if: |
+      github.event.issue.pull_request
+      && github.event_name == 'issue_comment'
+      && github.event.action == 'created'
+      && startsWith(github.event.comment.body, '/deploy-ccv0-demo')
+    steps:
+      - name: Check membership
+        uses: kata-containers/is-organization-member@1.0.1
+        id: is_organization_member
+        with:
+          organization: kata-containers
+          username: ${{ github.event.comment.user.login }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Fail if not member
+        run: |
+          result=${{ steps.is_organization_member.outputs.result }}
+          if [ $result == false ]; then
+              user=${{ github.event.comment.user.login }}
+              echo Either ${user} is not part of the kata-containers organization
+              echo or ${user} has its Organization Visibility set to Private at
+              echo https://github.com/orgs/kata-containers/people?query=${user}
+              echo 
+              echo Ensure you change your Organization Visibility to Public and
+              echo trigger the test again.
+              exit 1
+          fi
+
+  build-asset:
+    runs-on: ubuntu-latest
+    needs: check-comment-and-membership
+    strategy:
+      matrix:
+        asset:
+          - cloud-hypervisor
+          - firecracker
+          - kernel
+          - qemu
+          - rootfs-image
+          - rootfs-initrd
+          - shim-v2
+    steps:
+      - uses: actions/checkout@v2
+      - name: Prepare confidential container rootfs
+        if: ${{ matrix.asset == 'rootfs-initrd' }}
+        run: |
+          pushd include_rootfs/etc
+          curl -LO https://raw.githubusercontent.com/confidential-containers/documentation/main/demos/ssh-demo/aa-offline_fs_kbc-keys.json
+          mkdir kata-containers
+          envsubst < docs/how-to/data/confidential-agent-config.toml.in > kata-containers/agent.toml
+          popd
+        env:
+          AA_KBC_PARAMS: offline_fs_kbc::null
+
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          AA_KBC: offline_fs_kbc
+          INCLUDE_ROOTFS: include_rootfs
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: build-asset
+    steps:
+      - uses: actions/checkout@v2
+      - name: get-artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: kata-artifacts
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: kata-static-tarball
+          path: kata-static.tar.xz
+
+  kata-deploy:
+    needs: create-kata-tarball
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v2
+        with:
+          name: kata-static-tarball
+      - name: build-and-push-kata-deploy-ci
+        id: build-and-push-kata-deploy-ci
+        run: |
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          pushd $GITHUB_WORKSPACE
+          git checkout $tag
+          pkg_sha=$(git rev-parse HEAD)
+          popd
+          mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
+          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t quay.io/confidential-containers/runtime-payload:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
+          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
+          docker push quay.io/confidential-containers/runtime-payload:$pkg_sha
+          mkdir -p packaging/kata-deploy
+          ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
+          echo "::set-output name=PKG_SHA::${pkg_sha}"

.github/workflows/docs-url-alive-check.yaml

@@ -14,7 +14,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.19.2
+        go-version: 1.19.3
       env:
         GOPATH: ${{ runner.workspace }}/kata-containers
     - name: Set env

.github/workflows/kata-deploy-push.yaml

@@ -18,6 +18,7 @@ jobs:
       matrix:
         asset:
           - kernel
+          - kernel-dragonball-experimental
           - shim-v2
           - qemu
           - cloud-hypervisor
@@ -28,12 +29,6 @@ jobs:
           - nydus
     steps:
       - uses: actions/checkout@v2
-      - name: Install docker
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          curl -fsSL https://test.docker.com -o test-docker.sh
-          sh test-docker.sh
-
       - name: Build ${{ matrix.asset }}
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         run: |

.github/workflows/kata-deploy-test.yaml

@@ -1,5 +1,10 @@
 on:
   workflow_dispatch: # this is used to trigger the workflow on non-main branches
+    inputs:
+      pr:
+        description: 'PR number from the selected branch to test'
+        type: string
+        required: true
   issue_comment:
     types: [created, edited]
 
@@ -13,19 +18,20 @@ jobs:
       && github.event_name == 'issue_comment'
       && github.event.action == 'created'
       && startsWith(github.event.comment.body, '/test_kata_deploy')
+      || github.event_name == 'workflow_dispatch'
     steps:
-      - name: Check membership
+      - name: Check membership on comment or dispatch
         uses: kata-containers/is-organization-member@1.0.1
         id: is_organization_member
         with:
           organization: kata-containers
-          username: ${{ github.event.comment.user.login }}
+          username: ${{ github.event.comment.user.login || github.event.sender.login }}
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Fail if not member
         run: |
           result=${{ steps.is_organization_member.outputs.result }}
           if [ $result == false ]; then
-              user=${{ github.event.comment.user.login }}
+              user=${{ github.event.comment.user.login || github.event.sender.login }}
               echo Either ${user} is not part of the kata-containers organization
               echo or ${user} has its Organization Visibility set to Private at
               echo https://github.com/orgs/kata-containers/people?query=${user}
@@ -44,6 +50,7 @@ jobs:
           - cloud-hypervisor
           - firecracker
           - kernel
+          - kernel-dragonball-experimental
           - nydus
           - qemu
           - rootfs-image
@@ -54,18 +61,17 @@ jobs:
       - name: get-PR-ref
         id: get-PR-ref
         run: |
-            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
-            echo "reference for PR: " ${ref}
-            echo "##[set-output name=pr-ref;]${ref}"
+            if [ ${{ github.event_name }} == 'issue_comment' ]; then
+                ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
+            else # workflow_dispatch
+                ref="refs/pull/${{ github.event.inputs.pr }}/merge"
+            fi
+            echo "reference for PR: " ${ref} "event:" ${{ github.event_name }}
+            echo "pr-ref=${ref}" >> $GITHUB_OUTPUT
       - uses: actions/checkout@v2
         with:
           ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
 
-      - name: Install docker
-        run: |
-          curl -fsSL https://test.docker.com -o test-docker.sh
-          sh test-docker.sh
-
       - name: Build ${{ matrix.asset }}
         run: |
           make "${KATA_ASSET}-tarball"
@@ -90,9 +96,13 @@ jobs:
       - name: get-PR-ref
         id: get-PR-ref
         run: |
-            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
-            echo "reference for PR: " ${ref}
-            echo "##[set-output name=pr-ref;]${ref}"
+            if [ ${{ github.event_name }} == 'issue_comment' ]; then
+                ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
+            else # workflow_dispatch
+                ref="refs/pull/${{ github.event.inputs.pr }}/merge"
+            fi
+            echo "reference for PR: " ${ref} "event:" ${{ github.event_name }}
+            echo "pr-ref=${ref}" >> $GITHUB_OUTPUT
       - uses: actions/checkout@v2
         with:
           ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
@@ -117,9 +127,13 @@ jobs:
       - name: get-PR-ref
         id: get-PR-ref
         run: |
-            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
-            echo "reference for PR: " ${ref}
-            echo "##[set-output name=pr-ref;]${ref}"
+            if [ ${{ github.event_name }} == 'issue_comment' ]; then
+                ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
+            else # workflow_dispatch
+                ref="refs/pull/${{ github.event.inputs.pr }}/merge"
+            fi
+            echo "reference for PR: " ${ref} "event:" ${{ github.event_name }}
+            echo "pr-ref=${ref}" >> $GITHUB_OUTPUT
       - uses: actions/checkout@v2
         with:
           ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
@@ -137,7 +151,7 @@ jobs:
           docker push quay.io/kata-containers/kata-deploy-ci:$PR_SHA
           mkdir -p packaging/kata-deploy
           ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
-          echo "::set-output name=PKG_SHA::${PR_SHA}"
+          echo "PKG_SHA=${PR_SHA}" >> $GITHUB_OUTPUT
       - name: test-kata-deploy-ci-in-aks
         uses: ./packaging/kata-deploy/action
         with:

.github/workflows/release.yaml

@@ -13,6 +13,7 @@ jobs:
           - cloud-hypervisor
           - firecracker
           - kernel
+          - kernel-dragonball-experimental
           - nydus
           - qemu
           - rootfs-image
@@ -21,11 +22,6 @@ jobs:
           - virtiofsd
     steps:
       - uses: actions/checkout@v2
-      - name: Install docker
-        run: |
-          curl -fsSL https://test.docker.com -o test-docker.sh
-          sh test-docker.sh
-
       - name: Build ${{ matrix.asset }}
         run: |
           ./tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh
@@ -88,7 +84,7 @@ jobs:
           docker push quay.io/kata-containers/kata-deploy-ci:$pkg_sha
           mkdir -p packaging/kata-deploy
           ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
-          echo "::set-output name=PKG_SHA::${pkg_sha}"
+          echo "PKG_SHA=${pkg_sha}" >> $GITHUB_OUTPUT
       - name: test-kata-deploy-ci-in-aks
         uses: ./packaging/kata-deploy/action
         with:

.github/workflows/snap-release.yaml

@@ -4,6 +4,9 @@ on:
     tags:
       - '[0-9]+.[0-9]+.[0-9]+*'
 
+env:
+  SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.snapcraft_token }}
+
 jobs:
   release-snap:
     runs-on: ubuntu-20.04
@@ -14,9 +17,16 @@ jobs:
           fetch-depth: 0
 
       - name: Install Snapcraft
-        uses: samuelmeuli/action-snapcraft@v1
-        with:
-          snapcraft_token: ${{ secrets.snapcraft_token }}
+        run: |
+          # Required to avoid snapcraft install failure
+          sudo chown root:root /
+
+          # "--classic" is needed for the GitHub action runner
+          # environment.
+          sudo snap install snapcraft --classic
+
+          # Allow other parts to access snap binaries
+          echo /snap/bin >> "$GITHUB_PATH"
 
       - name: Build snap
         run: |

.github/workflows/snap.yaml

@@ -6,6 +6,7 @@ on:
       - synchronize
       - reopened
       - edited
+    paths-ignore: [ '**.md', '**.png', '**.jpg', '**.jpeg', '**.svg', '/docs/**' ]
 
 jobs:
   test:
@@ -19,7 +20,16 @@ jobs:
 
       - name: Install Snapcraft
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: samuelmeuli/action-snapcraft@v1
+        run: |
+          # Required to avoid snapcraft install failure
+          sudo chown root:root /
+
+          # "--classic" is needed for the GitHub action runner
+          # environment.
+          sudo snap install snapcraft --classic
+
+          # Allow other parts to access snap binaries
+          echo /snap/bin >> "$GITHUB_PATH"
 
       - name: Build snap
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}

.github/workflows/static-checks-dragonball.yaml

@@ -0,0 +1,33 @@
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+    paths-ignore: [ '**.md', '**.png', '**.jpg', '**.jpeg', '**.svg', '/docs/**' ]
+
+name: Static checks dragonball
+jobs:
+  test-dragonball:
+    runs-on: self-hosted
+    env:
+      RUST_BACKTRACE: "1"
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set env
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        run: |
+          echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
+      - name: Install Rust
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        run: |
+          ./ci/install_rust.sh
+          PATH=$PATH:"$HOME/.cargo/bin"
+      - name: Run Unit Test
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        run: |
+          cd src/dragonball
+          cargo version
+          rustc --version
+          sudo -E env PATH=$PATH LIBC=gnu SUPPORT_VIRTUALIZATION=true make test

.github/workflows/static-checks.yaml

@@ -8,60 +8,16 @@ on:
 
 name: Static checks
 jobs:
-  check-vendored-code:
-    runs-on: ubuntu-20.04
-    env:
-      TRAVIS: "true"
-      TRAVIS_BRANCH: ${{ github.base_ref }}
-      TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }}
-      TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }}
-      RUST_BACKTRACE: "1"
-      target_branch: ${{ github.base_ref }}
-    steps:
-    - name: Install Go
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/setup-go@v3
-      with:
-        go-version: 1.19.2
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Setup GOPATH
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}"
-        echo "TRAVIS: ${TRAVIS}"
-    - name: Set env
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
-    - name: Checkout code
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        path: ./src/github.com/${{ github.repository }}
-    - name: Setup travis references
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}"
-        target_branch=${TRAVIS_BRANCH}
-    - name: Setup
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    # Check whether the vendored code is up-to-date & working as the first thing
-    - name: Check vendored code
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make vendor
-
   static-checks:
     runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        cmd:
+          - "make vendor"
+          - "make static-checks"
+          - "make check"
+          - "make test"
+          - "sudo -E PATH=\"$PATH\" make test"
     env:
       TRAVIS: "true"
       TRAVIS_BRANCH: ${{ github.base_ref }}
@@ -70,13 +26,33 @@ jobs:
       RUST_BACKTRACE: "1"
       target_branch: ${{ github.base_ref }}
     steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        path: ./src/github.com/${{ github.repository }}
     - name: Install Go
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       uses: actions/setup-go@v3
       with:
-        go-version: 1.19.2
+        go-version: 1.19.3
       env:
         GOPATH: ${{ runner.workspace }}/kata-containers
+    - name: Check kernel config version
+      run: |
+        cd "${{ github.workspace }}/src/github.com/${{ github.repository }}"
+        kernel_dir="tools/packaging/kernel/"
+        kernel_version_file="${kernel_dir}kata_config_version"
+        modified_files=$(git diff --name-only origin/CCv0..HEAD)
+        result=$(git whatchanged origin/CCv0..HEAD "${kernel_dir}" >>"/dev/null")
+        if git whatchanged origin/CCv0..HEAD "${kernel_dir}" >>"/dev/null"; then
+          echo "Kernel directory has changed, checking if $kernel_version_file has been updated"
+          if echo "$modified_files" | grep -v "README.md" | grep "${kernel_dir}" >>"/dev/null"; then
+            echo "$modified_files" | grep "$kernel_version_file" >>/dev/null || ( echo "Please bump version in $kernel_version_file" && exit 1)
+          else
+            echo "Readme file changed, no need for kernel config version update."
+          fi
+          echo "Check passed"
+        fi
     - name: Setup GOPATH
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
@@ -89,12 +65,6 @@ jobs:
       run: |
         echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
         echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
-    - name: Checkout code
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        path: ./src/github.com/${{ github.repository }}
     - name: Setup travis references
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
@@ -114,6 +84,7 @@ jobs:
         rustup target add x86_64-unknown-linux-musl
         rustup component add rustfmt clippy
     - name: Setup seccomp
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
         libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
         gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
@@ -121,206 +92,7 @@ jobs:
         echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
         echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
         echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
-    - name: Static Checks
+    - name: Run check
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make static-checks
-
-
-  compiler-checks:
-    runs-on: ubuntu-20.04
-    env:
-      TRAVIS: "true"
-      TRAVIS_BRANCH: ${{ github.base_ref }}
-      TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }}
-      TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }}
-      RUST_BACKTRACE: "1"
-      target_branch: ${{ github.base_ref }}
-    steps:
-    - name: Install Go
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/setup-go@v3
-      with:
-        go-version: 1.19.2
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Setup GOPATH
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}"
-        echo "TRAVIS: ${TRAVIS}"
-    - name: Set env
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
-    - name: Checkout code
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        path: ./src/github.com/${{ github.repository }}
-    - name: Setup travis references
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}"
-        target_branch=${TRAVIS_BRANCH}
-    - name: Setup
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Installing rust
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh
-        PATH=$PATH:"$HOME/.cargo/bin"
-        rustup target add x86_64-unknown-linux-musl
-        rustup component add rustfmt clippy
-    - name: Setup seccomp
-      run: |
-        libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
-        gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
-        echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
-        echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
-        echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
-    - name: Run Compiler Checks
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make check
-
-  unit-tests:
-    runs-on: ubuntu-20.04
-    env:
-      TRAVIS: "true"
-      TRAVIS_BRANCH: ${{ github.base_ref }}
-      TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }}
-      TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }}
-      RUST_BACKTRACE: "1"
-      target_branch: ${{ github.base_ref }}
-    steps:
-    - name: Install Go
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/setup-go@v3
-      with:
-        go-version: 1.19.2
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Setup GOPATH
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}"
-        echo "TRAVIS: ${TRAVIS}"
-    - name: Set env
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
-    - name: Checkout code
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        path: ./src/github.com/${{ github.repository }}
-    - name: Setup travis references
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}"
-        target_branch=${TRAVIS_BRANCH}
-    - name: Setup
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Installing rust
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh
-        PATH=$PATH:"$HOME/.cargo/bin"
-        rustup target add x86_64-unknown-linux-musl
-        rustup component add rustfmt clippy
-    - name: Setup seccomp
-      run: |
-        libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
-        gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
-        echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
-        echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
-        echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
-    - name: Run Unit Tests
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make test
-
-  unit-tests-as-root:
-    runs-on: ubuntu-20.04
-    env:
-      TRAVIS: "true"
-      TRAVIS_BRANCH: ${{ github.base_ref }}
-      TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }}
-      TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }}
-      RUST_BACKTRACE: "1"
-      target_branch: ${{ github.base_ref }}
-    steps:
-    - name: Install Go
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/setup-go@v3
-      with:
-        go-version: 1.19.2 
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Setup GOPATH
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}"
-        echo "TRAVIS: ${TRAVIS}"
-    - name: Set env
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
-    - name: Checkout code
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        path: ./src/github.com/${{ github.repository }}
-    - name: Setup travis references
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}"
-        target_branch=${TRAVIS_BRANCH}
-    - name: Setup
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Installing rust
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh
-        PATH=$PATH:"$HOME/.cargo/bin"
-        rustup target add x86_64-unknown-linux-musl
-        rustup component add rustfmt clippy
-    - name: Setup seccomp
-      run: |
-        libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
-        gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
-        echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
-        echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
-        echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
-    - name: Run Unit Tests As Root User
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && sudo -E PATH="$PATH" make test
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && ${{ matrix.cmd }}

.gitignore

@@ -4,6 +4,8 @@
 **/*.rej
 **/target
 **/.vscode
+**/.idea
+**/.fleet
 pkg/logging/Cargo.lock
 src/agent/src/version.rs
 src/agent/kata-agent.service
@@ -11,4 +13,3 @@ src/agent/protocols/src/*.rs
 !src/agent/protocols/src/lib.rs
 build
 src/tools/log-parser/kata-log-parser
-

Makefile

@@ -8,6 +8,7 @@ COMPONENTS =
 
 COMPONENTS += libs
 COMPONENTS += agent
+COMPONENTS += dragonball
 COMPONENTS += runtime
 COMPONENTS += runtime-rs
 
@@ -15,11 +16,12 @@ COMPONENTS += runtime-rs
 TOOLS =
 
 TOOLS += agent-ctl
-TOOLS += trace-forwarder
-TOOLS += runk
+TOOLS += kata-ctl
 TOOLS += log-parser
+TOOLS += runk
+TOOLS += trace-forwarder
 
-STANDARD_TARGETS = build check clean install test vendor
+STANDARD_TARGETS = build check clean install static-checks-build test vendor
 
 default: all
 
@@ -35,7 +37,7 @@ generate-protocols:
 	make -C src/agent generate-protocols
 
 # Some static checks rely on generated source files of components.
-static-checks: build
+static-checks: static-checks-build
 	bash ci/static-checks.sh
 
 docs-url-alive-check:
@@ -43,10 +45,8 @@ docs-url-alive-check:
 
 .PHONY: \
 	all \
-	binary-tarball \
+	kata-tarball \
+	install-tarball \
 	default \
-	install-binary-tarball \
 	static-checks \
 	docs-url-alive-check
-
-

README.md

@@ -119,10 +119,8 @@ The table below lists the core parts of the project:
 | [runtime](src/runtime) | core | Main component run by a container manager and providing a containerd shimv2 runtime implementation. |
 | [runtime-rs](src/runtime-rs) | core | The Rust version runtime. |
 | [agent](src/agent) | core | Management process running inside the virtual machine / POD that sets up the container environment. |
-| [libraries](src/libs) | core | Library crates shared by multiple Kata Container components or published to [`crates.io`](https://crates.io/index.html) |
 | [`dragonball`](src/dragonball) | core | An optional built-in VMM brings out-of-the-box Kata Containers experience with optimizations on container workloads |
 | [documentation](docs) | documentation | Documentation common to all components (such as design and install documentation). |
-| [libraries](src/libs) | core | Library crates shared by multiple Kata Container components or published to [`crates.io`](https://crates.io/index.html) |
 | [tests](https://github.com/kata-containers/tests) | tests | Excludes unit tests which live with the main code. |
 
 ### Additional components
@@ -135,6 +133,7 @@ The table below lists the remaining parts of the project:
 | [kernel](https://www.kernel.org) | kernel | Linux kernel used by the hypervisor to boot the guest image. Patches are stored [here](tools/packaging/kernel). |
 | [osbuilder](tools/osbuilder) | infrastructure | Tool to create "mini O/S" rootfs and initrd images and kernel for the hypervisor. |
 | [`agent-ctl`](src/tools/agent-ctl) | utility | Tool that provides low-level access for testing the agent. |
+| [`kata-ctl`](src/tools/kata-ctl) | utility | Tool that provides advanced commands and debug facilities. |
 | [`trace-forwarder`](src/tools/trace-forwarder) | utility | Agent tracing helper. |
 | [`runk`](src/tools/runk) | utility | Standard OCI container runtime based on the agent. |
 | [`ci`](https://github.com/kata-containers/ci) | CI | Continuous Integration configuration files and scripts. |

VERSION

@@ -1 +1 @@
-3.0.2
+3.1.0-rc0

ci/install_libseccomp.sh

@@ -72,7 +72,8 @@ build_and_install_gperf() {
     curl -sLO "${gperf_tarball_url}"
     tar -xf "${gperf_tarball}"
     pushd "gperf-${gperf_version}"
-    # Unset $CC for configure, we will always use native for gperf
+    # gperf is a build time dependency of libseccomp and not to be used in the target.
+    # Unset $CC since that might point to a cross compiler.
     CC= ./configure --prefix="${gperf_install_dir}"
     make
     make install

ci/install_yq.sh

@@ -43,6 +43,16 @@ function install_yq() {
 	"aarch64")
 		goarch=arm64
 		;;
+	"arm64")
+		# If we're on an apple silicon machine, just assign amd64. 
+		# The version of yq we use doesn't have a darwin arm build, 
+		# but Rosetta can come to the rescue here.
+		if [ $goos == "Darwin" ]; then 
+			goarch=amd64
+		else 
+			goarch=arm64
+		fi
+		;;
 	"ppc64le")
 		goarch=ppc64le
 		;;
@@ -64,7 +74,7 @@ function install_yq() {
 	fi
 
 	## NOTE: ${var,,} => gives lowercase value of var
-	local yq_url="https://${yq_pkg}/releases/download/${yq_version}/yq_${goos,,}_${goarch}"
+	local yq_url="https://${yq_pkg}/releases/download/${yq_version}/yq_${goos}_${goarch}"
 	curl -o "${yq_path}" -LSsf "${yq_url}"
 	[ $? -ne 0 ] && die "Download ${yq_url} failed"
 	chmod +x "${yq_path}"

docs/Developer-Guide.md

@@ -33,51 +33,41 @@ You need to install the following to build Kata Containers components:
 - `make`.
 - `gcc` (required for building the shim and runtime).
 
-# Build and install the Kata Containers runtime
+# Build and install Kata Containers
+## Build and install the Kata Containers runtime
 
-```
-$ go get -d -u github.com/kata-containers/kata-containers
-$ cd $GOPATH/src/github.com/kata-containers/kata-containers/src/runtime
-$ make && sudo -E PATH=$PATH make install
+```bash
+$ git clone https://github.com/kata-containers/kata-containers.git
+$ pushd kata-containers/src/runtime
+$ make && sudo -E "PATH=$PATH" make install
+$ sudo mkdir -p /etc/kata-containers/
+$ sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers
+$ popd
 ```
 
 The build will create the following:
 
 - runtime binary: `/usr/local/bin/kata-runtime` and `/usr/local/bin/containerd-shim-kata-v2`
-- configuration file: `/usr/share/defaults/kata-containers/configuration.toml`
-
-# Check hardware requirements
-
-You can check if your system is capable of creating a Kata Container by running the following:
-
-```
-$ sudo kata-runtime check
-```
-
-If your system is *not* able to run Kata Containers, the previous command will error out and explain why.
+- configuration file: `/usr/share/defaults/kata-containers/configuration.toml` and `/etc/kata-containers/configuration.toml`
 
 ## Configure to use initrd or rootfs image
 
 Kata containers can run with either an initrd image or a rootfs image.
 
-If you want to test with `initrd`, make sure you have `initrd = /usr/share/kata-containers/kata-containers-initrd.img`
-in your configuration file, commenting out the `image` line:
+If you want to test with `initrd`, make sure you have uncommented `initrd = /usr/share/kata-containers/kata-containers-initrd.img`
+in your configuration file, commenting out the `image` line in
+`/etc/kata-containers/configuration.toml`. For example:
 
-`/usr/share/defaults/kata-containers/configuration.toml` and comment out the `image` line with the following. For example:
-
-```
-$ sudo mkdir -p /etc/kata-containers/
-$ sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers
+```bash
 $ sudo sed -i 's/^\(image =.*\)/# \1/g' /etc/kata-containers/configuration.toml
+$ sudo sed -i 's/^# \(initrd =.*\)/\1/g' /etc/kata-containers/configuration.toml
 ```
 You can create the initrd image as shown in the [create an initrd image](#create-an-initrd-image---optional) section.
 
-If you want to test with a rootfs `image`, make sure you have `image = /usr/share/kata-containers/kata-containers.img`
+If you want to test with a rootfs `image`, make sure you have uncommented `image = /usr/share/kata-containers/kata-containers.img`
 in your configuration file, commenting out the `initrd` line. For example:
 
-```
-$ sudo mkdir -p /etc/kata-containers/
-$ sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers
+```bash
 $ sudo sed -i 's/^\(initrd =.*\)/# \1/g' /etc/kata-containers/configuration.toml
 ```
 The rootfs image is created as shown in the [create a rootfs image](#create-a-rootfs-image) section.
@@ -90,19 +80,38 @@ rootfs `image`(100MB+).
 
 Enable seccomp as follows:
 
-```
+```bash
 $ sudo sed -i '/^disable_guest_seccomp/ s/true/false/' /etc/kata-containers/configuration.toml
 ```
 
 This will pass container seccomp profiles to the kata agent.
 
+## Enable SELinux on the guest
+
+> **Note:**
+>
+> - To enable SELinux on the guest, SELinux MUST be also enabled on the host.
+> - You MUST create and build a rootfs image for SELinux in advance.
+>   See [Create a rootfs image](#create-a-rootfs-image) and [Build a rootfs image](#build-a-rootfs-image).
+> - SELinux on the guest is supported in only a rootfs image currently, so
+>   you cannot enable SELinux with the agent init (`AGENT_INIT=yes`) yet.
+
+Enable guest SELinux in Enforcing mode as follows:
+
+```
+$ sudo sed -i '/^disable_guest_selinux/ s/true/false/g' /etc/kata-containers/configuration.toml
+```
+
+The runtime automatically will set `selinux=1` to the kernel parameters and `xattr` option to
+`virtiofsd` when `disable_guest_selinux` is set to `false`.
+
+If you want to enable SELinux in Permissive mode, add `enforcing=0` to the kernel parameters.
+
 ## Enable full debug
 
 Enable full debug as follows:
 
-```
-$ sudo mkdir -p /etc/kata-containers/
-$ sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers
+```bash
 $ sudo sed -i -e 's/^# *\(enable_debug\).*=.*$/\1 = true/g' /etc/kata-containers/configuration.toml
 $ sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.log=debug initcall_debug"/g' /etc/kata-containers/configuration.toml
 ```
@@ -175,7 +184,7 @@ and offers possible workarounds and fixes.
 it stores. When messages are suppressed, it is noted in the logs. This can be checked
 for by looking for those notifications, such as:
 
-```sh
+```bash
 $ sudo journalctl --since today | fgrep Suppressed
 Jun 29 14:51:17 mymachine systemd-journald[346]: Suppressed 4150 messages from /system.slice/docker.service
 ```
@@ -200,7 +209,7 @@ RateLimitBurst=0
 
 Restart `systemd-journald` for the changes to take effect:
 
-```sh
+```bash
 $ sudo systemctl restart systemd-journald
 ```
 
@@ -214,39 +223,52 @@ $ sudo systemctl restart systemd-journald
 
 The agent is built with a statically linked `musl.` The default `libc` used is `musl`, but on `ppc64le` and `s390x`, `gnu` should be used. To configure this:
 
-```
-$ export ARCH=$(uname -m)
+```bash
+$ export ARCH="$(uname -m)"
 $ if [ "$ARCH" = "ppc64le" -o "$ARCH" = "s390x" ]; then export LIBC=gnu; else export LIBC=musl; fi
-$ [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
-$ rustup target add ${ARCH}-unknown-linux-${LIBC}
+$ [ "${ARCH}" == "ppc64le" ] && export ARCH=powerpc64le
+$ rustup target add "${ARCH}-unknown-linux-${LIBC}"
 ```
 
 To build the agent:
 
-```
-$ go get -d -u github.com/kata-containers/kata-containers
-$ cd $GOPATH/src/github.com/kata-containers/kata-containers/src/agent && make
-```
-
 The agent is built with seccomp capability by default.
 If you want to build the agent without the seccomp capability, you need to run `make` with `SECCOMP=no` as follows.
 
+```bash
+$ make -C kata-containers/src/agent SECCOMP=no
 ```
-$ make -C $GOPATH/src/github.com/kata-containers/kata-containers/src/agent SECCOMP=no
+
+For building the agent with seccomp support using `musl`, set the environment
+variables for the [`libseccomp` crate](https://github.com/libseccomp-rs/libseccomp-rs).
+
+```bash
+$ export LIBSECCOMP_LINK_TYPE=static
+$ export LIBSECCOMP_LIB_PATH="the path of the directory containing libseccomp.a"
+$ make -C kata-containers/src/agent
 ```
 
+If the compilation fails when the agent tries to link the `libseccomp` library statically
+against `musl`, you will need to build `libseccomp` manually with `-U_FORTIFY_SOURCE`.
+You can use [our script](https://github.com/kata-containers/kata-containers/blob/main/ci/install_libseccomp.sh)
+to install `libseccomp` for the agent.
+
+```bash
+$ mkdir -p ${seccomp_install_path} ${gperf_install_path}
+$ kata-containers/ci/install_libseccomp.sh ${seccomp_install_path} ${gperf_install_path}
+$ export LIBSECCOMP_LIB_PATH="${seccomp_install_path}/lib"
+```
+
+On `ppc64le` and `s390x`, `glibc` is used. You will need to install the `libseccomp` library
+provided by your distribution.
+
+> e.g. `libseccomp-dev` for Ubuntu, or `libseccomp-devel` for CentOS
+
 > **Note:**
 >
 > - If you enable seccomp in the main configuration file but build the agent without seccomp capability,
 >   the runtime exits conservatively with an error message.
 
-## Get the osbuilder
-
-```
-$ go get -d -u github.com/kata-containers/kata-containers
-$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder
-```
-
 ## Create a rootfs image
 ### Create a local rootfs
 
@@ -254,24 +276,32 @@ As a prerequisite, you need to install Docker. Otherwise, you will not be
 able to run the `rootfs.sh` script with `USE_DOCKER=true` as expected in
 the following example.
 
-```
-$ export ROOTFS_DIR=${GOPATH}/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder/rootfs
-$ sudo rm -rf ${ROOTFS_DIR}
-$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder
-$ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true ./rootfs.sh ${distro}'
+```bash
+$ export distro="ubuntu" # example
+$ export ROOTFS_DIR="$(realpath kata-containers/tools/osbuilder/rootfs-builder/rootfs)"
+$ sudo rm -rf "${ROOTFS_DIR}"
+$ pushd kata-containers/tools/osbuilder/rootfs-builder
+$ script -fec 'sudo -E USE_DOCKER=true ./rootfs.sh "${distro}"'
+$ popd
 ```
 
 You MUST choose a distribution (e.g., `ubuntu`) for `${distro}`.
 You can get a supported distributions list in the Kata Containers by running the following.
 
-```
-$ ./rootfs.sh -l
+```bash
+$ ./kata-containers/tools/osbuilder/rootfs-builder/rootfs.sh -l
 ```
 
 If you want to build the agent without seccomp capability, you need to run the `rootfs.sh` script with `SECCOMP=no` as follows.
 
+```bash
+$ script -fec 'sudo -E AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh "${distro}"'
 ```
-$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
+
+If you want to enable SELinux on the guest, you MUST choose `centos` and run the `rootfs.sh` script with `SELINUX=yes` as follows.
+
+```
+$ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true SELINUX=yes ./rootfs.sh centos'
 ```
 
 > **Note:**
@@ -287,18 +317,32 @@ $ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no
 >
 > - You should only do this step if you are testing with the latest version of the agent.
 
-```
-$ sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/usr/bin ../../../src/agent/target/x86_64-unknown-linux-musl/release/kata-agent
-$ sudo install -o root -g root -m 0440 ../../../src/agent/kata-agent.service ${ROOTFS_DIR}/usr/lib/systemd/system/
-$ sudo install -o root -g root -m 0440 ../../../src/agent/kata-containers.target ${ROOTFS_DIR}/usr/lib/systemd/system/
+```bash
+$ sudo install -o root -g root -m 0550 -t "${ROOTFS_DIR}/usr/bin" "${ROOTFS_DIR}/../../../../src/agent/target/x86_64-unknown-linux-musl/release/kata-agent"
+$ sudo install -o root -g root -m 0440 "${ROOTFS_DIR}/../../../../src/agent/kata-agent.service" "${ROOTFS_DIR}/usr/lib/systemd/system/"
+$ sudo install -o root -g root -m 0440 "${ROOTFS_DIR}/../../../../src/agent/kata-containers.target" "${ROOTFS_DIR}/usr/lib/systemd/system/"
 ```
 
 ### Build a rootfs image
 
+```bash
+$ pushd  kata-containers/tools/osbuilder/image-builder
+$ script -fec 'sudo -E USE_DOCKER=true ./image_builder.sh "${ROOTFS_DIR}"'
+$ popd
 ```
-$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/image-builder
-$ script -fec 'sudo -E USE_DOCKER=true ./image_builder.sh ${ROOTFS_DIR}'
+
+If you want to enable SELinux on the guest, you MUST run the `image_builder.sh` script with `SELINUX=yes`
+to label the guest image as follows.
+To label the image on the host, you need to make sure that SELinux is enabled (`selinuxfs` is mounted) on the host
+and the rootfs MUST be created by running the `rootfs.sh` with `SELINUX=yes`.
+
 ```
+$ script -fec 'sudo -E USE_DOCKER=true SELINUX=yes ./image_builder.sh ${ROOTFS_DIR}'
+```
+
+Currently, the `image_builder.sh` uses `chcon` as an interim solution in order to apply `container_runtime_exec_t`
+to the `kata-agent`. Hence, if you run `restorecon` to the guest image after running the `image_builder.sh`,
+the `kata-agent` needs to be labeled `container_runtime_exec_t` again by yourself.
 
 > **Notes:**
 >
@@ -309,25 +353,31 @@ $ script -fec 'sudo -E USE_DOCKER=true ./image_builder.sh ${ROOTFS_DIR}'
 >   variable in the previous command and ensure the `qemu-img` command is
 >   available on your system.
 >   - If `qemu-img` is not installed, you will likely see errors such as `ERROR: File /dev/loop19p1 is not a block device` and `losetup: /tmp/tmp.bHz11oY851: Warning: file is smaller than 512 bytes; the loop device may be useless or invisible for system tools`. These can be mitigated by installing the `qemu-img` command (available in the `qemu-img` package on Fedora or the `qemu-utils` package on Debian).
+> - If `loop` module is not probed, you will likely see errors such as `losetup: cannot find an unused loop device`. Execute `modprobe loop` could resolve it.
 
 
 ### Install the rootfs image
 
-```
-$ commit=$(git log --format=%h -1 HEAD)
-$ date=$(date +%Y-%m-%d-%T.%N%z)
+```bash
+$ pushd kata-containers/tools/osbuilder/image-builder
+$ commit="$(git log --format=%h -1 HEAD)"
+$ date="$(date +%Y-%m-%d-%T.%N%z)"
 $ image="kata-containers-${date}-${commit}"
 $ sudo install -o root -g root -m 0640 -D kata-containers.img "/usr/share/kata-containers/${image}"
 $ (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
+$ popd
 ```
 
 ## Create an initrd image - OPTIONAL
 ### Create a local rootfs for initrd image
-```
-$ export ROOTFS_DIR="${GOPATH}/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder/rootfs"
-$ sudo rm -rf ${ROOTFS_DIR}
-$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder
-$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true ./rootfs.sh ${distro}'
+
+```bash
+$ export distro="ubuntu" # example
+$ export ROOTFS_DIR="$(realpath kata-containers/tools/osbuilder/rootfs-builder/rootfs)"
+$ sudo rm -rf "${ROOTFS_DIR}"
+$ pushd kata-containers/tools/osbuilder/rootfs-builder/
+$ script -fec 'sudo -E AGENT_INIT=yes USE_DOCKER=true ./rootfs.sh "${distro}"'
+$ popd
 ```
 `AGENT_INIT` controls if the guest image uses the Kata agent as the guest `init` process. When you create an initrd image,
 always set `AGENT_INIT` to `yes`.
@@ -335,14 +385,14 @@ always set `AGENT_INIT` to `yes`.
 You MUST choose a distribution (e.g., `ubuntu`) for `${distro}`.
 You can get a supported distributions list in the Kata Containers by running the following.
 
-```
-$ ./rootfs.sh -l
+```bash
+$ ./kata-containers/tools/osbuilder/rootfs-builder/rootfs.sh -l
 ```
 
 If you want to build the agent without seccomp capability, you need to run the `rootfs.sh` script with `SECCOMP=no` as follows.
 
-```
-$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
+```bash
+$ script -fec 'sudo -E AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh "${distro}"'
 ```
 
 > **Note:**
@@ -351,28 +401,31 @@ $ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no
 
 Optionally, add your custom agent binary to the rootfs with the following commands. The default `$LIBC` used
 is `musl`, but on ppc64le and s390x, `gnu` should be used. Also, Rust refers to ppc64le as `powerpc64le`:
-```
-$ export ARCH=$(uname -m)
-$ [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
-$ [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
-$ sudo install -o root -g root -m 0550 -T ../../../src/agent/target/${ARCH}-unknown-linux-${LIBC}/release/kata-agent ${ROOTFS_DIR}/sbin/init
+```bash
+$ export ARCH="$(uname -m)"
+$ [ "${ARCH}" == "ppc64le" ] || [ "${ARCH}" == "s390x" ] && export LIBC=gnu || export LIBC=musl
+$ [ "${ARCH}" == "ppc64le" ] && export ARCH=powerpc64le
+$ sudo install -o root -g root -m 0550 -T "${ROOTFS_DIR}/../../../../src/agent/target/${ARCH}-unknown-linux-${LIBC}/release/kata-agent" "${ROOTFS_DIR}/sbin/init"
 ```
 
 ### Build an initrd image
 
-```
-$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/initrd-builder
-$ script -fec 'sudo -E AGENT_INIT=yes USE_DOCKER=true ./initrd_builder.sh ${ROOTFS_DIR}'
+```bash
+$ pushd kata-containers/tools/osbuilder/initrd-builder
+$ script -fec 'sudo -E AGENT_INIT=yes USE_DOCKER=true ./initrd_builder.sh "${ROOTFS_DIR}"'
+$ popd
 ```
 
 ### Install the initrd image
 
-```
-$ commit=$(git log --format=%h -1 HEAD)
-$ date=$(date +%Y-%m-%d-%T.%N%z)
+```bash
+$ pushd kata-containers/tools/osbuilder/initrd-builder
+$ commit="$(git log --format=%h -1 HEAD)"
+$ date="$(date +%Y-%m-%d-%T.%N%z)"
 $ image="kata-containers-initrd-${date}-${commit}"
 $ sudo install -o root -g root -m 0640 -D kata-containers-initrd.img "/usr/share/kata-containers/${image}"
 $ (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers-initrd.img)
+$ popd
 ```
 
 # Install guest kernel images
@@ -391,44 +444,44 @@ Kata Containers makes use of upstream QEMU branch. The exact version
 and repository utilized can be found by looking at the [versions file](../versions.yaml).
 
 Find the correct version of QEMU from the versions file:
-```
-$ source ${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging/scripts/lib.sh
-$ qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version")
-$ echo ${qemu_version}
+```bash
+$ source kata-containers/tools/packaging/scripts/lib.sh
+$ qemu_version="$(get_from_kata_deps "assets.hypervisor.qemu.version")"
+$ echo "${qemu_version}"
 ```
 Get source from the matching branch of QEMU:
-```
-$ go get -d github.com/qemu/qemu
-$ cd ${GOPATH}/src/github.com/qemu/qemu
-$ git checkout ${qemu_version}
-$ your_qemu_directory=${GOPATH}/src/github.com/qemu/qemu
+```bash
+$ git clone -b "${qemu_version}" https://github.com/qemu/qemu.git
+$ your_qemu_directory="$(realpath qemu)"
 ```
 
 There are scripts to manage the build and packaging of QEMU. For the examples below, set your
 environment as:
-```
-$ go get -d github.com/kata-containers/kata-containers
-$ packaging_dir="${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging"
+```bash
+$ packaging_dir="$(realpath kata-containers/tools/packaging)"
 ```
 
 Kata often utilizes patches for not-yet-upstream and/or backported fixes for components,
 including QEMU. These can be found in the [packaging/QEMU directory](../tools/packaging/qemu/patches),
 and it's *recommended* that you apply them. For example, suppose that you are going to build QEMU
 version 5.2.0, do:
-```
-$ cd $your_qemu_directory
-$ $packaging_dir/scripts/apply_patches.sh $packaging_dir/qemu/patches/5.2.x/
+```bash
+$ "$packaging_dir/scripts/apply_patches.sh" "$packaging_dir/qemu/patches/5.2.x/"
 ```
 
 To build utilizing the same options as Kata, you should make use of the `configure-hypervisor.sh` script. For example:
-```
-$ cd $your_qemu_directory
-$ $packaging_dir/scripts/configure-hypervisor.sh kata-qemu > kata.cfg
+```bash
+$ pushd "$your_qemu_directory"
+$ "$packaging_dir/scripts/configure-hypervisor.sh" kata-qemu > kata.cfg
 $ eval ./configure "$(cat kata.cfg)"
 $ make -j $(nproc --ignore=1)
+# Optional
 $ sudo -E make install
+$ popd
 ```
 
+If you do not want to install the respective QEMU version, the configuration file can be modified to point to the correct binary. In `/etc/kata-containers/configuration.toml`, change `path = "/path/to/qemu/build/qemu-system-x86_64"` to point to the correct QEMU binary.
+
 See the [static-build script for QEMU](../tools/packaging/static-build/qemu/build-static-qemu.sh) for a reference on how to get, setup, configure and build QEMU for Kata.
 
 ### Build a custom QEMU for aarch64/arm64 - REQUIRED
@@ -439,11 +492,33 @@ See the [static-build script for QEMU](../tools/packaging/static-build/qemu/buil
 >   under upstream review for supporting NVDIMM on aarch64.
 >
 You could build the custom `qemu-system-aarch64` as required with the following command:
+```bash
+$ git clone https://github.com/kata-containers/tests.git
+$ script -fec 'sudo -E tests/.ci/install_qemu.sh'
 ```
-$ go get -d github.com/kata-containers/tests
-$ script -fec 'sudo -E ${GOPATH}/src/github.com/kata-containers/tests/.ci/install_qemu.sh'
+
+## Build `virtiofsd`
+
+When using the file system type virtio-fs (default), `virtiofsd` is required
+
+```bash
+$ pushd kata-containers/tools/packaging/static-build/virtiofsd
+$ ./build.sh
+$ popd
 ```
 
+Modify `/etc/kata-containers/configuration.toml` and update value `virtio_fs_daemon = "/path/to/kata-containers/tools/packaging/static-build/virtiofsd/virtiofsd/virtiofsd"` to point to the binary.
+
+# Check hardware requirements
+
+You can check if your system is capable of creating a Kata Container by running the following:
+
+```bash
+$ sudo kata-runtime check
+```
+
+If your system is *not* able to run Kata Containers, the previous command will error out and explain why.
+
 # Run Kata Containers with Containerd
 Refer to the [How to use Kata Containers and Containerd](how-to/containerd-kata.md) how-to guide.
 
@@ -474,7 +549,7 @@ See [Set up a debug console](#set-up-a-debug-console).
 
 ## Checking Docker default runtime
 
-```
+```bash
 $ sudo docker info 2>/dev/null | grep -i "default runtime" | cut -d: -f2- | grep -q runc  && echo "SUCCESS" || echo "ERROR: Incorrect default Docker runtime"
 ```
 ## Set up a debug console
@@ -491,7 +566,7 @@ contain either `/bin/sh` or `/bin/bash`.
 
 Enable debug_console_enabled in the `configuration.toml` configuration file:
 
-```
+```toml
 [agent.kata]
 debug_console_enabled = true
 ```
@@ -502,7 +577,7 @@ This will pass `agent.debug_console agent.debug_console_vport=1026` to agent as
 
 For Kata Containers `2.0.x` releases, the `kata-runtime exec` command depends on the`kata-monitor` running, in order to get the sandbox's `vsock` address to connect to. Thus, first start the `kata-monitor` process.
 
-```
+```bash
 $ sudo kata-monitor
 ```
 
@@ -564,10 +639,10 @@ an additional `coreutils` package.
 
 For example using CentOS:
 
-```
-$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder
-$ export ROOTFS_DIR=${GOPATH}/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder/rootfs
-$ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true EXTRA_PKGS="bash coreutils" ./rootfs.sh centos'
+```bash
+$ pushd kata-containers/tools/osbuilder/rootfs-builder
+$ export ROOTFS_DIR="$(realpath ./rootfs)"
+$ script -fec 'sudo -E USE_DOCKER=true EXTRA_PKGS="bash coreutils" ./rootfs.sh centos'
 ```
 
 #### Build the debug image
@@ -582,9 +657,10 @@ Install the image:
 >**Note**: When using an initrd image, replace the below rootfs image name `kata-containers.img` 
 >with the initrd image name `kata-containers-initrd.img`.
 
-```
+```bash
 $ name="kata-containers-centos-with-debug-console.img"
 $ sudo install -o root -g root -m 0640 kata-containers.img "/usr/share/kata-containers/${name}"
+$ popd
 ```
 
 Next, modify the `image=` values in the `[hypervisor.qemu]` section of the
@@ -593,7 +669,7 @@ to specify the full path to the image name specified in the previous code
 section. Alternatively, recreate the symbolic link so it points to
 the new debug image:
 
-```
+```bash
 $ (cd /usr/share/kata-containers && sudo ln -sf "$name" kata-containers.img)
 ```
 
@@ -604,7 +680,7 @@ to avoid all subsequently created containers from using the debug image.
 
 Create a container as normal. For example using `crictl`:
 
-```
+```bash
 $ sudo crictl run -r kata container.yaml pod.yaml
 ```
 
@@ -617,7 +693,7 @@ those for firecracker / cloud-hypervisor.
 
 Add `agent.debug_console` to the guest kernel command line to allow the agent process to start a debug console. 
 
-```
+```bash
 $ sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.debug_console"/g' "${kata_configuration_file}"
 ```
 
@@ -638,7 +714,7 @@ between the host and the guest. The kernel command line option `agent.debug_cons
 
 Add the parameter `agent.debug_console_vport=1026` to the kernel command line
 as shown below:
-```
+```bash
 sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.debug_console_vport=1026"/g' "${kata_configuration_file}"
 ```
 
@@ -651,7 +727,7 @@ Next, connect to the debug console. The VSOCKS paths vary slightly between each
 VMM solution.
 
 In case of cloud-hypervisor, connect to the `vsock` as shown:
-```
+```bash
 $ sudo su -c 'cd /var/run/vc/vm/${sandbox_id}/root/ && socat stdin unix-connect:clh.sock'
 CONNECT 1026
 ```
@@ -659,7 +735,7 @@ CONNECT 1026
 **Note**: You need to type `CONNECT 1026` and press `RETURN` key after entering the `socat` command.
 
 For firecracker, connect to the `hvsock` as shown:
-```
+```bash
 $ sudo su -c 'cd /var/run/vc/firecracker/${sandbox_id}/root/ && socat stdin unix-connect:kata.hvsock'
 CONNECT 1026
 ```
@@ -668,7 +744,7 @@ CONNECT 1026
 
 
 For QEMU, connect to the `vsock` as shown:
-```
+```bash
 $ sudo su -c 'cd /var/run/vc/vm/${sandbox_id} && socat "stdin,raw,echo=0,escape=0x11" "unix-connect:console.sock"'
 ```
 
@@ -681,7 +757,7 @@ If the image is created using
 [osbuilder](../tools/osbuilder), the following YAML
 file exists and contains details of the image and how it was created:
 
-```
+```bash
 $ cat /var/lib/osbuilder/osbuilder.yaml
 ```
 

docs/Release-Process.md

@@ -28,23 +28,6 @@
   $ ./update-repository-version.sh -p "$NEW_VERSION" "$BRANCH"
   ```
 
-### Point tests repository to stable branch
-
-  If you create a new stable branch, i.e. if your release changes a major or minor version number (not a patch release), then
-  you should modify the `tests` repository to point to that newly created stable branch and not the `main` branch.
-  The objective is that changes in the CI on the main branch will not impact the stable branch.
-
-  In the test directory, change references the main branch in:
-  * `README.md`
-  * `versions.yaml`
-  * `cmd/github-labels/labels.yaml.in`
-  * `cmd/pmemctl/pmemctl.sh`
-  * `.ci/lib.sh`
-  * `.ci/static-checks.sh`
-
-  See the commits in [the corresponding PR for stable-2.1](https://github.com/kata-containers/tests/pull/3504) for an example of the changes.
-
-
 ### Merge all bump version Pull requests
 
   - The above step will create a GitHub pull request in the Kata projects. Trigger the CI using `/test` command on each bump Pull request.
@@ -63,6 +46,24 @@
   $ ./tag_repos.sh -p -b "$BRANCH" tag
   ```
 
+### Point tests repository to stable branch
+
+  If your release changes a major or minor version number(not a patch release), then the above 
+  `./tag_repos.sh` script will create a new stable branch in all the repositories in addition to tagging them.
+  This happens when you are making the first `rc` release for a new major or minor version in Kata.
+  In this case, you should modify the `tests` repository to point to the newly created stable branch and not the `main` branch.
+  The objective is that changes in the CI on the main branch will not impact the stable branch.
+
+  In the test directory, change references of the `main` branch to the new stable branch in:
+  * `README.md`
+  * `versions.yaml`
+  * `cmd/github-labels/labels.yaml.in`
+  * `cmd/pmemctl/pmemctl.sh`
+  * `.ci/lib.sh`
+  * `.ci/static-checks.sh`
+
+  See the commits in [the corresponding PR for stable-2.1](https://github.com/kata-containers/tests/pull/3504) for an example of the changes.
+
 ### Check Git-hub Actions
 
   We make use of [GitHub actions](https://github.com/features/actions) in this [file](../.github/workflows/release.yaml) in the `kata-containers/kata-containers` repository to build and upload release artifacts. This action is auto triggered with the above step when a new tag is pushed to the `kata-containers/kata-containers` repository.

docs/design/README.md

@@ -7,7 +7,9 @@ Kata Containers design documents:
 - [Design requirements for Kata Containers](kata-design-requirements.md)
 - [VSocks](VSocks.md)
 - [VCPU handling](vcpu-handling.md)
+- [VCPU threads pinning](vcpu-threads-pinning.md)
 - [Host cgroups](host-cgroups.md)
+- [Agent systemd cgroup](agent-systemd-cgroup.md)
 - [`Inotify` support](inotify.md)
 - [Metrics(Kata 2.0)](kata-2-0-metrics.md)
 - [Design for Kata Containers `Lazyload` ability with `nydus`](kata-nydus-design.md)

docs/design/agent-systemd-cgroup.md

@@ -0,0 +1,84 @@
+# Systemd Cgroup for Agent
+
+As we know, we can interact with cgroups in two ways, **`cgroupfs`** and **`systemd`**. The former is achieved by reading and writing cgroup `tmpfs` files under `/sys/fs/cgroup` while the latter is done by configuring a transient unit by requesting systemd. Kata agent uses **`cgroupfs`** by default, unless you pass the parameter `--systemd-cgroup`.
+
+## usage
+
+For systemd, kata agent configures cgroups according to the following `linux.cgroupsPath` format standard provided by `runc` (`[slice]:[prefix]:[name]`). If you don't provide a valid `linux.cgroupsPath`, kata agent will treat it as `"system.slice:kata_agent:<container-id>"`. 
+
+> Here slice is a systemd slice under which the container is placed. If empty, it defaults to system.slice, except when cgroup v2 is used and rootless container is created, in which case it defaults to user.slice.
+>
+> Note that slice can contain dashes to denote a sub-slice (e.g. user-1000.slice is a correct notation, meaning a `subslice` of user.slice), but it must not contain slashes (e.g. user.slice/user-1000.slice is invalid).
+>
+> A slice of `-` represents a root slice.
+>
+> Next, prefix and name are used to compose the unit name, which is `<prefix>-<name>.scope`, unless name has `.slice` suffix, in which case prefix is ignored and the name is used as is.
+
+## supported properties
+
+The kata agent will translate the parameters in the `linux.resources` of `config.json` into systemd unit properties, and send it to systemd for configuration. Since systemd supports limited properties, only the following parameters in `linux.resources` will be applied. We will simply treat hybrid mode as legacy mode by the way.
+
+- CPU
+
+  - v1
+
+  | runtime spec resource | systemd property name |
+  | --------------------- | --------------------- |
+  | `cpu.shares`          | `CPUShares`           |
+
+  - v2
+
+  | runtime spec resource      | systemd property name      |
+  | -------------------------- | -------------------------- |
+  | `cpu.shares`               | `CPUShares`                |
+  | `cpu.period`               | `CPUQuotaPeriodUSec`(v242) |
+  | `cpu.period` & `cpu.quota` | `CPUQuotaPerSecUSec`       |
+
+- MEMORY
+
+  - v1
+
+  | runtime spec resource | systemd property name |
+  | --------------------- | --------------------- |
+  | `memory.limit`        | `MemoryLimit`         |
+
+  - v2
+
+  | runtime spec resource          | systemd property name |
+  | ------------------------------ | --------------------- |
+  | `memory.low`                   | `MemoryLow`           |
+  | `memory.max`                   | `MemoryMax`           |
+  | `memory.swap` & `memory.limit` | `MemorySwapMax`       |
+
+- PIDS
+
+  | runtime spec resource | systemd property name |
+  | --------------------- | --------------------- |
+  | `pids.limit `         | `TasksMax`            |
+
+- CPUSET
+
+  | runtime spec resource | systemd property name      |
+  | --------------------- | -------------------------- |
+  | `cpuset.cpus`         | `AllowedCPUs`(v244)        |
+  | `cpuset.mems`         | `AllowedMemoryNodes`(v244) |
+
+## Systemd Interface
+
+`session.rs` and `system.rs` in `src/agent/rustjail/src/cgroups/systemd/interface` are automatically generated by `zbus-xmlgen`, which is is an accompanying tool provided by `zbus` to generate Rust code from `D-Bus XML interface descriptions`. The specific commands to generate these two files are as follows: 
+
+```shell
+// system.rs
+zbus-xmlgen --system org.freedesktop.systemd1 /org/freedesktop/systemd1
+// session.rs
+zbus-xmlgen --session org.freedesktop.systemd1 /org/freedesktop/systemd1
+```
+
+The current implementation of `cgroups/systemd` uses `system.rs` while `session.rs` could be used to build rootless containers in the future.
+
+## references
+
+- [runc - systemd cgroup driver](https://github.com/opencontainers/runc/blob/main/docs/systemd.md)
+
+- [systemd.resource-control  — Resource control unit settings](https://www.freedesktop.org/software/systemd/man/systemd.resource-control.html)
+

docs/design/architecture_3.0/README.md

@@ -64,8 +64,8 @@ The kata-runtime is controlled by TOKIO_RUNTIME_WORKER_THREADS to run the OS thr
   ├─ TTRPC listener thread(M * tokio task)
   ├─ TTRPC client handler thread(7 * M * tokio task)
   ├─ container stdin io thread(M * tokio task)
-  ├─ container stdin io thread(M * tokio task)
-  └─ container stdin io thread(M * tokio task)
+  ├─ container stdout io thread(M * tokio task)
+  └─ container stderr io thread(M * tokio task)
 ```
 ### Extensible Framework
 The Kata 3.x runtime is designed with the extension of service, runtime, and hypervisor, combined with configuration to meet the needs of different scenarios. At present, the service provides a register mechanism to support multiple services. Services could interact with runtime through messages. In addition, the runtime handler handles messages from services. To meet the needs of a binary that supports multiple runtimes and hypervisors, the startup must obtain the runtime handler type and hypervisor type through configuration.

docs/design/direct-blk-device-assignment.md

@@ -81,7 +81,7 @@ Notes: given that the `mountInfo` is persisted to the disk by the Kata runtime,
 Instead of the CSI node driver writing the mount info into a `csiPlugin.json` file under the volume root, 
 as described in the original proposal, here we propose that the CSI node driver passes the mount information to 
 the Kata Containers runtime through a new `kata-runtime` commandline command. The `kata-runtime` then writes the mount 
-information to a `mount-info.json` file in a predefined location (`/run/kata-containers/shared/direct-volumes/[volume_path]/`).
+information to a `mountInfo.json` file in a predefined location (`/run/kata-containers/shared/direct-volumes/[volume_path]/`).
 
 When the Kata Containers runtime starts a container, it verifies whether a volume mount is a direct-assigned volume by checking 
 whether there is a `mountInfo` file under the computed Kata `direct-volumes` directory. If it is, the runtime parses the `mountInfo` file, 

docs/design/vcpu-threads-pinning.md

@@ -0,0 +1,37 @@
+# Design Doc for Kata Containers' VCPUs Pinning Feature
+
+## Background
+By now, vCPU threads of Kata Containers are scheduled randomly to CPUs. And each pod would request a specific set of CPUs which we call it CPU set (just the CPU set meaning in Linux cgroups).    
+
+If the number of vCPU threads are equal to that of CPUs claimed in CPU set, we can then pin each vCPU thread to one specified CPU, to reduce the cost of random scheduling. 
+
+## Detailed Design
+
+### Passing Config Parameters
+Two ways are provided to use this vCPU thread pinning feature: through `QEMU` configuration file and through annotations. Finally the pinning parameter is passed to `HypervisorConfig`.
+
+### Related Linux Thread Scheduling API
+
+| API Info          | Value                                                     |
+|-------------------|-----------------------------------------------------------|
+| Package           | `golang.org/x/sys/unix`                                     |
+| Method            | `unix.SchedSetaffinity(thread_id, &unixCPUSet)`             |
+| Official Doc Page | https://pkg.go.dev/golang.org/x/sys/unix#SchedSetaffinity |
+
+### When is VCPUs Pinning Checked?
+
+As shown in Section 1, when `num(vCPU threads) == num(CPUs in CPU set)`, we shall pin each vCPU thread to a specified CPU. And when this condition is broken, we should restore to the original random scheduling pattern.  
+So when may `num(CPUs in CPU set)` change? There are 5 possible scenes:
+
+| Possible scenes                   | Related Code                               |
+|-----------------------------------|--------------------------------------------|
+| when creating a container         | File Sandbox.go, in method `CreateContainer`  |
+| when starting a container         | File Sandbox.go, in method `StartContainer`   |
+| when deleting a container         | File Sandbox.go, in method `DeleteContainer`  |
+| when updating a container         | File Sandbox.go, in method `UpdateContainer`  |
+| when creating multiple containers | File Sandbox.go, in method `createContainers` |
+
+### Core Pinning Logics
+
+We can split the whole process into the following steps. Related methods are `checkVCPUsPinning` and `resetVCPUsPinning`, in file Sandbox.go.
+![](arch-images/vcpus-pinning-process.png) 

docs/design/virtualization.md

@@ -110,7 +110,7 @@ Devices and features used:
 - VFIO
 - hotplug
 - seccomp filters
-- [HTTP OpenAPI](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/vmm/src/api/openapi/cloud-hypervisor.yaml)
+- [HTTP OpenAPI](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/main/vmm/src/api/openapi/cloud-hypervisor.yaml)
 
 ### Summary
 

docs/how-to/README.md

@@ -42,4 +42,9 @@
 - [How to setup swap devices in guest kernel](how-to-setup-swap-devices-in-guest-kernel.md)
 - [How to run rootless vmm](how-to-run-rootless-vmm.md)
 - [How to run Docker with Kata Containers](how-to-run-docker-with-kata.md)
-- [How to run Kata Containers with `nydus`](how-to-use-virtio-fs-nydus-with-kata.md)
+- [How to run Kata Containers with `nydus`](how-to-use-virtio-fs-nydus-with-kata.md)
+- [How to run Kata Containers with AMD SEV-SNP](how-to-run-kata-containers-with-SNP-VMs.md)
+- [How to use EROFS to build rootfs in Kata Containers](how-to-use-erofs-build-rootfs.md)
+## Confidential Containers
+- [How to use build and test the Confidential Containers `CCv0` proof of concept](how-to-build-and-test-ccv0.md)
+- [How to generate a Kata Containers payload for the Confidential Containers Operator](how-to-generate-a-kata-containers-payload-for-the-confidential-containers-operator.md)

docs/how-to/ccv0.sh

@@ -0,0 +1,640 @@
+#!/bin/bash -e
+#
+# Copyright (c) 2021, 2022 IBM Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# Disclaimer: This script is work in progress for supporting the CCv0 prototype
+# It shouldn't be considered supported by the Kata Containers community, or anyone else
+
+# Based on https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md,
+# but with elements of the tests/.ci scripts used
+
+readonly script_name="$(basename "${BASH_SOURCE[0]}")"
+
+# By default in Golang >= 1.16 GO111MODULE is set to "on", but not all modules support it, so overwrite to "auto"
+export GO111MODULE="auto"
+
+# Setup kata containers environments if not set - we default to use containerd
+export CRI_CONTAINERD=${CRI_CONTAINERD:-"yes"}
+export CRI_RUNTIME=${CRI_RUNTIME:-"containerd"}
+export CRIO=${CRIO:-"no"}
+export KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}"
+export KUBERNETES=${KUBERNETES:-"no"}
+export AGENT_INIT="${AGENT_INIT:-${TEST_INITRD:-no}}"
+export AA_KBC="${AA_KBC:-offline_fs_kbc}"
+
+# Allow the user to overwrite the default repo and branch names if they want to build from a fork
+export katacontainers_repo="${katacontainers_repo:-github.com/kata-containers/kata-containers}"
+export katacontainers_branch="${katacontainers_branch:-CCv0}" 
+export kata_default_branch=${katacontainers_branch}
+export tests_repo="${tests_repo:-github.com/kata-containers/tests}"
+export tests_branch="${tests_branch:-CCv0}"
+export target_branch=${tests_branch} # kata-containers/ci/lib.sh uses target branch var to check out tests repo
+
+# if .bash_profile exists then use it, otherwise fall back to .profile
+export PROFILE="${HOME}/.profile"
+if [ -r "${HOME}/.bash_profile" ]; then
+    export PROFILE="${HOME}/.bash_profile"
+fi
+# Stop PS1: unbound variable error happening
+export PS1=${PS1:-}
+
+# Create a bunch of common, derived values up front so we don't need to create them in all the different functions
+. ${PROFILE}
+if [ -z ${GOPATH} ]; then
+    export GOPATH=${HOME}/go
+fi
+export tests_repo_dir="${GOPATH}/src/${tests_repo}"
+export katacontainers_repo_dir="${GOPATH}/src/${katacontainers_repo}"
+export ROOTFS_DIR="${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder/rootfs"
+export PULL_IMAGE="${PULL_IMAGE:-quay.io/kata-containers/confidential-containers:signed}" # Doesn't need authentication
+export CONTAINER_ID="${CONTAINER_ID:-0123456789}"
+source /etc/os-release || source /usr/lib/os-release
+grep -Eq "\<fedora\>" /etc/os-release 2> /dev/null && export USE_PODMAN=true
+
+
+# If we've already checked out the test repo then source the confidential scripts
+if [ "${KUBERNETES}" == "yes" ]; then
+    export BATS_TEST_DIRNAME="${tests_repo_dir}/integration/kubernetes/confidential"
+    [ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/lib.sh"
+else
+    export BATS_TEST_DIRNAME="${tests_repo_dir}/integration/containerd/confidential"
+    [ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/lib.sh"
+fi
+
+[ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
+
+export RUNTIME_CONFIG_PATH=/etc/kata-containers/configuration.toml
+
+usage() {
+    exit_code="$1"
+    cat <<EOF
+Overview:
+    Build and test kata containers from source
+    Optionally set kata-containers and tests repo and branch as exported variables before running
+    e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && ~/${script_name} build_and_install_all
+Usage:
+    ${script_name} [options] <command>
+Commands:
+- agent_create_container:           Run CreateContainer command against the agent with agent-ctl
+- agent_pull_image:                 Run PullImage command against the agent with agent-ctl
+- all:                              Build and install everything, test kata with containerd and capture the logs
+- build_and_add_agent_to_rootfs:    Builds the kata-agent and adds it to the rootfs
+- build_and_install_all:            Build and install everything
+- build_and_install_rootfs:         Builds and installs the rootfs image
+- build_kata_runtime:               Build and install the kata runtime
+- build_cloud_hypervisor            Checkout, patch, build and install Cloud Hypervisor
+- build_qemu:                       Checkout, patch, build and install QEMU
+- configure:                        Configure Kata to use rootfs and enable debug
+- connect_to_ssh_demo_pod:          Ssh into the ssh demo pod, showing that the decryption succeeded
+- copy_signature_files_to_guest     Copies signature verification files to guest
+- create_rootfs:                    Create a local rootfs
+- crictl_create_cc_container        Use crictl to create a new busybox container in the kata cc pod
+- crictl_create_cc_pod              Use crictl to create a new kata cc pod
+- crictl_delete_cc                  Use crictl to delete the kata cc pod sandbox and container in it
+- help:                             Display this help
+- init_kubernetes:                  initialize a Kubernetes cluster on this system
+- initialize:                       Install dependencies and check out kata-containers source
+- install_guest_kernel:             Setup, build and install the guest kernel
+- kubernetes_create_cc_pod:         Create a Kata CC runtime busybox-based pod in Kubernetes
+- kubernetes_create_ssh_demo_pod:   Create a Kata CC runtime pod based on the ssh demo
+- kubernetes_delete_cc_pod:         Delete the Kata CC runtime busybox-based pod in Kubernetes
+- kubernetes_delete_ssh_demo_pod:   Delete the Kata CC runtime pod based on the ssh demo
+- open_kata_shell:                  Open a shell into the kata runtime
+- rebuild_and_install_kata:         Rebuild the kata runtime and agent and build and install the image
+- shim_pull_image:                  Run PullImage command against the shim with ctr
+- test_capture_logs:                Test using kata with containerd and capture the logs in the user's home directory
+- test:                             Test using kata with containerd
+
+Options:
+    -d: Enable debug
+    -h: Display this help
+EOF
+    # if script sourced don't exit as this will exit the main shell, just return instead
+    [[ $_ != $0 ]] && return "$exit_code" || exit "$exit_code"
+}
+
+build_and_install_all() {
+    initialize
+    build_and_install_kata_runtime
+    configure
+    create_a_local_rootfs
+    build_and_install_rootfs
+    install_guest_kernel_image
+    case "$KATA_HYPERVISOR" in
+        "qemu") 
+            build_qemu
+            ;;
+        "cloud-hypervisor") 
+            build_cloud_hypervisor
+            ;;
+        *)
+            echo "Invalid option: $KATA_HYPERVISOR is not supported." >&2
+            ;;
+    esac
+
+    check_kata_runtime
+    if [ "${KUBERNETES}" == "yes" ]; then
+        init_kubernetes
+    fi
+}
+
+rebuild_and_install_kata() {
+    checkout_tests_repo
+    checkout_kata_containers_repo
+    build_and_install_kata_runtime
+    build_and_add_agent_to_rootfs
+    build_and_install_rootfs
+    check_kata_runtime
+}
+
+# Based on the jenkins_job_build.sh script in kata-containers/tests/.ci - checks out source code and installs dependencies
+initialize() {
+    # We need git to checkout and bootstrap the ci scripts and some other packages used in testing
+    sudo apt-get update && sudo apt-get install -y curl git qemu-utils
+    
+    grep -qxF "export GOPATH=\${HOME}/go" "${PROFILE}" || echo "export GOPATH=\${HOME}/go" >> "${PROFILE}"
+    grep -qxF "export GOROOT=/usr/local/go" "${PROFILE}" || echo "export GOROOT=/usr/local/go" >> "${PROFILE}"
+    grep -qxF "export PATH=\${GOPATH}/bin:/usr/local/go/bin:\${PATH}" "${PROFILE}" || echo "export PATH=\${GOPATH}/bin:/usr/local/go/bin:\${PATH}" >> "${PROFILE}"
+    
+    # Load the new go and PATH parameters from the profile
+    . ${PROFILE}
+    mkdir -p "${GOPATH}"
+
+    checkout_tests_repo
+
+    pushd "${tests_repo_dir}"
+    local ci_dir_name=".ci"
+    sudo -E PATH=$PATH -s "${ci_dir_name}/install_go.sh" -p -f
+    sudo -E PATH=$PATH -s "${ci_dir_name}/install_rust.sh"
+    # Need to change ownership of rustup so later process can create temp files there
+    sudo chown -R ${USER}:${USER} "${HOME}/.rustup"
+
+    checkout_kata_containers_repo
+
+    # Run setup, but don't install kata as we will build it ourselves in locations matching the developer guide
+    export INSTALL_KATA="no"
+    sudo -E PATH=$PATH -s ${ci_dir_name}/setup.sh
+    # Reload the profile to pick up installed dependencies
+    . ${PROFILE}
+    popd
+}
+
+checkout_tests_repo() {
+    echo "Creating repo: ${tests_repo} and branch ${tests_branch} into ${tests_repo_dir}..."
+    # Due to git https://github.blog/2022-04-12-git-security-vulnerability-announced/ the tests repo needs
+    # to be owned by root as it is re-checked out in rootfs.sh
+    mkdir -p $(dirname "${tests_repo_dir}")
+    [ -d "${tests_repo_dir}" ] || sudo -E git clone "https://${tests_repo}.git" "${tests_repo_dir}"
+    sudo -E chown -R root:root "${tests_repo_dir}"
+    pushd "${tests_repo_dir}"
+    sudo -E git fetch
+    if [ -n "${tests_branch}" ]; then
+        sudo -E git checkout ${tests_branch}
+    fi
+    sudo -E git reset --hard origin/${tests_branch}
+    popd
+
+    source "${BATS_TEST_DIRNAME}/lib.sh"
+    source "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
+}
+
+# Note: clone_katacontainers_repo using go, so that needs to be installed first
+checkout_kata_containers_repo() {
+    source "${tests_repo_dir}/.ci/lib.sh"
+    echo "Creating repo: ${katacontainers_repo} and branch ${kata_default_branch} into ${katacontainers_repo_dir}..."
+    clone_katacontainers_repo
+    sudo -E chown -R ${USER}:${USER} "${katacontainers_repo_dir}"
+}
+
+build_and_install_kata_runtime() {
+    pushd ${katacontainers_repo_dir}/src/runtime
+    make clean && make DEFAULT_HYPERVISOR=${KATA_HYPERVISOR} && sudo -E PATH=$PATH make DEFAULT_HYPERVISOR=${KATA_HYPERVISOR} install
+    popd
+}
+
+configure() {
+    configure_kata_to_use_rootfs
+    enable_full_debug
+    enable_agent_console
+
+    # Switch image offload to true in kata config
+    switch_image_service_offload "on"
+
+    configure_cc_containerd
+    # From crictl v1.24.1 the default timoout leads to the pod creation failing, so update it
+    sudo crictl config --set timeout=10
+}
+
+configure_kata_to_use_rootfs() {
+    sudo mkdir -p /etc/kata-containers/
+    sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers
+    sudo sed -i 's/^\(initrd =.*\)/# \1/g' ${RUNTIME_CONFIG_PATH}
+}
+
+build_and_add_agent_to_rootfs() {
+    build_a_custom_kata_agent
+    add_custom_agent_to_rootfs
+}
+
+build_a_custom_kata_agent() {
+    # Install libseccomp for static linking
+    sudo -E PATH=$PATH GOPATH=$GOPATH ${katacontainers_repo_dir}/ci/install_libseccomp.sh /tmp/kata-libseccomp /tmp/kata-gperf
+    export LIBSECCOMP_LINK_TYPE=static
+    export LIBSECCOMP_LIB_PATH=/tmp/kata-libseccomp/lib
+
+    . "$HOME/.cargo/env"
+    pushd ${katacontainers_repo_dir}/src/agent
+    sudo -E PATH=$PATH make
+
+    ARCH=$(uname -m)
+    [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
+    [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
+
+    # Run a make install into the rootfs directory in order to create the kata-agent.service file which is required when we add to the rootfs
+    sudo -E PATH=$PATH make install DESTDIR="${ROOTFS_DIR}"
+    popd
+}
+
+create_a_local_rootfs() {
+    sudo rm -rf "${ROOTFS_DIR}"
+    pushd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder
+    export distro="ubuntu"
+    [[ -z "${USE_PODMAN:-}" ]] && use_docker="${use_docker:-1}"
+    sudo -E OS_VERSION="${OS_VERSION:-}" GOPATH=$GOPATH EXTRA_PKGS="vim iputils-ping net-tools" DEBUG="${DEBUG:-}" USE_DOCKER="${use_docker:-}" SKOPEO=${SKOPEO:-} AA_KBC=${AA_KBC:-} UMOCI=yes SECCOMP=yes ./rootfs.sh -r ${ROOTFS_DIR} ${distro}
+
+     # Install_rust.sh during rootfs.sh switches us to the main branch of the tests repo, so switch back now
+    pushd "${tests_repo_dir}"
+    sudo -E git checkout ${tests_branch}
+    popd
+    # During the ./rootfs.sh call the kata agent is built as root, so we need to update the permissions, so we can rebuild it
+    sudo chown -R ${USER}:${USER} "${katacontainers_repo_dir}/src/agent/"
+
+    popd
+}
+
+add_custom_agent_to_rootfs() {
+    pushd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder
+
+    ARCH=$(uname -m)
+    [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
+    [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
+
+    sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/usr/bin ${katacontainers_repo_dir}/src/agent/target/${ARCH}-unknown-linux-${LIBC}/release/kata-agent
+    sudo install -o root -g root -m 0440 ../../../src/agent/kata-agent.service ${ROOTFS_DIR}/usr/lib/systemd/system/
+    sudo install -o root -g root -m 0440 ../../../src/agent/kata-containers.target ${ROOTFS_DIR}/usr/lib/systemd/system/
+    popd
+}
+
+build_and_install_rootfs() {
+    build_rootfs_image
+    install_rootfs_image
+}
+
+build_rootfs_image() {
+    pushd ${katacontainers_repo_dir}/tools/osbuilder/image-builder
+    # Logic from install_kata_image.sh - if we aren't using podman (ie on a fedora like), then use docker
+    [[ -z "${USE_PODMAN:-}" ]] && use_docker="${use_docker:-1}"
+    sudo -E USE_DOCKER="${use_docker:-}" ./image_builder.sh ${ROOTFS_DIR}
+    popd
+}
+
+install_rootfs_image() {
+    pushd ${katacontainers_repo_dir}/tools/osbuilder/image-builder
+    local commit=$(git log --format=%h -1 HEAD)
+    local date=$(date +%Y-%m-%d-%T.%N%z)
+    local image="kata-containers-${date}-${commit}"
+    sudo install -o root -g root -m 0640 -D kata-containers.img "/usr/share/kata-containers/${image}"
+    (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
+    echo "Built Rootfs from ${ROOTFS_DIR} to /usr/share/kata-containers/${image}"
+    ls -al /usr/share/kata-containers/
+    popd
+}
+
+install_guest_kernel_image() {
+    pushd ${katacontainers_repo_dir}/tools/packaging/kernel
+    sudo -E PATH=$PATH ./build-kernel.sh setup
+    sudo -E PATH=$PATH ./build-kernel.sh build
+    sudo chmod u+wrx /usr/share/kata-containers/ # Give user permission to install kernel
+    sudo -E PATH=$PATH ./build-kernel.sh install
+    popd
+}
+
+build_qemu() {
+    ${tests_repo_dir}/.ci/install_virtiofsd.sh
+    ${tests_repo_dir}/.ci/install_qemu.sh
+}
+
+build_cloud_hypervisor() {
+    ${tests_repo_dir}/.ci/install_virtiofsd.sh
+    ${tests_repo_dir}/.ci/install_cloud_hypervisor.sh
+}
+
+check_kata_runtime() {
+    sudo kata-runtime check
+}
+
+k8s_pod_file="${HOME}/busybox-cc.yaml"
+init_kubernetes() {
+    # Check that kubeadm was installed and install it otherwise
+    if ! [ -x "$(command -v kubeadm)" ]; then
+        pushd "${tests_repo_dir}/.ci"
+        sudo -E PATH=$PATH -s install_kubernetes.sh
+        if [ "${CRI_CONTAINERD}" == "yes" ]; then
+            sudo -E PATH=$PATH -s "configure_containerd_for_kubernetes.sh"
+        fi
+        popd
+    fi
+
+    # If kubernetes init has previously run we need to clean it by removing the image and resetting k8s
+    local cid=$(sudo docker ps -a -q -f name=^/kata-registry$)
+    if [ -n "${cid}" ]; then
+        sudo docker stop ${cid} && sudo docker rm ${cid}
+    fi
+    local k8s_nodes=$(kubectl get nodes -o name 2>/dev/null || true)
+    if [ -n "${k8s_nodes}" ]; then
+        sudo kubeadm reset -f
+    fi
+
+    export CI="true" && sudo -E PATH=$PATH -s ${tests_repo_dir}/integration/kubernetes/init.sh
+    sudo chown ${USER}:$(id -g -n ${USER}) "$HOME/.kube/config"
+    cat << EOF > ${k8s_pod_file}
+apiVersion: v1
+kind: Pod
+metadata:
+  name: busybox-cc
+spec:
+  runtimeClassName: kata
+  containers:
+  - name: nginx
+    image: quay.io/kata-containers/confidential-containers:signed
+    imagePullPolicy: Always  
+EOF
+}
+
+call_kubernetes_create_cc_pod() {
+    kubernetes_create_cc_pod ${k8s_pod_file}
+}
+
+call_kubernetes_delete_cc_pod() {
+    pod_name=$(kubectl get pods -o jsonpath='{.items..metadata.name}')
+    kubernetes_delete_cc_pod $pod_name
+}
+
+call_kubernetes_create_ssh_demo_pod() {
+    setup_decryption_files_in_guest
+    kubernetes_create_ssh_demo_pod
+}
+
+call_connect_to_ssh_demo_pod() {
+    connect_to_ssh_demo_pod
+}
+
+call_kubernetes_delete_ssh_demo_pod() {
+    pod=$(kubectl get pods -o jsonpath='{.items..metadata.name}')
+    kubernetes_delete_ssh_demo_pod $pod
+}
+
+crictl_sandbox_name=kata-cc-busybox-sandbox
+call_crictl_create_cc_pod() {
+    # Update iptables to allow forwarding to the cni0 bridge avoiding issues caused by the docker0 bridge
+    sudo iptables -P FORWARD ACCEPT
+    
+    # get_pod_config in tests_common exports `pod_config` that points to the prepared pod config yaml 
+    get_pod_config
+
+    crictl_delete_cc_pod_if_exists "${crictl_sandbox_name}"
+    crictl_create_cc_pod "${pod_config}"
+    sudo crictl pods
+}
+
+call_crictl_create_cc_container() {
+    # Create container configuration yaml based on our test copy of busybox
+    # get_pod_config in tests_common exports `pod_config` that points to the prepared pod config yaml 
+    get_pod_config
+
+    local container_config="${FIXTURES_DIR}/${CONTAINER_CONFIG_FILE:-container-config.yaml}"
+    local pod_name=${crictl_sandbox_name}
+    crictl_create_cc_container ${pod_name} ${pod_config} ${container_config}
+    sudo crictl ps -a
+}
+
+crictl_delete_cc() {
+    crictl_delete_cc_pod ${crictl_sandbox_name}
+}
+
+test_kata_runtime() {
+    echo "Running ctr with the kata runtime..."
+    local test_image="quay.io/kata-containers/confidential-containers:signed"
+    if [ -z $(ctr images ls -q name=="${test_image}") ]; then
+        sudo ctr image pull "${test_image}"
+    fi
+    sudo ctr run --runtime "io.containerd.kata.v2" --rm -t "${test_image}" test-kata uname -a
+}
+
+run_kata_and_capture_logs() {
+    echo "Clearing systemd journal..."
+    sudo systemctl stop systemd-journald
+    sudo rm -f /var/log/journal/*/* /run/log/journal/*/*
+    sudo systemctl start systemd-journald
+    test_kata_runtime
+    echo "Collecting logs..."
+    sudo journalctl -q -o cat -a -t kata-runtime > ${HOME}/kata-runtime.log
+    sudo journalctl -q -o cat -a -t kata > ${HOME}/shimv2.log
+    echo "Logs output to ${HOME}/kata-runtime.log and ${HOME}/shimv2.log"
+}
+
+get_ids() {
+    guest_cid=$(sudo ss -H --vsock | awk '{print $6}' | cut -d: -f1)
+    sandbox_id=$(ps -ef | grep containerd-shim-kata-v2 | egrep -o "id [^,][^,].* " | awk '{print $2}')
+}
+
+open_kata_shell() {
+    get_ids
+    sudo -E "PATH=$PATH" kata-runtime exec ${sandbox_id}
+}
+
+build_bundle_dir_if_necessary() {
+    bundle_dir="/tmp/bundle"
+    if [ ! -d "${bundle_dir}" ]; then
+        rootfs_dir="$bundle_dir/rootfs"
+        image="quay.io/kata-containers/confidential-containers:signed"
+        mkdir -p "$rootfs_dir" && (cd "$bundle_dir" && runc spec)
+        sudo docker export $(sudo docker create "$image") | tar -C "$rootfs_dir" -xvf -
+    fi
+    # There were errors in create container agent-ctl command due to /bin/ seemingly not being on the path, so hardcode it
+    sudo sed -i -e 's%^\(\t*\)"sh"$%\1"/bin/sh"%g' "${bundle_dir}/config.json"
+}
+
+build_agent_ctl() {
+    cd ${GOPATH}/src/${katacontainers_repo}/src/tools/agent-ctl/
+    if [ -e "${HOME}/.cargo/registry" ]; then
+        sudo chown -R ${USER}:${USER} "${HOME}/.cargo/registry"
+    fi
+    sudo -E PATH=$PATH -s  make
+    ARCH=$(uname -m)
+    [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
+    [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
+    cd "./target/${ARCH}-unknown-linux-${LIBC}/release/"
+}
+
+run_agent_ctl_command() {
+    get_ids
+    build_bundle_dir_if_necessary
+    command=$1
+    # If kata-agent-ctl pre-built in this directory, use it directly, otherwise build it first and switch to release
+    if [ ! -x kata-agent-ctl ]; then
+        build_agent_ctl
+    fi 
+     ./kata-agent-ctl -l debug connect --bundle-dir "${bundle_dir}" --server-address "vsock://${guest_cid}:1024" -c "${command}"
+}
+
+agent_pull_image() {
+    run_agent_ctl_command "PullImage image=${PULL_IMAGE} cid=${CONTAINER_ID} source_creds=${SOURCE_CREDS}"
+}
+
+agent_create_container() {
+    run_agent_ctl_command "CreateContainer cid=${CONTAINER_ID}"
+}
+
+shim_pull_image() {
+    get_ids
+    local ctr_shim_command="sudo ctr --namespace k8s.io shim --id ${sandbox_id} pull-image ${PULL_IMAGE} ${CONTAINER_ID}"
+    echo "Issuing command '${ctr_shim_command}'"
+    ${ctr_shim_command}
+}
+
+call_copy_signature_files_to_guest() {
+    # TODO #5173 - remove this once the kernel_params aren't ignored by the agent config
+    export DEBUG_CONSOLE="true"
+    
+    if [ "${SKOPEO:-}" = "yes" ]; then
+        add_kernel_params "agent.container_policy_file=/etc/containers/quay_verification/quay_policy.json"
+        setup_skopeo_signature_files_in_guest
+    else
+        # TODO #4888 - set config to specifically enable signature verification to be on in ImageClient
+        setup_offline_fs_kbc_signature_files_in_guest
+    fi
+}
+
+main() {
+    while getopts "dh" opt; do
+        case "$opt" in
+            d) 
+                export DEBUG="-d"
+                set -x
+                ;;
+            h) 
+                usage 0
+                ;;
+            \?)
+                echo "Invalid option: -$OPTARG" >&2
+                usage 1
+                ;;
+        esac
+    done
+
+    shift $((OPTIND - 1))
+
+    subcmd="${1:-}"
+
+    [ -z "${subcmd}" ] && usage 1
+
+    case "${subcmd}" in
+        all)
+            build_and_install_all
+            run_kata_and_capture_logs
+            ;;
+        build_and_install_all)
+            build_and_install_all
+            ;;
+        rebuild_and_install_kata)
+            rebuild_and_install_kata
+            ;;
+        initialize)
+            initialize
+            ;;
+        build_kata_runtime)
+            build_and_install_kata_runtime
+            ;;
+        configure)
+            configure
+            ;;
+        create_rootfs)
+            create_a_local_rootfs
+            ;;
+        build_and_add_agent_to_rootfs)
+            build_and_add_agent_to_rootfs
+            ;;
+        build_and_install_rootfs)
+            build_and_install_rootfs
+            ;;
+        install_guest_kernel)
+            install_guest_kernel_image
+            ;;
+        build_cloud_hypervisor)
+            build_cloud_hypervisor
+            ;;
+        build_qemu)
+            build_qemu
+            ;;
+        init_kubernetes)
+            init_kubernetes
+            ;;
+        crictl_create_cc_pod)
+            call_crictl_create_cc_pod
+            ;;
+        crictl_create_cc_container)
+            call_crictl_create_cc_container
+            ;;
+        crictl_delete_cc)
+            crictl_delete_cc
+            ;;
+        kubernetes_create_cc_pod)
+            call_kubernetes_create_cc_pod
+            ;;
+        kubernetes_delete_cc_pod)
+            call_kubernetes_delete_cc_pod
+            ;;
+        kubernetes_create_ssh_demo_pod)
+            call_kubernetes_create_ssh_demo_pod
+            ;;
+        connect_to_ssh_demo_pod)
+            call_connect_to_ssh_demo_pod
+            ;;
+        kubernetes_delete_ssh_demo_pod)
+            call_kubernetes_delete_ssh_demo_pod
+            ;;
+        test)
+            test_kata_runtime
+            ;;
+        test_capture_logs)
+            run_kata_and_capture_logs
+            ;;
+        open_kata_console)
+            open_kata_console
+            ;;
+        open_kata_shell)
+            open_kata_shell
+            ;;
+        agent_pull_image)
+            agent_pull_image
+            ;;
+        shim_pull_image)
+            shim_pull_image
+            ;;
+        agent_create_container)
+            agent_create_container
+            ;;
+        copy_signature_files_to_guest)
+            call_copy_signature_files_to_guest
+            ;;
+        *)
+            usage 1
+            ;;
+    esac
+}
+
+main $@

docs/how-to/containerd-kata.md

@@ -77,8 +77,8 @@ $ command -v containerd
 You can manually install CNI plugins as follows:
 
 ```bash
-$ go get github.com/containernetworking/plugins
-$ pushd $GOPATH/src/github.com/containernetworking/plugins
+$ git clone https://github.com/containernetworking/plugins.git
+$ pushd plugins
 $ ./build_linux.sh
 $ sudo mkdir /opt/cni
 $ sudo cp -r bin /opt/cni/
@@ -93,8 +93,8 @@ $ popd
 You can install the `cri-tools` from source code:
 
 ```bash
-$ go get github.com/kubernetes-sigs/cri-tools
-$ pushd $GOPATH/src/github.com/kubernetes-sigs/cri-tools
+$ git clone https://github.com/kubernetes-sigs/cri-tools.git
+$ pushd cri-tools
 $ make
 $ sudo -E make install
 $ popd
@@ -257,6 +257,48 @@ This launches a BusyBox container named `hello`, and it will be removed by `--rm
 The `--cni` flag enables CNI networking for the container. Without this flag, a container with just a
 loopback interface is created.
 
+### Launch containers using `ctr` command line with rootfs bundle
+
+#### Get rootfs
+Use the script to create rootfs
+```bash
+ctr i pull quay.io/prometheus/busybox:latest
+ctr i export rootfs.tar quay.io/prometheus/busybox:latest
+
+rootfs_tar=rootfs.tar
+bundle_dir="./bundle"
+mkdir -p "${bundle_dir}"
+
+# extract busybox rootfs
+rootfs_dir="${bundle_dir}/rootfs"
+mkdir -p "${rootfs_dir}"
+layers_dir="$(mktemp -d)"
+tar -C "${layers_dir}" -pxf "${rootfs_tar}"
+for ((i=0;i<$(cat ${layers_dir}/manifest.json | jq -r ".[].Layers | length");i++)); do
+  tar -C ${rootfs_dir} -xf ${layers_dir}/$(cat ${layers_dir}/manifest.json | jq -r ".[].Layers[${i}]")
+done
+```
+#### Get `config.json`
+Use runc spec to generate `config.json`
+```bash
+cd ./bundle/rootfs
+runc spec
+mv config.json ../
+```
+Change the root `path` in `config.json` to the absolute path of rootfs
+
+```JSON
+"root":{
+    "path":"/root/test/bundle/rootfs",
+    "readonly": false
+},
+```
+
+#### Run container
+```bash
+sudo ctr run -d --runtime io.containerd.run.kata.v2 --config bundle/config.json hello
+sudo ctr t exec --exec-id ${ID} -t hello sh
+```
 ### Launch Pods with `crictl` command line
 
 With the `crictl` command line of `cri-tools`, you can specify runtime class with `-r` or `--runtime` flag.

docs/how-to/data/confidential-agent-config.toml.in

@@ -0,0 +1,45 @@
+# Copyright (c) 2021 IBM Corp.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+aa_kbc_params = "$AA_KBC_PARAMS"
+https_proxy = "$HTTPS_PROXY"
+[endpoints]
+allowed = [
+"AddARPNeighborsRequest",
+"AddSwapRequest",
+"CloseStdinRequest",
+"CopyFileRequest",
+"CreateContainerRequest",
+"CreateSandboxRequest",
+"DestroySandboxRequest",
+#"ExecProcessRequest",
+"GetMetricsRequest",
+"GetOOMEventRequest",
+"GuestDetailsRequest",
+"ListInterfacesRequest",
+"ListRoutesRequest",
+"MemHotplugByProbeRequest",
+"OnlineCPUMemRequest",
+"PauseContainerRequest",
+"PullImageRequest",
+"ReadStreamRequest",
+"RemoveContainerRequest",
+#"ReseedRandomDevRequest",
+"ResizeVolumeRequest",
+"ResumeContainerRequest",
+"SetGuestDateTimeRequest",
+"SignalProcessRequest",
+"StartContainerRequest",
+"StartTracingRequest",
+"StatsContainerRequest",
+"StopTracingRequest",
+"TtyWinResizeRequest",
+"UpdateContainerRequest",
+"UpdateInterfaceRequest",
+"UpdateRoutesRequest",
+"VolumeStatsRequest",
+"WaitProcessRequest",
+"WriteStreamRequest"
+]

docs/how-to/how-to-build-and-test-ccv0.md

@@ -0,0 +1,475 @@
+# How to build, run and test Kata CCv0
+
+## Introduction and Background
+
+In order to try and make building (locally) and demoing the Kata Containers `CCv0` code base as simple as possible I've
+shared a script [`ccv0.sh`](./ccv0.sh). This script was originally my attempt to automate the steps of the 
+[Developer Guide](https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md) so that I could do
+different sections of them repeatedly and reliably as I was playing around with make changes to different parts of the 
+Kata code base. I then tried to weave in some of the [`tests/.ci`](https://github.com/kata-containers/tests/tree/main/.ci) 
+scripts in order to have less duplicated code.
+As we're progress on the confidential containers journey I hope to add more features to demonstrate the functionality
+we have working.
+
+*Disclaimer: This script has mostly just been used and tested by me ([@stevenhorsman](https://github.com/stevenhorsman)),*
+*so there might be issues with it. I'm happy to try and help solve these if possible, but this shouldn't be considered a*
+*fully supported process by the Kata Containers community.*
+
+### Basic script set-up and optional environment variables
+
+In order to build, configure and demo the CCv0 functionality, these are the set-up steps I take:
+- Provision a new VM
+    - *I choose a Ubuntu 20.04 8GB VM for this as I had one available. There are some dependences on apt-get installed*
+    *packages, so these will need re-working to be compatible with other platforms.*
+- Copy the script over to your VM *(I put it in the home directory)* and ensure it has execute permission by running 
+```bash
+$ chmod u+x ccv0.sh
+```
+- Optionally set up some environment variables
+    - By default the script checks out the `CCv0` branches of the `kata-containers/kata-containers` and 
+      `kata-containers/tests` repositories, but it is designed to be used to test of personal forks and branches as well. 
+      If you want to build and run these you can export the `katacontainers_repo`, `katacontainers_branch`, `tests_repo`
+      and `tests_branch` variables e.g.
+      ```bash
+      $ export katacontainers_repo=github.com/stevenhorsman/kata-containers
+      $ export katacontainers_branch=stevenh/agent-pull-image-endpoint
+      $ export tests_repo=github.com/stevenhorsman/tests
+      $ export tests_branch=stevenh/add-ccv0-changes-to-build
+      ```
+      before running the script.
+    - By default the build and configuration are using `QEMU` as the hypervisor. In order to use `Cloud Hypervisor` instead
+      set:
+      ```
+      $ export KATA_HYPERVISOR="cloud-hypervisor"
+      ```
+      before running the build.
+
+- At this point you can provision a Kata confidential containers pod and container with either
+  [`crictl`](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image),
+  or [Kubernetes](#using-kubernetes-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image)
+  and then test and use it. 
+
+### Using crictl for end-to-end provisioning of a Kata confidential containers pod with an unencrypted image
+
+- Run the full build process with Kubernetes turned off, so its configuration doesn't interfere with `crictl` using:
+  ```bash
+  $ export KUBERNETES="no"
+  $ export KATA_HYPERVISOR="qemu"
+  $ ~/ccv0.sh -d build_and_install_all
+  ```
+    > **Note**: Much of this script has to be run as `sudo`, so you are likely to get prompted for your password.
+    - *I run this script sourced just so that the required installed components are accessible on the `PATH` to the rest*
+      *of the process without having to reload the session.*
+    - The steps that `build_and_install_all` takes is:
+        - Checkout the git repos for the `tests` and `kata-containers` repos as specified by the environment variables
+        (default to `CCv0` branches if they are not supplied)
+        - Use the `tests/.ci` scripts to install the build dependencies
+        - Build and install the Kata runtime
+        - Configure Kata to use containerd and for debug and confidential containers features to be enabled (including
+          enabling console access to the Kata guest shell, which should only be done in development)
+        - Create, build and install a rootfs for the Kata hypervisor to use. For 'CCv0' this is currently based on Ubuntu
+        20.04.
+        - Build the Kata guest kernel
+        - Install the hypervisor (in order to select which hypervisor will be used, the `KATA_HYPERVISOR` environment
+        variable can be used to select between `qemu` or `cloud-hypervisor`)
+    > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
+    during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
+    this, login into Docker Hub and pull the images used with:
+  >  ```bash
+  >  $ sudo docker login
+  >  $ sudo docker pull ubuntu
+  >  ```
+  >  then re-run the command.
+    - The first time this runs it may take a while, but subsequent runs will be quicker as more things are already
+      installed and they can be further cut down by not running all the above steps 
+      [see "Additional script usage" below](#additional-script-usage)
+   
+- Create a new Kata sandbox pod using `crictl` with:
+  ```bash
+  $ ~/ccv0.sh crictl_create_cc_pod
+  ```
+    - This creates a pod configuration file, creates the pod from this using 
+    `sudo crictl runp -r kata ~/pod-config.yaml` and runs `sudo crictl pods` to show the pod
+- Create a new Kata confidential container with:
+  ```bash
+  $ ~/ccv0.sh crictl_create_cc_container
+  ```
+    - This creates a container (based on `busybox:1.33.1`) in the Kata cc sandbox and prints a list of containers.
+      This will have been created based on an image pulled in the Kata pod sandbox/guest, not on the host machine.
+
+As this point you should have a `crictl` pod and container that is using the Kata confidential containers runtime.
+You can [validate that the container image was pulled on the guest](#validate-that-the-container-image-was-pulled-on-the-guest)
+or [using the Kata pod sandbox for testing with `agent-ctl` or `ctr shim`](#using-a-kata-pod-sandbox-for-testing-with-agent-ctl-or-ctr-shim)
+
+#### Clean up the `crictl` pod sandbox and container
+- When the testing is complete you can delete the container and pod by running:
+  ```bash
+  $ ~/ccv0.sh crictl_delete_cc
+  ```
+### Using Kubernetes for end-to-end provisioning of a Kata confidential containers pod with an unencrypted image
+
+- Run the full build process with the Kubernetes environment variable set to `"yes"`, so the Kubernetes cluster is
+  configured and created using the VM
+  as a single node cluster: 
+  ```bash
+  $ export KUBERNETES="yes"
+  $ ~/ccv0.sh build_and_install_all
+  ```
+    > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
+    during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
+    this, login into Docker Hub and pull the images used with:
+  >  ```bash
+  >  $ sudo docker login
+  >  $ sudo docker pull registry:2
+  >  $ sudo docker pull ubuntu:20.04
+  >  ```
+  >  then re-run the command.
+- Check that your Kubernetes cluster has been correctly set-up by running : 
+  ```bash
+  $ kubectl get nodes
+  ```
+  and checking that you see a single node e.g.
+  ```text
+  NAME                             STATUS   ROLES                  AGE   VERSION
+  stevenh-ccv0-k8s1.fyre.ibm.com   Ready    control-plane,master   43s   v1.22.0
+  ```
+- Create a Kata confidential containers pod by running:
+  ```bash
+  $ ~/ccv0.sh kubernetes_create_cc_pod
+  ```
+- Wait a few seconds for pod to start then check that the pod's status is `Running` with 
+  ```bash
+  $ kubectl get pods
+  ```
+  which should show something like:
+  ```text
+  NAME         READY   STATUS    RESTARTS   AGE
+  busybox-cc   1/1     Running   0          54s
+  ```
+
+- As this point you should have a Kubernetes pod and container running, that is using the Kata 
+confidential containers runtime.
+You can [validate that the container image was pulled on the guest](#validate-that-the-container-image-was-pulled-on-the-guest)
+or [using the Kata pod sandbox for testing with `agent-ctl` or `ctr shim`](#using-a-kata-pod-sandbox-for-testing-with-agent-ctl-or-ctr-shim)
+
+#### Clean up the Kubernetes pod sandbox and container
+- When the testing is complete you can delete the container and pod by running:
+  ```bash
+  $ ~/ccv0.sh kubernetes_delete_cc_pod
+  ```
+
+### Validate that the container image was pulled on the guest
+
+There are a couple of ways we can check that the container pull image action was offloaded to the guest, by checking
+the guest's file system for the unpacked bundle and checking the host's directories to ensure it wasn't also pulled
+there.
+- To check the guest's file system:
+    - Open a shell into the Kata guest with:
+      ```bash
+      $ ~/ccv0.sh open_kata_shell
+      ```
+    - List the files in the directory that the container image bundle should have been unpacked to with:
+      ```bash
+      $ ls -ltr /run/kata-containers/confidential-containers_signed/
+      ```
+    - This should give something like
+        ```
+        total 72
+        -rw-r--r--  1 root root  2977 Jan 20 10:03 config.json
+        drwxr-xr-x 12 root root   240 Jan 20 10:03 rootfs
+        ```
+        which shows how the image has been pulled and then unbundled on the guest.
+    - Leave the Kata guest shell by running:
+      ```bash
+      $ exit
+      ```
+- To verify that the image wasn't pulled on the host system we can look at the shared sandbox on the host and we
+  should only see a single bundle for the pause container as the `busybox` based container image should have been
+  pulled on the guest:
+    - Find all the `rootfs` directories under in the pod's shared directory with:
+      ```bash
+      $ pod_id=$(ps -ef | grep containerd-shim-kata-v2 | egrep -o "id [^,][^,].* " | awk '{print $2}')
+      $ sudo find /run/kata-containers/shared/sandboxes/${pod_id}/shared -name rootfs
+      ```
+      which should only show a single `rootfs` directory if the container image was pulled on the guest, not the host
+    - Looking that `rootfs` directory with
+      ```bash
+      $ sudo ls -ltr $(sudo find /run/kata-containers/shared/sandboxes/${pod_id}/shared -name rootfs)
+      ```
+      shows something similar to
+      ```
+      total 668
+      -rwxr-xr-x 1 root root 682696 Aug 25 13:58 pause
+      drwxr-xr-x 2 root root      6 Jan 20 02:01 proc
+      drwxr-xr-x 2 root root      6 Jan 20 02:01 dev
+      drwxr-xr-x 2 root root      6 Jan 20 02:01 sys
+      drwxr-xr-x 2 root root     25 Jan 20 02:01 etc
+      ```
+      which is clearly the pause container indicating that the `busybox` based container image is not exposed to the host.
+
+### Using a Kata pod sandbox for testing with `agent-ctl` or `ctr shim`
+
+Once you have a kata pod sandbox created as described above, either using 
+[`crictl`](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image), or [Kubernetes](#using-kubernetes-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image)
+, you can use this to test specific components of the Kata confidential
+containers architecture. This can be useful for development and debugging to isolate and test features
+that aren't broadly supported end-to-end. Here are some examples:
+
+- In the first terminal run the pull image on guest command against the Kata agent, via the shim (`containerd-shim-kata-v2`).
+This can be achieved using the [containerd](https://github.com/containerd/containerd) CLI tool, `ctr`, which can be used to
+interact with the shim directly. The command takes the form
+`ctr --namespace k8s.io shim --id <sandbox-id> pull-image <image> <new-container-id>` and can been run directly, or through
+the `ccv0.sh` script to automatically fill in the variables:
+  - Optionally, set up some environment variables to set the image and credentials used:
+    - By default the shim pull image test in `ccv0.sh` will use the `busybox:1.33.1` based test image
+     `quay.io/kata-containers/confidential-containers:signed` which requires no authentication. To use a different
+      image, set the `PULL_IMAGE` environment variable e.g. 
+      ```bash
+      $ export PULL_IMAGE="docker.io/library/busybox:latest"
+      ```
+      Currently the containerd shim pull image
+      code doesn't support using a container registry that requires authentication, so if this is required, see the 
+      below steps to run the pull image command against the agent directly.
+  - Run the pull image agent endpoint with:
+    ```bash
+    $ ~/ccv0.sh shim_pull_image
+    ```
+    which we print the `ctr shim` command for reference
+- Alternatively you can issue the command directly to the `kata-agent` pull image endpoint, which also supports
+  credentials in order to pull from an authenticated registry:
+    - Optionally set up some environment variables to set the image and credentials used:
+        - Set the `PULL_IMAGE` environment variable e.g. `export PULL_IMAGE="docker.io/library/busybox:latest"`
+          if a specific container image is required.
+        - If the container registry for the image requires authentication then this can be set with an environment
+          variable `SOURCE_CREDS`. For example to use Docker Hub (`docker.io`) as an authenticated user first run 
+          `export SOURCE_CREDS="<dockerhub username>:<dockerhub api key>"`
+            > **Note**: the credentials support on the agent request is a tactical solution for the short-term
+              proof of concept to allow more images to be pulled and tested. Once we have support for getting
+              keys into the Kata guest image using the attestation-agent and/or KBS I'd expect container registry
+              credentials to be looked up using that mechanism.
+    - Run the pull image agent endpoint with
+      ```bash
+      $ ~/ccv0.sh agent_pull_image
+      ```
+      and you should see output which includes `Command PullImage (1 of 1) returned (Ok(()), false)` to indicate
+      that the `PullImage` request was successful e.g.
+      ```
+      Finished release [optimized] target(s) in 0.21s
+      {"msg":"announce","level":"INFO","ts":"2021-09-15T08:40:14.189360410-07:00","subsystem":"rpc","name":"kata-agent-ctl","pid":"830920","version":"0.1.0","source":"kata-agent-ctl","config":"Config { server_address: \"vsock://1970354082:1024\", bundle_dir: \"/tmp/bundle\", timeout_nano: 0, interactive: false, ignore_errors: false }"}
+      {"msg":"client setup complete","level":"INFO","ts":"2021-09-15T08:40:14.193639057-07:00","pid":"830920","source":"kata-agent-ctl","name":"kata-agent-ctl","subsystem":"rpc","version":"0.1.0","server-address":"vsock://1970354082:1024"}
+      {"msg":"Run command PullImage (1 of 1)","level":"INFO","ts":"2021-09-15T08:40:14.196643765-07:00","pid":"830920","source":"kata-agent-ctl","subsystem":"rpc","name":"kata-agent-ctl","version":"0.1.0"}
+      {"msg":"response received","level":"INFO","ts":"2021-09-15T08:40:43.828200633-07:00","source":"kata-agent-ctl","name":"kata-agent-ctl","subsystem":"rpc","version":"0.1.0","pid":"830920","response":""}
+      {"msg":"Command PullImage (1 of 1) returned (Ok(()), false)","level":"INFO","ts":"2021-09-15T08:40:43.828261708-07:00","subsystem":"rpc","pid":"830920","source":"kata-agent-ctl","version":"0.1.0","name":"kata-agent-ctl"}
+      ```
+      > **Note**: The first time that `~/ccv0.sh agent_pull_image` is run, the `agent-ctl` tool will be built
+      which may take a few minutes.
+- To validate that the image pull was successful, you can open a shell into the Kata guest with:
+  ```bash
+  $ ~/ccv0.sh open_kata_shell
+  ```
+- Check the `/run/kata-containers/` directory to verify that the container image bundle has been created in a directory
+  named either `01234556789` (for the container id), or the container image name, e.g.
+  ```bash
+  $ ls -ltr /run/kata-containers/confidential-containers_signed/
+  ```
+  which should show something like
+  ```
+  total 72
+  drwxr-xr-x 10 root root   200 Jan  1  1970 rootfs
+  -rw-r--r--  1 root root  2977 Jan 20 16:45 config.json
+  ```
+- Leave the Kata shell by running:
+  ```bash
+  $ exit
+  ```
+
+## Verifying signed images
+
+For this sample demo, we use local attestation to pass through the required
+configuration to do container image signature verification. Due to this, the ability to verify images is limited
+to a pre-created selection of test images in our test
+repository [`quay.io/kata-containers/confidential-containers`](https://quay.io/repository/kata-containers/confidential-containers?tab=tags).
+For pulling images not in this test repository (called an *unprotected* registry below), we fall back to the behaviour
+of not enforcing signatures. More documentation on how to customise this to match your own containers through local,
+or remote attestation will be available in future.
+
+In our test repository there are three tagged images:
+
+| Test Image | Base Image used | Signature status | GPG key status |
+| --- | --- | --- | --- |
+| `quay.io/kata-containers/confidential-containers:signed` | `busybox:1.33.1` | [signature](https://github.com/kata-containers/tests/tree/CCv0/integration/confidential/fixtures/quay_verification/x86_64/signatures.tar) embedded in kata rootfs |  [public key](https://github.com/kata-containers/tests/tree/CCv0/integration/confidential/fixtures/quay_verification/x86_64/public.gpg) embedded in kata rootfs |
+| `quay.io/kata-containers/confidential-containers:unsigned` | `busybox:1.33.1` | not signed | not signed |
+| `quay.io/kata-containers/confidential-containers:other_signed` | `nginx:1.21.3` | [signature](https://github.com/kata-containers/tests/tree/CCv0/integration/confidential/fixtures/quay_verification/x86_64/signatures.tar) embedded in kata rootfs | GPG key not kept |
+
+Using a standard unsigned `busybox` image that can be pulled from another, *unprotected*, `quay.io` repository we can
+test a few scenarios.
+
+In this sample, with local attestation, we pass in the the public GPG key and signature files, and the [`offline_fs_kbc`
+configuration](https://github.com/confidential-containers/attestation-agent/blob/main/src/kbc_modules/offline_fs_kbc/README.md)
+into the guest image which specifies that any container image from `quay.io/kata-containers`
+must be signed with the embedded GPG key and the agent configuration needs updating to enable this. 
+With this policy set a few tests of image verification can be done to test different scenarios by attempting
+to create containers from these images using `crictl`:
+
+- If you don't already have the Kata Containers CC code built and configured for `crictl`, then follow the 
+[instructions above](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image)
+up to the `~/ccv0.sh crictl_create_cc_pod` command.
+
+- In order to enable the guest image, you will need to setup the required configuration, policy and signature files
+needed by running 
+`~/ccv0.sh copy_signature_files_to_guest` and then run `~/ccv0.sh crictl_create_cc_pod` which will delete and recreate 
+your pod - adding in the new files.
+ 
+- To test the fallback behaviour works using an unsigned image from an *unprotected* registry we can pull the `busybox`
+image by running:
+  ```bash
+  $ export CONTAINER_CONFIG_FILE=container-config_unsigned-unprotected.yaml
+  $ ~/ccv0.sh crictl_create_cc_container
+  ```
+  - This finishes showing the running container e.g.
+  ```text
+  CONTAINER           IMAGE                               CREATED                  STATE               NAME                        ATTEMPT             POD ID
+  98c70fefe997a       quay.io/prometheus/busybox:latest   Less than a second ago   Running             prometheus-busybox-signed   0                   70119e0539238
+  ```
+- To test that an unsigned image from our *protected* test container registry is rejected we can run:
+  ```bash
+  $ export CONTAINER_CONFIG_FILE=container-config_unsigned-protected.yaml
+  $ ~/ccv0.sh crictl_create_cc_container
+  ```
+  - This correctly results in an error message from `crictl`:
+  `PullImage from image service failed" err="rpc error: code = Internal desc = Security validate failed: Validate image failed: The signatures do not satisfied! Reject reason: [Match reference failed.]" image="quay.io/kata-containers/confidential-containers:unsigned"`
+- To test that the signed image our *protected* test container registry is accepted we can run:
+  ```bash
+  $ export CONTAINER_CONFIG_FILE=container-config.yaml
+  $ ~/ccv0.sh crictl_create_cc_container
+  ```
+  - This finishes by showing a new `kata-cc-busybox-signed` running container e.g.
+  ```text
+  CONTAINER           IMAGE                                                    CREATED                  STATE               NAME                        ATTEMPT             POD ID
+  b4d85c2132ed9       quay.io/kata-containers/confidential-containers:signed   Less than a second ago   Running             kata-cc-busybox-signed      0                   70119e0539238
+  ...
+  ```
+- Finally to check the image with a valid signature, but invalid GPG key (the real trusted piece of information we really
+want to protect with the attestation agent in future) fails we can run: 
+  ```bash
+  $ export CONTAINER_CONFIG_FILE=container-config_signed-protected-other.yaml
+  $ ~/ccv0.sh crictl_create_cc_container
+  ```
+  - Again this results in an error message from `crictl`:
+  `"PullImage from image service failed" err="rpc error: code = Internal desc = Security validate failed: Validate image failed: The signatures do not satisfied! Reject reason: [signature verify failed! There is no pubkey can verify the signature!]" image="quay.io/kata-containers/confidential-containers:other_signed"`
+
+### Using Kubernetes to create a Kata confidential containers pod from the encrypted ssh demo sample image
+
+The [ssh-demo](https://github.com/confidential-containers/documentation/tree/main/demos/ssh-demo) explains how to
+demonstrate creating a Kata confidential containers pod from an encrypted image with the runtime created by the
+[confidential-containers operator](https://github.com/confidential-containers/documentation/blob/main/demos/operator-demo).
+To be fully confidential, this should be run on a Trusted Execution Environment, but it can be tested on generic
+hardware as well.
+
+If you wish to build the Kata confidential containers runtime to do this yourself, then you can using the following
+steps:
+
+- Run the full build process with the Kubernetes environment variable set to `"yes"`, so the Kubernetes cluster is
+  configured and created using the VM as a single node cluster and with `AA_KBC` set to `offline_fs_kbc`. 
+  ```bash
+  $ export KUBERNETES="yes"
+  $ export AA_KBC=offline_fs_kbc
+  $ ~/ccv0.sh build_and_install_all
+  ```
+    - The `AA_KBC=offline_fs_kbc` mode will ensure that, when creating the rootfs of the Kata guest, the 
+      [attestation-agent](https://github.com/confidential-containers/attestation-agent) will be added along with the
+      [sample offline KBC](https://github.com/confidential-containers/documentation/blob/main/demos/ssh-demo/aa-offline_fs_kbc-keys.json)
+      and an agent configuration file
+    > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
+    during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
+    this, login into Docker Hub and pull the images used with:
+  >  ```bash
+  >  $ sudo docker login
+  >  $ sudo docker pull registry:2
+  >  $ sudo docker pull ubuntu:20.04
+  >  ```
+  >  then re-run the command.
+- Check that your Kubernetes cluster has been correctly set-up by running : 
+  ```bash
+  $ kubectl get nodes
+  ```
+  and checking that you see a single node e.g.
+  ```text
+  NAME                             STATUS   ROLES                  AGE   VERSION
+  stevenh-ccv0-k8s1.fyre.ibm.com   Ready    control-plane,master   43s   v1.22.0
+  ```
+- Create a sample Kata confidential containers ssh pod by running:
+  ```bash
+  $ ~/ccv0.sh kubernetes_create_ssh_demo_pod
+  ```
+- As this point you should have a Kubernetes pod running the Kata confidential containers runtime that has pulled
+the [sample image](https://hub.docker.com/r/katadocker/ccv0-ssh) which was encrypted by the key file that we included
+in the rootfs.
+During the pod deployment the image was pulled and then decrypted using the key file, on the Kata guest image, without
+it ever being available to the host.
+
+- To validate that the container is working you, can connect to the image via SSH by running:
+  ```bash
+  $ ~/ccv0.sh connect_to_ssh_demo_pod
+  ```
+  - During this connection the host key fingerprint is shown and should match:
+    `ED25519 key fingerprint is SHA256:wK7uOpqpYQczcgV00fGCh+X97sJL3f6G1Ku4rvlwtR0.`
+  - After you are finished connecting then run:
+    ```bash
+    $ exit
+    ```
+  
+- To delete the sample SSH demo pod run:
+  ```bash
+  $ ~/ccv0.sh kubernetes_delete_ssh_demo_pod
+  ```
+
+## Additional script usage
+
+As well as being able to use the script as above to build all of `kata-containers` from scratch it can be used to just
+re-build bits of it by running the script with different parameters. For example after the first build you will often
+not need to re-install the dependencies, the hypervisor or the Guest kernel, but just test code changes made to the
+runtime and agent. This can be done by running `~/ccv0.sh rebuild_and_install_kata`. (*Note this does a hard checkout*
+*from git, so if your changes are only made locally it is better to do the individual steps e.g.* 
+`~/ccv0.sh build_kata_runtime && ~/ccv0.sh build_and_add_agent_to_rootfs && ~/ccv0.sh build_and_install_rootfs`).
+There are commands for a lot of steps in building, setting up and testing and the full list can be seen by running
+`~/ccv0.sh help`:
+```
+$ ~/ccv0.sh help
+Overview:
+    Build and test kata containers from source
+    Optionally set kata-containers and tests repo and branch as exported variables before running
+    e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && ~/ccv0.sh build_and_install_all
+Usage:
+    ccv0.sh [options] <command>
+Commands:
+- help:                         Display this help
+- all:                          Build and install everything, test kata with containerd and capture the logs
+- build_and_install_all:        Build and install everything
+- initialize:                   Install dependencies and check out kata-containers source
+- rebuild_and_install_kata:     Rebuild the kata runtime and agent and build and install the image
+- build_kata_runtime:           Build and install the kata runtime
+- configure:                    Configure Kata to use rootfs and enable debug
+- create_rootfs:                Create a local rootfs
+- build_and_add_agent_to_rootfs:Builds the kata-agent and adds it to the rootfs
+- build_and_install_rootfs:     Builds and installs the rootfs image
+- install_guest_kernel:         Setup, build and install the guest kernel
+- build_cloud_hypervisor        Checkout, patch, build and install Cloud Hypervisor
+- build_qemu:                   Checkout, patch, build and install QEMU
+- init_kubernetes:              initialize a Kubernetes cluster on this system
+- crictl_create_cc_pod          Use crictl to create a new kata cc pod
+- crictl_create_cc_container    Use crictl to create a new busybox container in the kata cc pod
+- crictl_delete_cc              Use crictl to delete the kata cc pod sandbox and container in it
+- kubernetes_create_cc_pod:     Create a Kata CC runtime busybox-based pod in Kubernetes
+- kubernetes_delete_cc_pod:     Delete the Kata CC runtime busybox-based pod in Kubernetes
+- open_kata_shell:              Open a shell into the kata runtime
+- agent_pull_image:             Run PullImage command against the agent with agent-ctl
+- shim_pull_image:              Run PullImage command against the shim with ctr
+- agent_create_container:       Run CreateContainer command against the agent with agent-ctl
+- test:                         Test using kata with containerd
+- test_capture_logs:            Test using kata with containerd and capture the logs in the user's home directory
+
+Options:
+    -d: Enable debug
+    -h: Display this help
+```

docs/how-to/how-to-generate-a-kata-containers-payload-for-the-confidential-containers-operator.md

@@ -0,0 +1,44 @@
+# Generating a Kata Containers payload for the Confidential Containers Operator
+
+[Confidential Containers
+Operator](https://github.com/confidential-containers/operator) consumes a Kata
+Containers payload, generated from the `CCv0` branch, and here one can find all
+the necessary info on how to build such a payload.
+
+## Requirements
+
+* `make` installed in the machine
+* Docker installed in the machine
+* `sudo` access to the machine
+
+## Process
+
+* Clone [Kata Containers](https://github.com/kata-containers/kata-containers)
+  ```sh
+  git clone --branch CCv0 https://github.com/kata-containers/kata-containers
+  ```
+  * In case you've already cloned the repo, make sure to switch to the `CCv0` branch
+    ```sh
+    git checkout CCv0
+    ```
+  * Ensure your tree is clean and in sync with upstream `CCv0`
+    ```sh
+    git clean -xfd
+    git reset --hard <upstream>/CCv0
+    ```
+* Make sure you're authenticated to `quay.io`
+    ```sh
+    sudo docker login quay.io
+    ```
+* From the top repo directory, run:
+  ```sh
+  sudo make cc-payload
+  ```
+* Make sure the image was upload to the [Confidential Containers
+  runtime-payload
+registry](https://quay.io/repository/confidential-containers/runtime-payload?tab=tags)
+
+## Notes
+
+Make sure to run it on a machine that's not the one you're hacking on, prepare a
+cup of tea, and get back to it an hour later (at least).

docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md

@@ -0,0 +1,159 @@
+# Kata Containers with AMD SEV-SNP VMs
+
+## Disclaimer
+
+This guide is designed for developers and is - same as the Developer Guide - not intended for production systems or end users. It is advisable to only follow this guide on non-critical development systems.
+
+## Prerequisites
+
+To run  Kata Containers in SNP-VMs, the following software stack is used.
+
+![Kubernetes integration with shimv2](./images/SNP-stack.svg)
+
+The host BIOS and kernel must be capable of supporting AMD SEV-SNP and configured accordingly. For Kata Containers, the host kernel with branch [`sev-snp-iommu-avic_5.19-rc6_v3`](https://github.com/AMDESE/linux/tree/sev-snp-iommu-avic_5.19-rc6_v3) and commit [`3a88547`](https://github.com/AMDESE/linux/commit/3a885471cf89156ea555341f3b737ad2a8d9d3d0) is known to work in conjunction with SEV Firmware version 1.51.3 (0xh\_1.33.03) available on AMD's [SEV developer website](https://developer.amd.com/sev/). See [AMD's guide](https://github.com/AMDESE/AMDSEV/tree/sev-snp-devel) to configure the host accordingly. Verify that you are able to run SEV-SNP encrypted VMs first. The guest components required for Kata Containers are built as described below.
+
+**Tip**: It is easiest to first have Kata Containers running on your system and then modify it to run containers in SNP-VMs. Follow the [Developer guide](../Developer-Guide.md#warning) and then follow the below steps. Nonetheless, you can just follow this guide from the start.
+
+## How to build
+
+Follow all of the below steps to install Kata Containers with SNP-support from scratch. These steps mostly follow the developer guide with modifications to support SNP
+
+__Steps from the Developer Guide:__
+- Get all the [required components](../Developer-Guide.md#requirements-to-build-individual-components) for building the kata-runtime
+- [Build the and install kata-runtime](../Developer-Guide.md#build-and-install-the-kata-containers-runtime)
+- [Build a custom agent](../Developer-Guide.md#build-a-custom-kata-agent---optional)
+- [Create an initrd image](../Developer-Guide.md#create-an-initrd-image---optional) by first building a rootfs, then building the initrd based on the rootfs, use a custom agent and install. `ubuntu` works as the distribution of choice.
+- Get the [required components](../../tools/packaging/kernel/README.md#requirements) to build a custom kernel
+
+__SNP-specific steps:__
+- Build the SNP-specific kernel as shown below (see this [guide](../../tools/packaging/kernel/README.md#build-kata-containers-kernel) for more information)
+```bash
+$ pushd kata-containers/tools/packaging/kernel/
+$ ./build-kernel.sh -a x86_64 -x snp setup
+$ ./build-kernel.sh -a x86_64 -x snp build
+$ sudo -E PATH="${PATH}" ./build-kernel.sh -x snp install
+$ popd
+```
+- Build a current OVMF capable of SEV-SNP:
+```bash
+$ pushd kata-containers/tools/packaging/static-build/ovmf
+$ ./build.sh
+$ tar -xvf edk2-x86_64.tar.gz
+$ popd
+```
+- Build a custom QEMU
+```bash
+$ source kata-containers/tools/packaging/scripts/lib.sh
+$ qemu_url="$(get_from_kata_deps "assets.hypervisor.qemu.snp.url")"
+$ qemu_branch="$(get_from_kata_deps "assets.hypervisor.qemu.snp.branch")"
+$ qemu_commit="$(get_from_kata_deps "assets.hypervisor.qemu.snp.commit")"
+$ git clone -b "${qemu_branch}" "${qemu_url}"
+$ pushd qemu
+$ git checkout "${qemu_commit}"
+$ ./configure --enable-virtfs --target-list=x86_64-softmmu --enable-debug
+$ make -j "$(nproc)"
+$ popd
+```
+
+### Kata Containers Configuration for SNP
+
+The configuration file located at `/etc/kata-containers/configuration.toml` must be adapted as follows to support SNP-VMs:
+- Use the SNP-specific kernel for the guest VM (change path)
+```toml
+kernel = "/usr/share/kata-containers/vmlinuz-snp.container"
+```
+- Enable the use of an initrd (uncomment)
+```toml
+initrd = "/usr/share/kata-containers/kata-containers-initrd.img"
+```
+- Disable the use of a rootfs (comment out)
+```toml
+# image = "/usr/share/kata-containers/kata-containers.img"
+```
+- Use the custom QEMU capable of SNP (change path)
+```toml
+path = "/path/to/qemu/build/qemu-system-x86_64"
+```
+- Use `virtio-9p` device since `virtio-fs` is unsupported due to bugs / shortcomings in QEMU version [`snp-v3`](https://github.com/AMDESE/qemu/tree/snp-v3) for SEV and SEV-SNP (change value)
+```toml
+shared_fs = "virtio-9p"
+```
+- Disable `virtiofsd` since it is no longer required (comment out)
+```toml
+# virtio_fs_daemon = "/usr/libexec/virtiofsd"
+```
+- Disable NVDIMM (uncomment)
+```toml
+disable_image_nvdimm = true
+```
+- Disable shared memory (uncomment)
+```toml
+file_mem_backend = ""
+```
+- Enable confidential guests (uncomment)
+```toml
+confidential_guest = true
+```
+- Enable SNP-VMs (uncomment)
+```toml
+sev_snp_guest = true
+```
+  - Configure an OVMF (add path)
+```toml
+firmware = "/path/to/kata-containers/tools/packaging/static-build/ovmf/opt/kata/share/ovmf/OVMF.fd"
+```
+
+## Test Kata Containers with Containerd
+
+With Kata Containers configured to support SNP-VMs, we use containerd to test and deploy containers in these VMs.
+
+### Install Containerd
+If not already present, follow [this guide](./containerd-kata.md#install) to install containerd and its related components including `CNI` and the `cri-tools` (skip Kata Containers since we already installed it)
+
+### Containerd Configuration
+
+Follow [this guide](./containerd-kata.md#configuration) to configure containerd to use Kata Containers
+
+## Run Kata Containers in SNP-VMs
+
+Run the below commands to start a container. See [this guide](./containerd-kata.md#run) for more information
+```bash
+$ sudo ctr image pull docker.io/library/busybox:latest
+$ sudo ctr run --cni --runtime io.containerd.run.kata.v2 -t --rm docker.io/library/busybox:latest hello sh
+```
+
+### Check for active SNP:
+
+Inside the running container, run the following commands to check if SNP is active. It should look something like this:
+```
+/ # dmesg | grep -i sev
+[    0.299242] Memory Encryption Features active: AMD SEV SEV-ES SEV-SNP
+[    0.472286] SEV: Using SNP CPUID table, 31 entries present.
+[    0.514574] SEV: SNP guest platform device initialized.
+[    0.885425] sev-guest sev-guest: Initialized SEV guest driver (using vmpck_id 0)
+```
+
+### Obtain an SNP Attestation Report
+
+To obtain an attestation report inside the container, the `/dev/sev-guest` must first be configured. As of now, the VM does not perform this step, however it can be performed inside the container, either in the terminal or in code.
+
+Example for shell:
+```
+/ # SNP_MAJOR=$(cat /sys/devices/virtual/misc/sev-guest/dev | awk -F: '{print $1}')
+/ # SNP_MINOR=$(cat /sys/devices/virtual/misc/sev-guest/dev | awk -F: '{print $2}')
+/ # mknod -m 600 /dev/sev-guest c "${SNP_MAJOR}" "${SNP_MINOR}"
+```
+
+## Known Issues
+
+- Support for cgroups v2 is still [work in progress](https://github.com/kata-containers/kata-containers/issues/927). If issues occur due to cgroups v2 becoming the default in newer systems, one possible solution is to downgrade cgroups to v1:
+```bash
+sudo sed -i 's/^\(GRUB_CMDLINE_LINUX=".*\)"/\1 systemd.unified_cgroup_hierarchy=0"/' /etc/default/grub
+sudo update-grub
+sudo reboot
+```
+- If both SEV and SEV-SNP are supported by the host, Kata Containers uses SEV-SNP by default. You can verify what features are enabled by checking `/sys/module/kvm_amd/parameters/sev` and `sev_snp`. This means that Kata Containers can not run both SEV-SNP-VMs and SEV-VMs at the same time. If SEV is to be used by Kata Containers instead, reload the `kvm_amd` kernel module without SNP-support, this will disable SNP-support for the entire platform.
+```bash
+sudo rmmod kvm_amd && sudo modprobe kvm_amd sev_snp=0
+```
+

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -57,6 +57,7 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.enable_iothreads` | `boolean`| enable IO to be processed in a separate thread. Supported currently for virtio-`scsi` driver |
 | `io.katacontainers.config.hypervisor.enable_mem_prealloc` | `boolean` | the memory space used for `nvdimm` device by the hypervisor |
 | `io.katacontainers.config.hypervisor.enable_vhost_user_store` | `boolean` | enable vhost-user storage device (QEMU) |
+| `io.katacontainers.config.hypervisor.vhost_user_reconnect_timeout_sec` | `string`| the timeout for reconnecting vhost user socket (QEMU) 
 | `io.katacontainers.config.hypervisor.enable_virtio_mem` | `boolean` | enable virtio-mem (QEMU) |
 | `io.katacontainers.config.hypervisor.entropy_source` (R) | string| the path to a host source of entropy (`/dev/random`, `/dev/urandom` or real hardware RNG device) |
 | `io.katacontainers.config.hypervisor.file_mem_backend` (R) | string | file based memory backend root directory |
@@ -87,7 +88,7 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.use_vsock` | `boolean` | specify use of `vsock` for agent communication |
 | `io.katacontainers.config.hypervisor.vhost_user_store_path` (R) | `string` | specify the directory path where vhost-user devices related folders, sockets and device nodes should be (QEMU) |
 | `io.katacontainers.config.hypervisor.virtio_fs_cache_size` | uint32 | virtio-fs DAX cache size in `MiB` |
-| `io.katacontainers.config.hypervisor.virtio_fs_cache` | string | the cache mode for virtio-fs, valid values are `always`, `auto` and `none` |
+| `io.katacontainers.config.hypervisor.virtio_fs_cache` | string | the cache mode for virtio-fs, valid values are `always`, `auto` and `never` |
 | `io.katacontainers.config.hypervisor.virtio_fs_daemon` | string | virtio-fs `vhost-user` daemon path |
 | `io.katacontainers.config.hypervisor.virtio_fs_extra_args` | string | extra options passed to `virtiofs` daemon |
 | `io.katacontainers.config.hypervisor.enable_guest_swap` | `boolean` | enable swap in the guest |

docs/how-to/how-to-setup-swap-devices-in-guest-kernel.md

@@ -17,9 +17,9 @@ Enable setup swap device in guest kernel as follows:
 $ sudo sed -i -e 's/^#enable_guest_swap.*$/enable_guest_swap = true/g' /etc/kata-containers/configuration.toml
 ```
 
-## Run a Kata Container utilizing swap device
+## Run a Kata Containers utilizing swap device
 
-Use following command to start a Kata Container with swappiness 60 and 1GB swap device (swap_in_bytes - memory_limit_in_bytes).
+Use following command to start a Kata Containers with swappiness 60 and 1GB swap device (swap_in_bytes - memory_limit_in_bytes).
 ```
 $ pod_yaml=pod.yaml
 $ container_yaml=container.yaml
@@ -27,6 +27,8 @@ $ image="quay.io/prometheus/busybox:latest"
 $ cat << EOF > "${pod_yaml}"
 metadata:
   name: busybox-sandbox1
+  uid: $(uuidgen)
+  namespace: default
 EOF
 $ cat << EOF > "${container_yaml}"
 metadata:
@@ -43,12 +45,12 @@ command:
 - top
 EOF
 $ sudo crictl pull $image
-$ podid=$(sudo crictl runp $pod_yaml)
+$ podid=$(sudo crictl runp --runtime kata $pod_yaml)
 $ cid=$(sudo crictl create $podid $container_yaml $pod_yaml)
 $ sudo crictl start $cid
 ```
 
-Kata Container setups swap device for this container only when `io.katacontainers.container.resource.swappiness` is set.
+Kata Containers setups swap device for this container only when `io.katacontainers.container.resource.swappiness` is set.
 
 The following table shows the swap size how to decide if `io.katacontainers.container.resource.swappiness` is set.
 |`io.katacontainers.container.resource.swap_in_bytes`|`memory_limit_in_bytes`|swap size|

docs/how-to/how-to-use-erofs-build-rootfs.md

@@ -0,0 +1,90 @@
+# Configure Kata Containers to use EROFS build rootfs
+
+## Introduction
+For kata containers, rootfs is used in the read-only way. EROFS can noticeably decrease metadata overhead.
+
+`mkfs.erofs` can generate compressed and uncompressed EROFS images.
+
+For uncompressed images, no files are compressed. However, it is optional to inline the data blocks at the end of the file with the metadata.
+
+For compressed images, each file will be compressed using the lz4 or lz4hc algorithm, and it will be confirmed whether it can save space. Use No compression of the file if compression does not save space.
+
+## Performance comparison
+|                 | EROFS | EXT4 | XFS |
+|-----------------|-------| --- | --- |
+| Image Size [MB] | 106(uncompressed) | 256 | 126 |
+
+
+## Guidance
+### Install the `erofs-utils`
+#### `apt/dnf` install
+On newer `Ubuntu/Debian` systems, it can be installed directly using the `apt` command, and on `Fedora` it can be installed directly using the `dnf` command.
+
+```shell
+# Debian/Ubuntu
+$ apt install erofs-utils
+# Fedora
+$ dnf install erofs-utils
+```
+
+#### Source install
+[https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git](https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git)
+
+##### Compile dependencies
+If you need to enable the `Lz4` compression feature, `Lz4 1.8.0+` is required, and `Lz4 1.9.3+` is strongly recommended.
+
+##### Compilation process
+For some old lz4 versions (lz4-1.8.0~1.8.3), if lz4-static is not installed, the lz4hc algorithm will not be supported. lz4-static can be installed with apt install lz4-static.x86_64. However, these versions have some bugs in compression, and it is not recommended to use these versions directly.
+If you use `lz4 1.9.0+`, you can directly use the following command to compile.
+
+```shell
+$ ./autogen.sh
+$ ./configure
+$ make
+```
+
+The compiled `mkfs.erofs` program will be saved in the `mkfs` directory. Afterwards, the generated tools can be installed to a system directory using make install (requires root privileges).
+
+### Create a local rootfs
+```shell
+$ export distro="ubuntu"
+$ export FS_TYPE="erofs"
+$ export ROOTFS_DIR="realpath kata-containers/tools/osbuilder/rootfs-builder/rootfs"
+$ sudo rm -rf "${ROOTFS_DIR}"
+$ pushd kata-containers/tools/osbuilder/rootfs-builder
+$ script -fec 'sudo -E SECCOMP=no ./rootfs.sh "${distro}"'
+$ popd
+```
+
+### Add a custom agent to the image - OPTIONAL
+> Note:
+> - You should only do this step if you are testing with the latest version of the agent.
+```shell
+$ sudo install -o root -g root -m 0550 -t "${ROOTFS_DIR}/usr/bin" "${ROOTFS_DIR}/../../../../src/agent/target/x86_64-unknown-linux-musl/release/kata-agent"
+$ sudo install -o root -g root -m 0440 "${ROOTFS_DIR}/../../../../src/agent/kata-agent.service" "${ROOTFS_DIR}/usr/lib/systemd/system/"
+$ sudo install -o root -g root -m 0440 "${ROOTFS_DIR}/../../../../src/agent/kata-containers.target" "${ROOTFS_DIR}/usr/lib/systemd/system/"
+```
+
+### Build a root image
+```shell
+$ pushd  kata-containers/tools/osbuilder/image-builder
+$ script -fec 'sudo -E ./image_builder.sh "${ROOTFS_DIR}"'
+$ popd
+```
+
+### Install the rootfs image
+```shell
+$ pushd kata-containers/tools/osbuilder/image-builder
+$ commit="$(git log --format=%h -1 HEAD)"
+$ date="$(date +%Y-%m-%d-%T.%N%z)"
+$ rootfs="erofs"
+$ image="kata-containers-${rootfs}-${date}-${commit}"
+$ sudo install -o root -g root -m 0640 -D kata-containers.img "/usr/share/kata-containers/${image}"
+$ (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
+$ popd
+```
+
+### Use `EROFS` in the runtime
+```shell
+$ sudo sed -i -e 's/^# *\(rootfs_type\).*=.*$/\1 = erofs/g' /etc/kata-containers/configuration.toml
+```

docs/how-to/how-to-use-kata-containers-with-firecracker.md

@@ -104,7 +104,7 @@ sudo dmsetup create "${POOL_NAME}" \
 
 cat << EOF
 #
-# Add this to your config.toml configuration file and restart `containerd` daemon
+# Add this to your config.toml configuration file and restart containerd daemon
 #
 [plugins]
   [plugins.devmapper]
@@ -212,7 +212,7 @@ Next, we need to configure containerd. Add a file in your path (e.g. `/usr/local
 
 ```
 #!/bin/bash
-KATA_CONF_FILE=/etc/containers/configuration-fc.toml /usr/local/bin/containerd-shim-kata-v2 $@
+KATA_CONF_FILE=/etc/kata-containers/configuration-fc.toml /usr/local/bin/containerd-shim-kata-v2 $@
 ```
 > **Note:** You may need to edit the paths of the configuration file and the `containerd-shim-kata-v2` to correspond to your setup.
 

docs/how-to/how-to-use-virtio-fs-nydus-with-kata.md

@@ -32,6 +32,7 @@ The `nydus-sandbox.yaml` looks like below:
 metadata:
   attempt: 1
   name: nydus-sandbox
+  uid: nydus-uid
   namespace: default
 log_directory: /tmp
 linux:

docs/how-to/how-to-use-virtio-mem-with-kata.md

@@ -42,6 +42,8 @@ $ image="quay.io/prometheus/busybox:latest"
 $ cat << EOF > "${pod_yaml}"
 metadata:
   name: busybox-sandbox1
+  uid: $(uuidgen)
+  namespace: default
 EOF
 $ cat << EOF > "${container_yaml}"
 metadata:

docs/install/kata-containers-3.0-rust-runtime-installation-guide.md

@@ -24,7 +24,7 @@ architectures:
 
 | Installation method                                  | Description                                                                                  | Automatic updates | Use case                                                                                      | Availability
 |------------------------------------------------------|----------------------------------------------------------------------------------------------|-------------------|-----------------------------------------------------------------------------------------------|----------- |
-| [Using kata-deploy](#kata-deploy-installation)       | The preferred way to deploy the Kata Containers distributed binaries on a Kubernetes cluster | **No!**           | Best way to give it a try on kata-containers on an already up and running Kubernetes cluster. | No |
+| [Using kata-deploy](#kata-deploy-installation)       | The preferred way to deploy the Kata Containers distributed binaries on a Kubernetes cluster | **No!**           | Best way to give it a try on kata-containers on an already up and running Kubernetes cluster. | Yes |
 | [Using official distro packages](#official-packages) | Kata packages provided by Linux distributions official repositories                          | yes               | Recommended for most users. | No |                                                                   
 | [Using snap](#snap-installation)                     | Easy to install                                                                              | yes               | Good alternative to official distro packages.                                                 | No |
 | [Automatic](#automatic-installation)                 | Run a single command to install a full system                                                | **No!**           | For those wanting the latest release quickly.                                                 | No |
@@ -32,7 +32,8 @@ architectures:
 | [Build from source](#build-from-source-installation) | Build the software components manually                                                       | **No!**           | Power users and developers only.  | Yes |              
 
 ### Kata Deploy Installation
-`ToDo`
+
+Follow the [`kata-deploy`](../../tools/packaging/kata-deploy/README.md).
 ### Official packages
 `ToDo`
 ### Snap Installation
@@ -48,14 +49,14 @@ architectures:
 
 * Download `Rustup` and install  `Rust`
     > **Notes:**
-    > Rust version 1.58 is needed
+    > Rust version 1.62.0 is needed
 
     Example for `x86_64`
     ```
     $ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
     $ source $HOME/.cargo/env
-    $ rustup install 1.58
-    $ rustup default 1.58-x86_64-unknown-linux-gnu
+    $ rustup install 1.62.0
+    $ rustup default 1.62.0-x86_64-unknown-linux-gnu
     ```
 
 * Musl support for fully static binary
@@ -83,7 +84,7 @@ $ git clone https://github.com/kata-containers/kata-containers.git
 $ cd kata-containers/src/runtime-rs
 $ make && sudo make install
 ```
-After running the command above, the default config file `configuration.toml` will be installed under `/usr/share/defaults/kata-containers/`,  the binary file `containerd-shim-kata-v2` will be installed under `/user/local/bin` .
+After running the command above, the default config file `configuration.toml` will be installed under `/usr/share/defaults/kata-containers/`,  the binary file `containerd-shim-kata-v2` will be installed under `/usr/local/bin/` .
 
 ### Build Kata Containers Kernel
 Follow the [Kernel installation guide](/tools/packaging/kernel/README.md).

docs/install/minikube-installation-guide.md

@@ -71,12 +71,6 @@ To use containerd, modify the `--container-runtime` argument:
 > **Notes:**
 > - Adjust the `--memory 6144` line to suit your environment and requirements. Kata Containers default to
 > requesting 2048MB per container. We recommended you supply more than that to the Minikube node.
-> - Prior to Minikube/Kubernetes v1.14, the beta `RuntimeClass` feature also needed enabling with
-> the following.
->
-> | what | why |
-> | ---- | --- |
-> | `--feature-gates=RuntimeClass=true` | Kata needs to use the `RuntimeClass` Kubernetes feature |
 
 The full command is therefore:
 
@@ -138,17 +132,9 @@ $ kubectl -n kube-system exec ${podname} -- ps -ef | fgrep infinity
 
 ## Enabling Kata Containers
 
-> **Note:** Only Minikube/Kubernetes versions <= 1.13 require this step. Since version
-> v1.14, the `RuntimeClass` is enabled by default. Performing this step on Kubernetes > v1.14 is
-> however benign.
-
 Now you have installed the Kata Containers components in the Minikube node. Next, you need to configure
 Kubernetes `RuntimeClass` to know when to use Kata Containers to run a pod.
 
-```sh
-$ kubectl apply -f https://raw.githubusercontent.com/kubernetes/node-api/master/manifests/runtimeclass_crd.yaml > runtimeclass_crd.yaml
-```
-
 ### Register the runtime
 
 Now register the `kata qemu` runtime with that class. This should result in no errors:

docs/use-cases/NVIDIA-GPU-passthrough-and-Kata.md

@@ -545,6 +545,12 @@ Create the hook execution file for Kata:
 /usr/bin/nvidia-container-toolkit -debug $@
 ```
 
+Make sure the hook shell is executable:
+
+```sh
+chmod +x $ROOTFS_DIR/usr/share/oci/hooks/prestart/nvidia-container-toolkit.sh
+```
+
 As the last step one can do some cleanup of files or package caches. Build the
 rootfs and configure it for use with Kata according to the development guide.
 

docs/use-cases/using-Intel-QAT-and-kata.md

@@ -49,7 +49,7 @@ the latest driver.
 $ export QAT_DRIVER_VER=qat1.7.l.4.14.0-00031.tar.gz
 $ export QAT_DRIVER_URL=https://downloadmirror.intel.com/30178/eng/${QAT_DRIVER_VER}
 $ export QAT_CONF_LOCATION=~/QAT_conf
-$ export QAT_DOCKERFILE=https://raw.githubusercontent.com/intel/intel-device-plugins-for-kubernetes/master/demo/openssl-qat-engine/Dockerfile
+$ export QAT_DOCKERFILE=https://raw.githubusercontent.com/intel/intel-device-plugins-for-kubernetes/main/demo/openssl-qat-engine/Dockerfile
 $ export QAT_SRC=~/src/QAT
 $ export GOPATH=~/src/go
 $ export KATA_KERNEL_LOCATION=~/kata

docs/use-cases/using-Intel-SGX-and-kata.md

@@ -61,6 +61,9 @@ spec:
           name: eosgx-demo-job-1
           image: oeciteam/oe-helloworld:latest
           imagePullPolicy: IfNotPresent
+          volumeMounts:
+          - mountPath: /dev
+            name: dev-mount
           securityContext:
             readOnlyRootFilesystem: true
             capabilities:

docs/use-cases/using-SPDK-vhostuser-and-kata.md

@@ -197,11 +197,6 @@ vhost_user_store_path = "<Path of the base directory for vhost-user device>"
 > under `[hypervisor.qemu]` section.
 
 
-For the subdirectories of `vhost_user_store_path`: `block` is used for block
-device; `block/sockets` is where we expect UNIX domain sockets for vhost-user
-block devices to live; `block/devices` is where simulated block device nodes
-for vhost-user block devices are created.
-
 For the subdirectories of `vhost_user_store_path`:
 -  `block` is used for block device;
 -  `block/sockets` is where we expect UNIX domain sockets for vhost-user

src/agent/Cargo.toml

@@ -23,7 +23,7 @@ regex = "1.5.6"
 serial_test = "0.5.1"
 kata-sys-util = { path = "../libs/kata-sys-util" }
 kata-types = { path = "../libs/kata-types" }
-sysinfo = "0.23.0"
+url = "2.2.2"
 
 # Async helpers
 async-trait = "0.1.42"
@@ -31,7 +31,7 @@ async-recursion = "0.3.2"
 futures = "0.3.17"
 
 # Async runtime
-tokio = { version = "1.14.0", features = ["full"] }
+tokio = { version = "1.21.2", features = ["full"] }
 tokio-vsock = "0.3.1"
 
 netlink-sys = { version = "0.7.0", features = ["tokio_socket",]}
@@ -52,7 +52,7 @@ log = "0.4.11"
 prometheus = { version = "0.13.0", features = ["process"] }
 procfs = "0.12.0"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.2.10" }
+cgroups = { package = "cgroups-rs", version = "0.3.1" }
 
 # Tracing
 tracing = "0.1.26"
@@ -66,12 +66,23 @@ serde = { version = "1.0.129", features = ["derive"] }
 toml = "0.5.8"
 clap = { version = "3.0.1", features = ["derive"] }
 
+# "vendored" feature for openssl is required by musl build
+openssl = { version = "0.10.38", features = ["vendored"] }
+
+# Image pull/decrypt
+[target.'cfg(target_arch = "s390x")'.dependencies]
+image-rs = { git = "https://github.com/confidential-containers/image-rs", rev = "v0.4.0", default-features = false, features = ["kata-cc-s390x"] }
+
+[target.'cfg(not(target_arch = "s390x"))'.dependencies]
+image-rs = { git = "https://github.com/confidential-containers/image-rs", rev = "v0.4.0", default-features = true, features = ["kata-cc", "signature-simple"] }
+
 [dev-dependencies]
 tempfile = "3.1.0"
 test-utils = { path = "../libs/test-utils" }
 which = "4.3.0"
 
 [workspace]
+resolver = "2"
 members = [
     "rustjail",
 ]

src/agent/Makefile

@@ -107,6 +107,8 @@ endef
 ##TARGET default: build code
 default: $(TARGET) show-header
 
+static-checks-build: $(GENERATED_CODE)
+
 $(TARGET): $(GENERATED_CODE) $(TARGET_PATH)
 
 $(TARGET_PATH): show-summary

src/agent/rustjail/Cargo.toml

@@ -11,6 +11,7 @@ serde_json = "1.0.39"
 serde_derive = "1.0.91"
 oci = { path = "../../libs/oci" }
 protocols = { path ="../../libs/protocols" }
+kata-sys-util = { path = "../../libs/kata-sys-util" }
 caps = "0.5.0"
 nix = "0.24.2"
 scopeguard = "1.0.0"
@@ -24,7 +25,7 @@ scan_fmt = "0.2.6"
 regex = "1.5.6"
 path-absolutize = "1.2.0"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.2.10" }
+cgroups = { package = "cgroups-rs", version = "0.3.1" }
 rlimit = "0.5.3"
 cfg-if = "0.1.0"
 
@@ -32,7 +33,10 @@ tokio = { version = "1.2.0", features = ["sync", "io-util", "process", "time", "
 futures = "0.3.17"
 async-trait = "0.1.31"
 inotify = "0.9.2"
-libseccomp = { version = "0.2.3", optional = true }
+libseccomp = { version = "0.3.0", optional = true }
+zbus = "2.3.0"
+bit-vec= "0.6.3"
+xattr = "0.2.3"
 
 [dev-dependencies]
 serial_test = "0.5.0"

src/agent/rustjail/src/cgroups/fs/mod.rs

@@ -32,6 +32,7 @@ use protocols::agent::{
     BlkioStats, BlkioStatsEntry, CgroupStats, CpuStats, CpuUsage, HugetlbStats, MemoryData,
     MemoryStats, PidsStats, ThrottlingData,
 };
+use std::any::Any;
 use std::collections::HashMap;
 use std::fs;
 use std::path::Path;
@@ -75,7 +76,7 @@ macro_rules! set_resource {
 
 impl CgroupManager for Manager {
     fn apply(&self, pid: pid_t) -> Result<()> {
-        self.cgroup.add_task(CgroupPid::from(pid as u64))?;
+        self.cgroup.add_task_by_tgid(CgroupPid::from(pid as u64))?;
         Ok(())
     }
 
@@ -193,6 +194,79 @@ impl CgroupManager for Manager {
 
         Ok(result)
     }
+
+    fn update_cpuset_path(&self, guest_cpuset: &str, container_cpuset: &str) -> Result<()> {
+        if guest_cpuset.is_empty() {
+            return Ok(());
+        }
+        info!(sl!(), "update_cpuset_path to: {}", guest_cpuset);
+
+        let h = cgroups::hierarchies::auto();
+        let root_cg = h.root_control_group();
+
+        let root_cpuset_controller: &CpuSetController = root_cg.controller_of().unwrap();
+        let path = root_cpuset_controller.path();
+        let root_path = Path::new(path);
+        info!(sl!(), "root cpuset path: {:?}", &path);
+
+        let container_cpuset_controller: &CpuSetController = self.cgroup.controller_of().unwrap();
+        let path = container_cpuset_controller.path();
+        let container_path = Path::new(path);
+        info!(sl!(), "container cpuset path: {:?}", &path);
+
+        let mut paths = vec![];
+        for ancestor in container_path.ancestors() {
+            if ancestor == root_path {
+                break;
+            }
+            paths.push(ancestor);
+        }
+        info!(sl!(), "parent paths to update cpuset: {:?}", &paths);
+
+        let mut i = paths.len();
+        loop {
+            if i == 0 {
+                break;
+            }
+            i -= 1;
+
+            // remove cgroup root from path
+            let r_path = &paths[i]
+                .to_str()
+                .unwrap()
+                .trim_start_matches(root_path.to_str().unwrap());
+            info!(sl!(), "updating cpuset for parent path {:?}", &r_path);
+            let cg = new_cgroup(cgroups::hierarchies::auto(), r_path)?;
+            let cpuset_controller: &CpuSetController = cg.controller_of().unwrap();
+            cpuset_controller.set_cpus(guest_cpuset)?;
+        }
+
+        if !container_cpuset.is_empty() {
+            info!(
+                sl!(),
+                "updating cpuset for container path: {:?} cpuset: {}",
+                &container_path,
+                container_cpuset
+            );
+            container_cpuset_controller.set_cpus(container_cpuset)?;
+        }
+
+        Ok(())
+    }
+
+    fn get_cgroup_path(&self, cg: &str) -> Result<String> {
+        if cgroups::hierarchies::is_cgroup2_unified_mode() {
+            let cg_path = format!("/sys/fs/cgroup/{}", self.cpath);
+            return Ok(cg_path);
+        }
+
+        // for cgroup v1
+        Ok(self.paths.get(cg).map(|s| s.to_string()).unwrap())
+    }
+
+    fn as_any(&self) -> Result<&dyn Any> {
+        Ok(self)
+    }
 }
 
 fn set_network_resources(
@@ -252,19 +326,28 @@ fn set_devices_resources(
 }
 
 fn set_hugepages_resources(
-    _cg: &cgroups::Cgroup,
+    cg: &cgroups::Cgroup,
     hugepage_limits: &[LinuxHugepageLimit],
     res: &mut cgroups::Resources,
 ) {
     info!(sl!(), "cgroup manager set hugepage");
     let mut limits = vec![];
+    let hugetlb_controller = cg.controller_of::<HugeTlbController>();
 
     for l in hugepage_limits.iter() {
-        let hr = HugePageResource {
-            size: l.page_size.clone(),
-            limit: l.limit,
-        };
-        limits.push(hr);
+        if hugetlb_controller.is_some() && hugetlb_controller.unwrap().size_supported(&l.page_size)
+        {
+            let hr = HugePageResource {
+                size: l.page_size.clone(),
+                limit: l.limit,
+            };
+            limits.push(hr);
+        } else {
+            warn!(
+                sl!(),
+                "{} page size support cannot be verified, dropping requested limit", l.page_size
+            );
+        }
     }
     res.hugepages.limits = limits;
 }
@@ -458,8 +541,11 @@ fn linux_device_to_cgroup_device(d: &LinuxDevice) -> Option<DeviceResource> {
 }
 
 fn linux_device_group_to_cgroup_device(d: &LinuxDeviceCgroup) -> Option<DeviceResource> {
-    let dev_type = match DeviceType::from_char(d.r#type.chars().next()) {
-        Some(t) => t,
+    let dev_type = match &d.r#type {
+        Some(t_s) => match DeviceType::from_char(t_s.chars().next()) {
+            Some(t_c) => t_c,
+            None => return None,
+        },
         None => return None,
     };
 
@@ -516,7 +602,7 @@ lazy_static! {
             // all mknod to all char devices
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: "c".to_string(),
+                r#type: Some("c".to_string()),
                 major: Some(WILDCARD),
                 minor: Some(WILDCARD),
                 access: "m".to_string(),
@@ -525,7 +611,7 @@ lazy_static! {
             // all mknod to all block devices
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: "b".to_string(),
+                r#type: Some("b".to_string()),
                 major: Some(WILDCARD),
                 minor: Some(WILDCARD),
                 access: "m".to_string(),
@@ -534,7 +620,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/console
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: "c".to_string(),
+                r#type: Some("c".to_string()),
                 major: Some(5),
                 minor: Some(1),
                 access: "rwm".to_string(),
@@ -543,7 +629,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/pts/<N>
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: "c".to_string(),
+                r#type: Some("c".to_string()),
                 major: Some(136),
                 minor: Some(WILDCARD),
                 access: "rwm".to_string(),
@@ -552,7 +638,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/ptmx
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: "c".to_string(),
+                r#type: Some("c".to_string()),
                 major: Some(5),
                 minor: Some(2),
                 access: "rwm".to_string(),
@@ -561,7 +647,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/net/tun
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: "c".to_string(),
+                r#type: Some("c".to_string()),
                 major: Some(10),
                 minor: Some(200),
                 access: "rwm".to_string(),
@@ -608,17 +694,6 @@ fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> SingularPtrField<CpuUsage> {
         });
     }
 
-    if cg.v2() {
-        return SingularPtrField::some(CpuUsage {
-            total_usage: 0,
-            percpu_usage: vec![],
-            usage_in_kernelmode: 0,
-            usage_in_usermode: 0,
-            unknown_fields: UnknownFields::default(),
-            cached_size: CachedSize::default(),
-        });
-    }
-
     // try to get from cpu controller
     let cpu_controller: &CpuController = get_controller_or_return_singular_none!(cg);
     let stat = cpu_controller.cpu().stat;
@@ -649,7 +724,7 @@ fn get_memory_stats(cg: &cgroups::Cgroup) -> SingularPtrField<MemoryStats> {
     let value = memory.use_hierarchy;
     let use_hierarchy = value == 1;
 
-    // gte memory datas
+    // get memory data
     let usage = SingularPtrField::some(MemoryData {
         usage: memory.usage_in_bytes,
         max_usage: memory.max_usage_in_bytes,
@@ -940,9 +1015,9 @@ pub fn get_mounts(paths: &HashMap<String, String>) -> Result<HashMap<String, Str
     Ok(m)
 }
 
-fn new_cgroup(h: Box<dyn cgroups::Hierarchy>, path: &str) -> Cgroup {
+fn new_cgroup(h: Box<dyn cgroups::Hierarchy>, path: &str) -> Result<Cgroup> {
     let valid_path = path.trim_start_matches('/').to_string();
-    cgroups::Cgroup::new(h, valid_path.as_str())
+    cgroups::Cgroup::new(h, valid_path.as_str()).map_err(anyhow::Error::from)
 }
 
 impl Manager {
@@ -964,83 +1039,16 @@ impl Manager {
             m.insert(key.to_string(), p);
         }
 
+        let cg = new_cgroup(cgroups::hierarchies::auto(), cpath)?;
+
         Ok(Self {
             paths: m,
             mounts,
             // rels: paths,
             cpath: cpath.to_string(),
-            cgroup: new_cgroup(cgroups::hierarchies::auto(), cpath),
+            cgroup: cg,
         })
     }
-
-    pub fn update_cpuset_path(&self, guest_cpuset: &str, container_cpuset: &str) -> Result<()> {
-        if guest_cpuset.is_empty() {
-            return Ok(());
-        }
-        info!(sl!(), "update_cpuset_path to: {}", guest_cpuset);
-
-        let h = cgroups::hierarchies::auto();
-        let root_cg = h.root_control_group();
-
-        let root_cpuset_controller: &CpuSetController = root_cg.controller_of().unwrap();
-        let path = root_cpuset_controller.path();
-        let root_path = Path::new(path);
-        info!(sl!(), "root cpuset path: {:?}", &path);
-
-        let container_cpuset_controller: &CpuSetController = self.cgroup.controller_of().unwrap();
-        let path = container_cpuset_controller.path();
-        let container_path = Path::new(path);
-        info!(sl!(), "container cpuset path: {:?}", &path);
-
-        let mut paths = vec![];
-        for ancestor in container_path.ancestors() {
-            if ancestor == root_path {
-                break;
-            }
-            paths.push(ancestor);
-        }
-        info!(sl!(), "parent paths to update cpuset: {:?}", &paths);
-
-        let mut i = paths.len();
-        loop {
-            if i == 0 {
-                break;
-            }
-            i -= 1;
-
-            // remove cgroup root from path
-            let r_path = &paths[i]
-                .to_str()
-                .unwrap()
-                .trim_start_matches(root_path.to_str().unwrap());
-            info!(sl!(), "updating cpuset for parent path {:?}", &r_path);
-            let cg = new_cgroup(cgroups::hierarchies::auto(), r_path);
-            let cpuset_controller: &CpuSetController = cg.controller_of().unwrap();
-            cpuset_controller.set_cpus(guest_cpuset)?;
-        }
-
-        if !container_cpuset.is_empty() {
-            info!(
-                sl!(),
-                "updating cpuset for container path: {:?} cpuset: {}",
-                &container_path,
-                container_cpuset
-            );
-            container_cpuset_controller.set_cpus(container_cpuset)?;
-        }
-
-        Ok(())
-    }
-
-    pub fn get_cg_path(&self, cg: &str) -> Option<String> {
-        if cgroups::hierarchies::is_cgroup2_unified_mode() {
-            let cg_path = format!("/sys/fs/cgroup/{}", self.cpath);
-            return Some(cg_path);
-        }
-
-        // for cgroup v1
-        self.paths.get(cg).map(|s| s.to_string())
-    }
 }
 
 // get the guest's online cpus.

src/agent/rustjail/src/cgroups/mock.rs

@@ -11,6 +11,7 @@ use anyhow::Result;
 use cgroups::freezer::FreezerState;
 use libc::{self, pid_t};
 use oci::LinuxResources;
+use std::any::Any;
 use std::collections::HashMap;
 use std::string::String;
 
@@ -53,6 +54,18 @@ impl CgroupManager for Manager {
     fn get_pids(&self) -> Result<Vec<pid_t>> {
         Ok(Vec::new())
     }
+
+    fn update_cpuset_path(&self, _: &str, _: &str) -> Result<()> {
+        Ok(())
+    }
+
+    fn get_cgroup_path(&self, _: &str) -> Result<String> {
+        Ok("".to_string())
+    }
+
+    fn as_any(&self) -> Result<&dyn Any> {
+        Ok(self)
+    }
 }
 
 impl Manager {
@@ -63,12 +76,4 @@ impl Manager {
             cpath: cpath.to_string(),
         })
     }
-
-    pub fn update_cpuset_path(&self, _: &str, _: &str) -> Result<()> {
-        Ok(())
-    }
-
-    pub fn get_cg_path(&self, _: &str) -> Option<String> {
-        Some("".to_string())
-    }
 }

src/agent/rustjail/src/cgroups/mod.rs

@@ -4,8 +4,10 @@
 //
 
 use anyhow::{anyhow, Result};
+use core::fmt::Debug;
 use oci::LinuxResources;
 use protocols::agent::CgroupStats;
+use std::any::Any;
 
 use cgroups::freezer::FreezerState;
 
@@ -38,4 +40,22 @@ pub trait Manager {
     fn set(&self, _container: &LinuxResources, _update: bool) -> Result<()> {
         Err(anyhow!("not supported!"))
     }
+
+    fn update_cpuset_path(&self, _: &str, _: &str) -> Result<()> {
+        Err(anyhow!("not supported!"))
+    }
+
+    fn get_cgroup_path(&self, _: &str) -> Result<String> {
+        Err(anyhow!("not supported!"))
+    }
+
+    fn as_any(&self) -> Result<&dyn Any> {
+        Err(anyhow!("not supported!"))
+    }
+}
+
+impl Debug for dyn Manager + Send + Sync {
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+        write!(f, "CgroupManager")
+    }
 }

src/agent/rustjail/src/cgroups/systemd.rs

@@ -1,10 +0,0 @@
-// Copyright (c) 2019 Ant Financial
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-use crate::cgroups::Manager as CgroupManager;
-
-pub struct Manager {}
-
-impl CgroupManager for Manager {}

src/agent/rustjail/src/cgroups/systemd/cgroups_path.rs

@@ -0,0 +1,95 @@
+// Copyright 2021-2022 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use anyhow::{anyhow, Result};
+
+use super::common::{DEFAULT_SLICE, SCOPE_SUFFIX, SLICE_SUFFIX};
+use std::string::String;
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct CgroupsPath {
+    pub slice: String,
+    pub prefix: String,
+    pub name: String,
+}
+
+impl CgroupsPath {
+    pub fn new(cgroups_path_str: &str) -> Result<Self> {
+        let path_vec: Vec<&str> = cgroups_path_str.split(':').collect();
+        if path_vec.len() != 3 {
+            return Err(anyhow!("invalid cpath: {:?}", cgroups_path_str));
+        }
+
+        Ok(CgroupsPath {
+            slice: if path_vec[0].is_empty() {
+                DEFAULT_SLICE.to_string()
+            } else {
+                path_vec[0].to_owned()
+            },
+            prefix: path_vec[1].to_owned(),
+            name: path_vec[2].to_owned(),
+        })
+    }
+
+    // ref: https://github.com/opencontainers/runc/blob/main/docs/systemd.md
+    // return: (parent_slice, unit_name)
+    pub fn parse(&self) -> Result<(String, String)> {
+        Ok((
+            parse_parent(self.slice.to_owned())?,
+            get_unit_name(self.prefix.to_owned(), self.name.to_owned()),
+        ))
+    }
+}
+
+fn parse_parent(slice: String) -> Result<String> {
+    if !slice.ends_with(SLICE_SUFFIX) || slice.contains('/') {
+        return Err(anyhow!("invalid slice name: {}", slice));
+    } else if slice == "-.slice" {
+        return Ok(String::new());
+    }
+
+    let mut slice_path = String::new();
+    let mut prefix = String::new();
+    for subslice in slice.trim_end_matches(SLICE_SUFFIX).split('-') {
+        if subslice.is_empty() {
+            return Err(anyhow!("invalid slice name: {}", slice));
+        }
+        slice_path = format!("{}/{}{}{}", slice_path, prefix, subslice, SLICE_SUFFIX);
+        prefix = format!("{}{}-", prefix, subslice);
+    }
+    slice_path.remove(0);
+    Ok(slice_path)
+}
+
+fn get_unit_name(prefix: String, name: String) -> String {
+    if name.ends_with(SLICE_SUFFIX) {
+        name
+    } else if prefix.is_empty() {
+        format!("{}{}", name, SCOPE_SUFFIX)
+    } else {
+        format!("{}-{}{}", prefix, name, SCOPE_SUFFIX)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::CgroupsPath;
+
+    #[test]
+    fn test_cgroup_path_parse() {
+        let slice = "system.slice";
+        let prefix = "kata_agent";
+        let name = "123";
+        let cgroups_path =
+            CgroupsPath::new(format!("{}:{}:{}", slice, prefix, name).as_str()).unwrap();
+        assert_eq!(slice, cgroups_path.slice.as_str());
+        assert_eq!(prefix, cgroups_path.prefix.as_str());
+        assert_eq!(name, cgroups_path.name.as_str());
+
+        let (parent_slice, unit_name) = cgroups_path.parse().unwrap();
+        assert_eq!(format!("{}", slice), parent_slice);
+        assert_eq!(format!("{}-{}.scope", prefix, name), unit_name);
+    }
+}

src/agent/rustjail/src/cgroups/systemd/common.rs

@@ -0,0 +1,17 @@
+// Copyright 2021-2022 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+pub const DEFAULT_SLICE: &str = "system.slice";
+pub const SLICE_SUFFIX: &str = ".slice";
+pub const SCOPE_SUFFIX: &str = ".scope";
+pub const UNIT_MODE: &str = "replace";
+
+pub type Properties<'a> = Vec<(&'a str, zbus::zvariant::Value<'a>)>;
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub enum CgroupHierarchy {
+    Legacy,
+    Unified,
+}

src/agent/rustjail/src/cgroups/systemd/dbus_client.rs

@@ -0,0 +1,126 @@
+// Copyright 2021-2022 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::vec;
+
+use super::common::CgroupHierarchy;
+use super::common::{Properties, SLICE_SUFFIX, UNIT_MODE};
+use super::interface::system::ManagerProxyBlocking as SystemManager;
+use anyhow::{Context, Result};
+use zbus::zvariant::Value;
+
+pub trait SystemdInterface {
+    fn start_unit(
+        &self,
+        pid: i32,
+        parent: &str,
+        unit_name: &str,
+        cg_hierarchy: &CgroupHierarchy,
+    ) -> Result<()>;
+
+    fn set_properties(&self, unit_name: &str, properties: &Properties) -> Result<()>;
+
+    fn stop_unit(&self, unit_name: &str) -> Result<()>;
+
+    fn get_version(&self) -> Result<String>;
+
+    fn unit_exist(&self, unit_name: &str) -> Result<bool>;
+
+    fn add_process(&self, pid: i32, unit_name: &str) -> Result<()>;
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct DBusClient {}
+
+impl DBusClient {
+    fn build_proxy(&self) -> Result<SystemManager<'static>> {
+        let connection = zbus::blocking::Connection::system()?;
+        let proxy = SystemManager::new(&connection)?;
+        Ok(proxy)
+    }
+}
+
+impl SystemdInterface for DBusClient {
+    fn start_unit(
+        &self,
+        pid: i32,
+        parent: &str,
+        unit_name: &str,
+        cg_hierarchy: &CgroupHierarchy,
+    ) -> Result<()> {
+        let proxy = self.build_proxy()?;
+
+        // enable CPUAccounting & MemoryAccounting & (Block)IOAccounting by default
+        let mut properties: Properties = vec![
+            ("CPUAccounting", Value::Bool(true)),
+            ("DefaultDependencies", Value::Bool(false)),
+            ("MemoryAccounting", Value::Bool(true)),
+            ("TasksAccounting", Value::Bool(true)),
+            ("Description", Value::Str("kata-agent container".into())),
+            ("PIDs", Value::Array(vec![pid as u32].into())),
+        ];
+
+        match *cg_hierarchy {
+            CgroupHierarchy::Legacy => properties.push(("IOAccounting", Value::Bool(true))),
+            CgroupHierarchy::Unified => properties.push(("BlockIOAccounting", Value::Bool(true))),
+        }
+
+        if unit_name.ends_with(SLICE_SUFFIX) {
+            properties.push(("Wants", Value::Str(parent.into())));
+        } else {
+            properties.push(("Slice", Value::Str(parent.into())));
+            properties.push(("Delegate", Value::Bool(true)));
+        }
+
+        proxy
+            .start_transient_unit(unit_name, UNIT_MODE, &properties, &[])
+            .with_context(|| format!("failed to start transient unit {}", unit_name))?;
+        Ok(())
+    }
+
+    fn set_properties(&self, unit_name: &str, properties: &Properties) -> Result<()> {
+        let proxy = self.build_proxy()?;
+
+        proxy
+            .set_unit_properties(unit_name, true, properties)
+            .with_context(|| format!("failed to set unit properties {}", unit_name))?;
+
+        Ok(())
+    }
+
+    fn stop_unit(&self, unit_name: &str) -> Result<()> {
+        let proxy = self.build_proxy()?;
+
+        proxy
+            .stop_unit(unit_name, UNIT_MODE)
+            .with_context(|| format!("failed to stop unit {}", unit_name))?;
+        Ok(())
+    }
+
+    fn get_version(&self) -> Result<String> {
+        let proxy = self.build_proxy()?;
+
+        let systemd_version = proxy
+            .version()
+            .with_context(|| "failed to get systemd version".to_string())?;
+        Ok(systemd_version)
+    }
+
+    fn unit_exist(&self, unit_name: &str) -> Result<bool> {
+        let proxy = self.build_proxy()?;
+
+        Ok(proxy.get_unit(unit_name).is_ok())
+    }
+
+    fn add_process(&self, pid: i32, unit_name: &str) -> Result<()> {
+        let proxy = self.build_proxy()?;
+
+        proxy
+            .attach_processes_to_unit(unit_name, "/", &[pid as u32])
+            .with_context(|| format!("failed to add process {}", unit_name))?;
+
+        Ok(())
+    }
+}

src/agent/rustjail/src/cgroups/systemd/interface/mod.rs

@@ -0,0 +1,7 @@
+// Copyright 2021-2022 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+pub(crate) mod session;
+pub(crate) mod system;

src/agent/rustjail/src/cgroups/systemd/manager.rs

@@ -0,0 +1,129 @@
+// Copyright 2021-2022 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use crate::cgroups::Manager as CgroupManager;
+use crate::protocols::agent::CgroupStats;
+use anyhow::Result;
+use cgroups::freezer::FreezerState;
+use libc::{self, pid_t};
+use oci::LinuxResources;
+use std::any::Any;
+use std::collections::HashMap;
+use std::convert::TryInto;
+use std::string::String;
+use std::vec;
+
+use super::super::fs::Manager as FsManager;
+
+use super::cgroups_path::CgroupsPath;
+use super::common::{CgroupHierarchy, Properties};
+use super::dbus_client::{DBusClient, SystemdInterface};
+use super::subsystem::transformer::Transformer;
+use super::subsystem::{cpu::Cpu, cpuset::CpuSet, memory::Memory, pids::Pids};
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct Manager {
+    pub paths: HashMap<String, String>,
+    pub mounts: HashMap<String, String>,
+    pub cgroups_path: CgroupsPath,
+    pub cpath: String,
+    pub unit_name: String,
+    // dbus client for set properties
+    dbus_client: DBusClient,
+    // fs manager for get properties
+    fs_manager: FsManager,
+    // cgroup version for different dbus properties
+    cg_hierarchy: CgroupHierarchy,
+}
+
+impl CgroupManager for Manager {
+    fn apply(&self, pid: pid_t) -> Result<()> {
+        let unit_name = self.unit_name.as_str();
+        if self.dbus_client.unit_exist(unit_name).unwrap() {
+            self.dbus_client.add_process(pid, self.unit_name.as_str())?;
+        } else {
+            self.dbus_client.start_unit(
+                (pid as u32).try_into().unwrap(),
+                self.cgroups_path.slice.as_str(),
+                self.unit_name.as_str(),
+                &self.cg_hierarchy,
+            )?;
+        }
+
+        Ok(())
+    }
+
+    fn set(&self, r: &LinuxResources, _: bool) -> Result<()> {
+        let mut properties: Properties = vec![];
+
+        let systemd_version = self.dbus_client.get_version()?;
+        let systemd_version_str = systemd_version.as_str();
+
+        Cpu::apply(r, &mut properties, &self.cg_hierarchy, systemd_version_str)?;
+        Memory::apply(r, &mut properties, &self.cg_hierarchy, systemd_version_str)?;
+        Pids::apply(r, &mut properties, &self.cg_hierarchy, systemd_version_str)?;
+        CpuSet::apply(r, &mut properties, &self.cg_hierarchy, systemd_version_str)?;
+
+        self.dbus_client
+            .set_properties(self.unit_name.as_str(), &properties)?;
+
+        Ok(())
+    }
+
+    fn get_stats(&self) -> Result<CgroupStats> {
+        self.fs_manager.get_stats()
+    }
+
+    fn freeze(&self, state: FreezerState) -> Result<()> {
+        self.fs_manager.freeze(state)
+    }
+
+    fn destroy(&mut self) -> Result<()> {
+        self.dbus_client.stop_unit(self.unit_name.as_str())?;
+        self.fs_manager.destroy()
+    }
+
+    fn get_pids(&self) -> Result<Vec<pid_t>> {
+        self.fs_manager.get_pids()
+    }
+
+    fn update_cpuset_path(&self, guest_cpuset: &str, container_cpuset: &str) -> Result<()> {
+        self.fs_manager
+            .update_cpuset_path(guest_cpuset, container_cpuset)
+    }
+
+    fn get_cgroup_path(&self, cg: &str) -> Result<String> {
+        self.fs_manager.get_cgroup_path(cg)
+    }
+
+    fn as_any(&self) -> Result<&dyn Any> {
+        Ok(self)
+    }
+}
+
+impl Manager {
+    pub fn new(cgroups_path_str: &str) -> Result<Self> {
+        let cgroups_path = CgroupsPath::new(cgroups_path_str)?;
+        let (parent_slice, unit_name) = cgroups_path.parse()?;
+        let cpath = parent_slice + "/" + &unit_name;
+
+        let fs_manager = FsManager::new(cpath.as_str())?;
+
+        Ok(Manager {
+            paths: fs_manager.paths.clone(),
+            mounts: fs_manager.mounts.clone(),
+            cgroups_path,
+            cpath,
+            unit_name,
+            dbus_client: DBusClient {},
+            fs_manager,
+            cg_hierarchy: if cgroups::hierarchies::is_cgroup2_unified_mode() {
+                CgroupHierarchy::Unified
+            } else {
+                CgroupHierarchy::Legacy
+            },
+        })
+    }
+}

src/agent/rustjail/src/cgroups/systemd/mod.rs

@@ -0,0 +1,12 @@
+// Copyright 2021-2022 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+pub mod manager;
+
+mod cgroups_path;
+mod common;
+mod dbus_client;
+mod interface;
+mod subsystem;

src/agent/rustjail/src/cgroups/systemd/subsystem/cpu.rs

@@ -0,0 +1,139 @@
+// Copyright 2021-2022 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use super::super::common::{CgroupHierarchy, Properties};
+use super::transformer::Transformer;
+
+use anyhow::Result;
+use oci::{LinuxCpu, LinuxResources};
+use zbus::zvariant::Value;
+
+const BASIC_SYSTEMD_VERSION: &str = "242";
+const DEFAULT_CPUQUOTAPERIOD: u64 = 100 * 1000;
+const SEC2MICROSEC: u64 = 1000 * 1000;
+const BASIC_INTERVAL: u64 = 10 * 1000;
+
+pub struct Cpu {}
+
+impl Transformer for Cpu {
+    fn apply(
+        r: &LinuxResources,
+        properties: &mut Properties,
+        cgroup_hierarchy: &CgroupHierarchy,
+        systemd_version: &str,
+    ) -> Result<()> {
+        if let Some(cpu_resources) = &r.cpu {
+            match cgroup_hierarchy {
+                CgroupHierarchy::Legacy => {
+                    Self::legacy_apply(cpu_resources, properties, systemd_version)?
+                }
+                CgroupHierarchy::Unified => {
+                    Self::unified_apply(cpu_resources, properties, systemd_version)?
+                }
+            }
+        }
+
+        Ok(())
+    }
+}
+
+impl Cpu {
+    // v1:
+    // cpu.shares <-> CPUShares
+    // cpu.period <-> CPUQuotaPeriodUSec
+    // cpu.period & cpu.quota <-> CPUQuotaPerSecUSec
+    fn legacy_apply(
+        cpu_resources: &LinuxCpu,
+        properties: &mut Properties,
+        systemd_version: &str,
+    ) -> Result<()> {
+        if let Some(shares) = cpu_resources.shares {
+            properties.push(("CPUShares", Value::U64(shares)));
+        }
+
+        if let Some(period) = cpu_resources.period {
+            if period != 0 && systemd_version >= BASIC_SYSTEMD_VERSION {
+                properties.push(("CPUQuotaPeriodUSec", Value::U64(period)));
+            }
+        }
+
+        if let Some(quota) = cpu_resources.quota {
+            let period = cpu_resources.period.unwrap_or(DEFAULT_CPUQUOTAPERIOD);
+            if period != 0 {
+                let cpu_quota_per_sec_usec = resolve_cpuquota(quota, period);
+                properties.push(("CPUQuotaPerSecUSec", Value::U64(cpu_quota_per_sec_usec)));
+            }
+        }
+
+        Ok(())
+    }
+
+    // v2:
+    // cpu.shares <-> CPUShares
+    // cpu.period <-> CPUQuotaPeriodUSec
+    // cpu.period & cpu.quota <-> CPUQuotaPerSecUSec
+    fn unified_apply(
+        cpu_resources: &LinuxCpu,
+        properties: &mut Properties,
+        systemd_version: &str,
+    ) -> Result<()> {
+        if let Some(shares) = cpu_resources.shares {
+            let unified_shares = get_unified_cpushares(shares);
+            properties.push(("CPUShares", Value::U64(unified_shares)));
+        }
+
+        if let Some(period) = cpu_resources.period {
+            if period != 0 && systemd_version >= BASIC_SYSTEMD_VERSION {
+                properties.push(("CPUQuotaPeriodUSec", Value::U64(period)));
+            }
+        }
+
+        if let Some(quota) = cpu_resources.quota {
+            let period = cpu_resources.period.unwrap_or(DEFAULT_CPUQUOTAPERIOD);
+            if period != 0 {
+                let cpu_quota_per_sec_usec = resolve_cpuquota(quota, period);
+                properties.push(("CPUQuotaPerSecUSec", Value::U64(cpu_quota_per_sec_usec)));
+            }
+        }
+
+        Ok(())
+    }
+}
+
+// ref: https://github.com/containers/crun/blob/main/crun.1.md#cgroup-v2
+// [2-262144] to [1-10000]
+fn get_unified_cpushares(shares: u64) -> u64 {
+    if shares == 0 {
+        return 100;
+    }
+
+    1 + ((shares - 2) * 9999) / 262142
+}
+
+fn resolve_cpuquota(quota: i64, period: u64) -> u64 {
+    let mut cpu_quota_per_sec_usec = u64::MAX;
+    if quota > 0 {
+        cpu_quota_per_sec_usec = (quota as u64) * SEC2MICROSEC / period;
+        if cpu_quota_per_sec_usec % BASIC_INTERVAL != 0 {
+            cpu_quota_per_sec_usec =
+                ((cpu_quota_per_sec_usec / BASIC_INTERVAL) + 1) * BASIC_INTERVAL;
+        }
+    }
+    cpu_quota_per_sec_usec
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::cgroups::systemd::subsystem::cpu::resolve_cpuquota;
+
+    #[test]
+    fn test_unified_cpuquota() {
+        let quota: i64 = 1000000;
+        let period: u64 = 500000;
+        let cpu_quota_per_sec_usec = resolve_cpuquota(quota, period);
+
+        assert_eq!(2000000, cpu_quota_per_sec_usec);
+    }
+}

src/agent/rustjail/src/cgroups/systemd/subsystem/cpuset.rs

@@ -0,0 +1,124 @@
+// Copyright 2021-2022 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use super::super::common::{CgroupHierarchy, Properties};
+
+use super::transformer::Transformer;
+
+use anyhow::{bail, Result};
+use bit_vec::BitVec;
+use oci::{LinuxCpu, LinuxResources};
+use std::convert::{TryFrom, TryInto};
+use zbus::zvariant::Value;
+
+const BASIC_SYSTEMD_VERSION: &str = "244";
+
+pub struct CpuSet {}
+
+impl Transformer for CpuSet {
+    fn apply(
+        r: &LinuxResources,
+        properties: &mut Properties,
+        _: &CgroupHierarchy,
+        systemd_version: &str,
+    ) -> Result<()> {
+        if let Some(cpuset_resources) = &r.cpu {
+            Self::apply(cpuset_resources, properties, systemd_version)?;
+        }
+
+        Ok(())
+    }
+}
+
+// v1 & v2:
+// cpuset.cpus <-> AllowedCPUs (v244)
+// cpuset.mems <-> AllowedMemoryNodes (v244)
+impl CpuSet {
+    fn apply(
+        cpuset_resources: &LinuxCpu,
+        properties: &mut Properties,
+        systemd_version: &str,
+    ) -> Result<()> {
+        if systemd_version < BASIC_SYSTEMD_VERSION {
+            return Ok(());
+        }
+
+        let cpus = cpuset_resources.cpus.as_str();
+        if !cpus.is_empty() {
+            let cpus_vec: BitMask = cpus.try_into()?;
+            properties.push(("AllowedCPUs", Value::Array(cpus_vec.0.into())));
+        }
+
+        let mems = cpuset_resources.mems.as_str();
+        if !mems.is_empty() {
+            let mems_vec: BitMask = mems.try_into()?;
+            properties.push(("AllowedMemoryNodes", Value::Array(mems_vec.0.into())));
+        }
+
+        Ok(())
+    }
+}
+
+struct BitMask(Vec<u8>);
+
+impl TryFrom<&str> for BitMask {
+    type Error = anyhow::Error;
+
+    fn try_from(bitmask_str: &str) -> Result<Self, Self::Error> {
+        let mut bitmask_vec = BitVec::from_elem(8, false);
+        let bitmask_str_vec: Vec<&str> = bitmask_str.split(',').collect();
+        for bitmask in bitmask_str_vec.iter() {
+            let range: Vec<&str> = bitmask.split('-').collect();
+            match range.len() {
+                1 => {
+                    let idx: usize = range[0].parse()?;
+                    while idx >= bitmask_vec.len() {
+                        bitmask_vec.grow(8, false);
+                    }
+                    bitmask_vec.set(adjust_index(idx), true);
+                }
+                2 => {
+                    let left_index = range[0].parse()?;
+                    let right_index = range[1].parse()?;
+                    while right_index >= bitmask_vec.len() {
+                        bitmask_vec.grow(8, false);
+                    }
+                    for idx in left_index..=right_index {
+                        bitmask_vec.set(adjust_index(idx), true);
+                    }
+                }
+                _ => bail!("invalid bitmask str {}", bitmask_str),
+            }
+        }
+        let mut result_vec = bitmask_vec.to_bytes();
+        result_vec.reverse();
+
+        Ok(BitMask(result_vec))
+    }
+}
+
+#[inline(always)]
+fn adjust_index(idx: usize) -> usize {
+    idx / 8 * 8 + 7 - idx % 8
+}
+
+#[cfg(test)]
+mod tests {
+    use std::convert::TryInto;
+
+    use crate::cgroups::systemd::subsystem::cpuset::BitMask;
+
+    #[test]
+    fn test_bitmask_conversion() {
+        let cpus_vec: BitMask = "2-4".try_into().unwrap();
+        assert_eq!(vec![0b11100 as u8], cpus_vec.0);
+
+        let cpus_vec: BitMask = "1,7".try_into().unwrap();
+        assert_eq!(vec![0b10000010 as u8], cpus_vec.0);
+
+        let cpus_vec: BitMask = "0,2-3,7".try_into().unwrap();
+        assert_eq!(vec![0b10001101 as u8], cpus_vec.0);
+    }
+}

src/agent/rustjail/src/cgroups/systemd/subsystem/memory.rs

@@ -0,0 +1,117 @@
+// Copyright 2021-2022 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use super::super::common::{CgroupHierarchy, Properties};
+
+use super::transformer::Transformer;
+
+use anyhow::{bail, Result};
+use oci::{LinuxMemory, LinuxResources};
+use zbus::zvariant::Value;
+
+pub struct Memory {}
+
+impl Transformer for Memory {
+    fn apply(
+        r: &LinuxResources,
+        properties: &mut Properties,
+        cgroup_hierarchy: &CgroupHierarchy,
+        _: &str,
+    ) -> Result<()> {
+        if let Some(memory_resources) = &r.memory {
+            match cgroup_hierarchy {
+                CgroupHierarchy::Legacy => Self::legacy_apply(memory_resources, properties)?,
+                CgroupHierarchy::Unified => Self::unified_apply(memory_resources, properties)?,
+            }
+        }
+
+        Ok(())
+    }
+}
+
+impl Memory {
+    // v1:
+    // memory.limit <-> MemoryLimit
+    fn legacy_apply(memory_resources: &LinuxMemory, properties: &mut Properties) -> Result<()> {
+        if let Some(limit) = memory_resources.limit {
+            let limit = match limit {
+                1..=i64::MAX => limit as u64,
+                0 => u64::MAX,
+                _ => bail!("invalid memory.limit"),
+            };
+            properties.push(("MemoryLimit", Value::U64(limit)));
+        }
+
+        Ok(())
+    }
+
+    // v2:
+    // memory.low <-> MemoryLow
+    // memory.max <-> MemoryMax
+    // memory.swap & memory.limit <-> MemorySwapMax
+    fn unified_apply(memory_resources: &LinuxMemory, properties: &mut Properties) -> Result<()> {
+        if let Some(limit) = memory_resources.limit {
+            let limit = match limit {
+                1..=i64::MAX => limit as u64,
+                0 => u64::MAX,
+                _ => bail!("invalid memory.limit: {}", limit),
+            };
+            properties.push(("MemoryMax", Value::U64(limit)));
+        }
+
+        if let Some(reservation) = memory_resources.reservation {
+            let reservation = match reservation {
+                1..=i64::MAX => reservation as u64,
+                0 => u64::MAX,
+                _ => bail!("invalid memory.reservation: {}", reservation),
+            };
+            properties.push(("MemoryLow", Value::U64(reservation)));
+        }
+
+        let swap = match memory_resources.swap {
+            Some(0) => u64::MAX,
+            Some(1..=i64::MAX) => match memory_resources.limit {
+                Some(1..=i64::MAX) => {
+                    (memory_resources.limit.unwrap() - memory_resources.swap.unwrap()) as u64
+                }
+                _ => bail!("invalid memory.limit when memory.swap specified"),
+            },
+            None => u64::MAX,
+            _ => bail!("invalid memory.swap"),
+        };
+
+        properties.push(("MemorySwapMax", Value::U64(swap)));
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Memory;
+    use super::Properties;
+    use super::Value;
+
+    #[test]
+    fn test_unified_memory() {
+        let memory_resources = oci::LinuxMemory {
+            limit: Some(736870912),
+            reservation: Some(536870912),
+            swap: Some(536870912),
+            kernel: Some(0),
+            kernel_tcp: Some(0),
+            swappiness: Some(0),
+            disable_oom_killer: Some(false),
+        };
+        let mut properties: Properties = vec![];
+
+        assert_eq!(
+            true,
+            Memory::unified_apply(&memory_resources, &mut properties).is_ok()
+        );
+
+        assert_eq!(Value::U64(200000000), properties[2].1);
+    }
+}

src/agent/rustjail/src/cgroups/systemd/subsystem/mod.rs

@@ -0,0 +1,10 @@
+// Copyright 2021-2022 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+pub mod cpu;
+pub mod cpuset;
+pub mod memory;
+pub mod pids;
+pub mod transformer;

src/agent/rustjail/src/cgroups/systemd/subsystem/pids.rs

@@ -0,0 +1,60 @@
+// Copyright 2021-2022 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use super::super::common::{CgroupHierarchy, Properties};
+
+use super::transformer::Transformer;
+
+use anyhow::Result;
+use oci::{LinuxPids, LinuxResources};
+use zbus::zvariant::Value;
+
+pub struct Pids {}
+
+impl Transformer for Pids {
+    fn apply(
+        r: &LinuxResources,
+        properties: &mut Properties,
+        _: &CgroupHierarchy,
+        _: &str,
+    ) -> Result<()> {
+        if let Some(pids_resources) = &r.pids {
+            Self::apply(pids_resources, properties)?;
+        }
+
+        Ok(())
+    }
+}
+
+// pids.limit <-> TasksMax
+impl Pids {
+    fn apply(pids_resources: &LinuxPids, properties: &mut Properties) -> Result<()> {
+        let limit = if pids_resources.limit > 0 {
+            pids_resources.limit as u64
+        } else {
+            u64::MAX
+        };
+
+        properties.push(("TasksMax", Value::U64(limit)));
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Pids;
+    use super::Properties;
+    use super::Value;
+
+    #[test]
+    fn test_subsystem_workflow() {
+        let pids_resources = oci::LinuxPids { limit: 0 };
+        let mut properties: Properties = vec![];
+
+        assert_eq!(true, Pids::apply(&pids_resources, &mut properties).is_ok());
+
+        assert_eq!(Value::U64(u64::MAX), properties[0].1);
+    }
+}

src/agent/rustjail/src/cgroups/systemd/subsystem/transformer.rs

@@ -0,0 +1,17 @@
+// Copyright 2021-2022 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use super::super::common::{CgroupHierarchy, Properties};
+use anyhow::Result;
+use oci::LinuxResources;
+
+pub trait Transformer {
+    fn apply(
+        r: &LinuxResources,
+        properties: &mut Properties,
+        cgroup_hierarchy: &CgroupHierarchy,
+        systemd_version: &str,
+    ) -> Result<()>;
+}

src/agent/rustjail/src/container.rs

@@ -6,7 +6,7 @@
 use anyhow::{anyhow, Context, Result};
 use libc::pid_t;
 use oci::{ContainerState, LinuxDevice, LinuxIdMapping};
-use oci::{Hook, Linux, LinuxNamespace, LinuxResources, Spec};
+use oci::{Linux, LinuxNamespace, LinuxResources, Spec};
 use std::clone::Clone;
 use std::ffi::CString;
 use std::fmt::Display;
@@ -22,6 +22,7 @@ use crate::capabilities;
 use crate::cgroups::fs::Manager as FsManager;
 #[cfg(test)]
 use crate::cgroups::mock::Manager as FsManager;
+use crate::cgroups::systemd::manager::Manager as SystemdManager;
 use crate::cgroups::Manager;
 #[cfg(feature = "standard-oci-runtime")]
 use crate::console;
@@ -29,6 +30,7 @@ use crate::log_child;
 use crate::process::Process;
 #[cfg(feature = "seccomp")]
 use crate::seccomp;
+use crate::selinux;
 use crate::specconv::CreateOpts;
 use crate::{mount, validator};
 
@@ -49,6 +51,7 @@ use std::os::unix::io::AsRawFd;
 use protobuf::SingularPtrField;
 
 use oci::State as OCIState;
+use regex::Regex;
 use std::collections::HashMap;
 use std::os::unix::io::FromRawFd;
 use std::str::FromStr;
@@ -64,6 +67,9 @@ use rlimit::{setrlimit, Resource, Rlim};
 use tokio::io::AsyncBufReadExt;
 use tokio::sync::Mutex;
 
+use kata_sys_util::hooks::HookStates;
+use kata_sys_util::validate::valid_env;
+
 pub const EXEC_FIFO_FILENAME: &str = "exec.fifo";
 
 const INIT: &str = "INIT";
@@ -107,7 +113,6 @@ impl Default for ContainerStatus {
 }
 
 // We might want to change this to thiserror in the future
-const MissingCGroupManager: &str = "failed to get container's cgroup Manager";
 const MissingLinux: &str = "no linux config";
 const InvalidNamespace: &str = "invalid namespace type";
 
@@ -201,6 +206,8 @@ lazy_static! {
             },
         ]
     };
+
+    pub static ref SYSTEMD_CGROUP_PATH_FORMAT:Regex = Regex::new(r"^[\w\-.]*:[\w\-.]*:[\w\-.]*$").unwrap();
 }
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -239,7 +246,7 @@ pub struct LinuxContainer {
     pub id: String,
     pub root: String,
     pub config: Config,
-    pub cgroup_manager: Option<FsManager>,
+    pub cgroup_manager: Box<dyn Manager + Send + Sync>,
     pub init_process_pid: pid_t,
     pub init_process_start_time: u64,
     pub uid_map_path: String,
@@ -288,16 +295,11 @@ impl Container for LinuxContainer {
             ));
         }
 
-        if self.cgroup_manager.is_some() {
-            self.cgroup_manager
-                .as_ref()
-                .unwrap()
-                .freeze(FreezerState::Frozen)?;
+        self.cgroup_manager.as_ref().freeze(FreezerState::Frozen)?;
 
-            self.status.transition(ContainerState::Paused);
-            return Ok(());
-        }
-        Err(anyhow!(MissingCGroupManager))
+        self.status.transition(ContainerState::Paused);
+
+        Ok(())
     }
 
     fn resume(&mut self) -> Result<()> {
@@ -306,16 +308,11 @@ impl Container for LinuxContainer {
             return Err(anyhow!("container status is: {:?}, not paused", status));
         }
 
-        if self.cgroup_manager.is_some() {
-            self.cgroup_manager
-                .as_ref()
-                .unwrap()
-                .freeze(FreezerState::Thawed)?;
+        self.cgroup_manager.as_ref().freeze(FreezerState::Thawed)?;
 
-            self.status.transition(ContainerState::Running);
-            return Ok(());
-        }
-        Err(anyhow!(MissingCGroupManager))
+        self.status.transition(ContainerState::Running);
+
+        Ok(())
     }
 }
 
@@ -390,7 +387,9 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     let buf = read_sync(crfd)?;
     let cm_str = std::str::from_utf8(&buf)?;
 
-    let cm: FsManager = serde_json::from_str(cm_str)?;
+    // deserialize cm_str into FsManager and SystemdManager separately
+    let fs_cm: Result<FsManager, serde_json::Error> = serde_json::from_str(cm_str);
+    let systemd_cm: Result<SystemdManager, serde_json::Error> = serde_json::from_str(cm_str);
 
     #[cfg(feature = "standard-oci-runtime")]
     let csocket_fd = console::setup_console_socket(&std::env::var(CONSOLE_SOCKET_FD)?)?;
@@ -531,6 +530,8 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         }
     }
 
+    let selinux_enabled = selinux::is_enabled()?;
+
     sched::unshare(to_new & !CloneFlags::CLONE_NEWUSER)?;
 
     if userns {
@@ -548,7 +549,18 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
 
     if to_new.contains(CloneFlags::CLONE_NEWNS) {
         // setup rootfs
-        mount::init_rootfs(cfd_log, &spec, &cm.paths, &cm.mounts, bind_device)?;
+        if let Ok(systemd_cm) = systemd_cm {
+            mount::init_rootfs(
+                cfd_log,
+                &spec,
+                &systemd_cm.paths,
+                &systemd_cm.mounts,
+                bind_device,
+            )?;
+        } else {
+            let fs_cm = fs_cm.unwrap();
+            mount::init_rootfs(cfd_log, &spec, &fs_cm.paths, &fs_cm.mounts, bind_device)?;
+        }
     }
 
     if init {
@@ -621,6 +633,18 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         capctl::prctl::set_no_new_privs().map_err(|_| anyhow!("cannot set no new privileges"))?;
     }
 
+    // Set SELinux label
+    if !oci_process.selinux_label.is_empty() {
+        if !selinux_enabled {
+            return Err(anyhow!(
+                "SELinux label for the process is provided but SELinux is not enabled on the running kernel"
+            ));
+        }
+
+        log_child!(cfd_log, "Set SELinux label to the container process");
+        selinux::set_exec_label(&oci_process.selinux_label)?;
+    }
+
     // Log unknown seccomp system calls in advance before the log file descriptor closes.
     #[cfg(feature = "seccomp")]
     if let Some(ref scmp) = linux.seccomp {
@@ -830,22 +854,17 @@ impl BaseContainer for LinuxContainer {
     }
 
     fn stats(&self) -> Result<StatsContainerResponse> {
-        let mut r = StatsContainerResponse::default();
-
-        if self.cgroup_manager.is_some() {
-            r.cgroup_stats =
-                SingularPtrField::some(self.cgroup_manager.as_ref().unwrap().get_stats()?);
-        }
-
         // what about network interface stats?
 
-        Ok(r)
+        Ok(StatsContainerResponse {
+            cgroup_stats: SingularPtrField::some(self.cgroup_manager.as_ref().get_stats()?),
+            ..Default::default()
+        })
     }
 
     fn set(&mut self, r: LinuxResources) -> Result<()> {
-        if self.cgroup_manager.is_some() {
-            self.cgroup_manager.as_ref().unwrap().set(&r, true)?;
-        }
+        self.cgroup_manager.as_ref().set(&r, true)?;
+
         self.config
             .spec
             .as_mut()
@@ -1018,7 +1037,8 @@ impl BaseContainer for LinuxContainer {
             &logger,
             spec,
             &p,
-            self.cgroup_manager.as_ref().unwrap(),
+            self.cgroup_manager.as_ref(),
+            self.config.use_systemd_cgroup,
             &st,
             &mut pipe_w,
             &mut pipe_r,
@@ -1081,12 +1101,14 @@ impl BaseContainer for LinuxContainer {
             }
         }
 
-        if spec.hooks.is_some() {
-            info!(self.logger, "poststop");
-            let hooks = spec.hooks.as_ref().unwrap();
-            for h in hooks.poststop.iter() {
-                execute_hook(&self.logger, h, &st).await?;
-            }
+        // guest Poststop hook
+        // * should be executed after the container is deleted but before the delete operation returns
+        // * the executable file is in agent namespace
+        // * should also be executed in agent namespace.
+        if let Some(hooks) = spec.hooks.as_ref() {
+            info!(self.logger, "guest Poststop hook");
+            let mut hook_states = HookStates::new();
+            hook_states.execute_hooks(&hooks.poststop, Some(st))?;
         }
 
         self.status.transition(ContainerState::Stopped);
@@ -1096,19 +1118,19 @@ impl BaseContainer for LinuxContainer {
         )?;
         fs::remove_dir_all(&self.root)?;
 
-        if let Some(cgm) = self.cgroup_manager.as_mut() {
-            // Kill all of the processes created in this container to prevent
-            // the leak of some daemon process when this container shared pidns
-            // with the sandbox.
-            let pids = cgm.get_pids().context("get cgroup pids")?;
-            for i in pids {
-                if let Err(e) = signal::kill(Pid::from_raw(i), Signal::SIGKILL) {
-                    warn!(self.logger, "kill the process {} error: {:?}", i, e);
-                }
+        let cgm = self.cgroup_manager.as_mut();
+        // Kill all of the processes created in this container to prevent
+        // the leak of some daemon process when this container shared pidns
+        // with the sandbox.
+        let pids = cgm.get_pids().context("get cgroup pids")?;
+        for i in pids {
+            if let Err(e) = signal::kill(Pid::from_raw(i), Signal::SIGKILL) {
+                warn!(self.logger, "kill the process {} error: {:?}", i, e);
             }
-
-            cgm.destroy().context("destroy cgroups")?;
         }
+
+        cgm.destroy().context("destroy cgroups")?;
+
         Ok(())
     }
 
@@ -1132,16 +1154,14 @@ impl BaseContainer for LinuxContainer {
             .ok_or_else(|| anyhow!("OCI spec was not found"))?;
         let st = self.oci_state()?;
 
-        // run poststart hook
-        if spec.hooks.is_some() {
-            info!(self.logger, "poststart hook");
-            let hooks = spec
-                .hooks
-                .as_ref()
-                .ok_or_else(|| anyhow!("OCI hooks were not found"))?;
-            for h in hooks.poststart.iter() {
-                execute_hook(&self.logger, h, &st).await?;
-            }
+        // guest Poststart hook
+        // * should be executed after the container is started but before the delete operation returns
+        // * the executable file is in agent namespace
+        // * should also be executed in agent namespace.
+        if let Some(hooks) = spec.hooks.as_ref() {
+            info!(self.logger, "guest Poststart hook");
+            let mut hook_states = HookStates::new();
+            hook_states.execute_hooks(&hooks.poststart, Some(st))?;
         }
 
         unistd::close(fd)?;
@@ -1280,11 +1300,13 @@ pub fn setup_child_logger(fd: RawFd, child_logger: Logger) -> tokio::task::JoinH
     })
 }
 
+#[allow(clippy::too_many_arguments)]
 async fn join_namespaces(
     logger: &Logger,
     spec: &Spec,
     p: &Process,
-    cm: &FsManager,
+    cm: &(dyn Manager + Send + Sync),
+    use_systemd_cgroup: bool,
     st: &OCIState,
     pipe_w: &mut PipeStream,
     pipe_r: &mut PipeStream,
@@ -1311,7 +1333,11 @@ async fn join_namespaces(
     info!(logger, "wait child received oci process");
     read_async(pipe_r).await?;
 
-    let cm_str = serde_json::to_string(cm)?;
+    let cm_str = if use_systemd_cgroup {
+        serde_json::to_string(cm.as_any()?.downcast_ref::<SystemdManager>().unwrap())
+    } else {
+        serde_json::to_string(cm.as_any()?.downcast_ref::<FsManager>().unwrap())
+    }?;
     write_async(pipe_w, SYNC_DATA, cm_str.as_str()).await?;
 
     // wait child setup user namespace
@@ -1334,13 +1360,16 @@ async fn join_namespaces(
     }
 
     // apply cgroups
-    if p.init && res.is_some() {
-        info!(logger, "apply cgroups!");
-        cm.set(res.unwrap(), false)?;
+    // For FsManger, it's no matter about the order of apply and set.
+    // For SystemdManger, apply must be precede set because we can only create a systemd unit with specific processes(pids).
+    if res.is_some() {
+        info!(logger, "apply processes to cgroups!");
+        cm.apply(p.pid)?;
     }
 
-    if res.is_some() {
-        cm.apply(p.pid)?;
+    if p.init && res.is_some() {
+        info!(logger, "set properties to cgroups!");
+        cm.set(res.unwrap(), false)?;
     }
 
     info!(logger, "notify child to continue");
@@ -1353,13 +1382,14 @@ async fn join_namespaces(
 
         info!(logger, "get ready to run prestart hook!");
 
-        // run prestart hook
-        if spec.hooks.is_some() {
-            info!(logger, "prestart hook");
-            let hooks = spec.hooks.as_ref().unwrap();
-            for h in hooks.prestart.iter() {
-                execute_hook(&logger, h, st).await?;
-            }
+        // guest Prestart hook
+        // * should be executed during the start operation, and before the container command is executed
+        // * the executable file is in agent namespace
+        // * should also be executed in agent namespace.
+        if let Some(hooks) = spec.hooks.as_ref() {
+            info!(logger, "guest Prestart hook");
+            let mut hook_states = HookStates::new();
+            hook_states.execute_hooks(&hooks.prestart, Some(st.clone()))?;
         }
 
         // notify child run prestart hooks completed
@@ -1419,7 +1449,7 @@ impl LinuxContainer {
     pub fn new<T: Into<String> + Display + Clone>(
         id: T,
         base: T,
-        config: Config,
+        mut config: Config,
         logger: &Logger,
     ) -> Result<Self> {
         let base = base.into();
@@ -1448,24 +1478,46 @@ impl LinuxContainer {
 
         let linux = spec.linux.as_ref().unwrap();
 
-        let cpath = if linux.cgroups_path.is_empty() {
-            format!("/{}", id.as_str())
+        // determine which cgroup driver to take and then assign to config.use_systemd_cgroup
+        // systemd: "[slice]:[prefix]:[name]"
+        // fs: "/path_a/path_b"
+        let cpath = if SYSTEMD_CGROUP_PATH_FORMAT.is_match(linux.cgroups_path.as_str()) {
+            config.use_systemd_cgroup = true;
+            if linux.cgroups_path.len() == 2 {
+                format!("system.slice:kata_agent:{}", id.as_str())
+            } else {
+                linux.cgroups_path.clone()
+            }
         } else {
-            linux.cgroups_path.clone()
+            config.use_systemd_cgroup = false;
+            if linux.cgroups_path.is_empty() {
+                format!("/{}", id.as_str())
+            } else {
+                linux.cgroups_path.clone()
+            }
         };
 
-        let cgroup_manager = FsManager::new(cpath.as_str()).map_err(|e| {
-            anyhow!(format!(
-                "fail to create cgroup manager with path {}: {:}",
-                cpath, e
-            ))
-        })?;
+        let cgroup_manager: Box<dyn Manager + Send + Sync> = if config.use_systemd_cgroup {
+            Box::new(SystemdManager::new(cpath.as_str()).map_err(|e| {
+                anyhow!(format!(
+                    "fail to create cgroup manager with path {}: {:}",
+                    cpath, e
+                ))
+            })?)
+        } else {
+            Box::new(FsManager::new(cpath.as_str()).map_err(|e| {
+                anyhow!(format!(
+                    "fail to create cgroup manager with path {}: {:}",
+                    cpath, e
+                ))
+            })?)
+        };
         info!(logger, "new cgroup_manager {:?}", &cgroup_manager);
 
         Ok(LinuxContainer {
             id: id.clone(),
             root,
-            cgroup_manager: Some(cgroup_manager),
+            cgroup_manager,
             status: ContainerStatus::new(),
             uid_map_path: String::from(""),
             gid_map_path: "".to_string(),
@@ -1517,143 +1569,6 @@ fn set_sysctls(sysctls: &HashMap<String, String>) -> Result<()> {
     Ok(())
 }
 
-use std::process::Stdio;
-use std::time::Duration;
-use tokio::io::{AsyncReadExt, AsyncWriteExt};
-
-pub async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
-    let logger = logger.new(o!("action" => "execute-hook"));
-
-    let binary = PathBuf::from(h.path.as_str());
-    let path = binary.canonicalize()?;
-    if !path.exists() {
-        return Err(anyhow!("Path {:?} does not exist", path));
-    }
-
-    let mut args = h.args.clone();
-    // the hook.args[0] is the hook binary name which shouldn't be included
-    // in the Command.args
-    if args.len() > 1 {
-        args.remove(0);
-    }
-
-    // all invalid envs will be omitted, only valid envs will be passed to hook.
-    let env: HashMap<&str, &str> = h.env.iter().filter_map(|e| valid_env(e)).collect();
-
-    // Avoid the exit signal to be reaped by the global reaper.
-    let _wait_locker = WAIT_PID_LOCKER.lock().await;
-    let mut child = tokio::process::Command::new(path)
-        .args(args.iter())
-        .envs(env.iter())
-        .kill_on_drop(true)
-        .stdin(Stdio::piped())
-        .stdout(Stdio::piped())
-        .stderr(Stdio::piped())
-        .spawn()?;
-
-    // default timeout 10s
-    let mut timeout: u64 = 10;
-
-    // if timeout is set if hook, then use the specified value
-    if let Some(t) = h.timeout {
-        if t > 0 {
-            timeout = t as u64;
-        }
-    }
-
-    let state = serde_json::to_string(st)?;
-    let path = h.path.clone();
-
-    let join_handle = tokio::spawn(async move {
-        if let Some(mut stdin) = child.stdin.take() {
-            match stdin.write_all(state.as_bytes()).await {
-                Ok(_) => {}
-                Err(e) => {
-                    info!(logger, "write to child stdin failed: {:?}", e);
-                }
-            }
-        }
-
-        // read something from stdout and stderr for debug
-        if let Some(stdout) = child.stdout.as_mut() {
-            let mut out = String::new();
-            match stdout.read_to_string(&mut out).await {
-                Ok(_) => {
-                    info!(logger, "child stdout: {}", out.as_str());
-                }
-                Err(e) => {
-                    info!(logger, "read from child stdout failed: {:?}", e);
-                }
-            }
-        }
-
-        let mut err = String::new();
-        if let Some(stderr) = child.stderr.as_mut() {
-            match stderr.read_to_string(&mut err).await {
-                Ok(_) => {
-                    info!(logger, "child stderr: {}", err.as_str());
-                }
-                Err(e) => {
-                    info!(logger, "read from child stderr failed: {:?}", e);
-                }
-            }
-        }
-
-        match child.wait().await {
-            Ok(exit) => {
-                let code = exit
-                    .code()
-                    .ok_or_else(|| anyhow!("hook exit status has no status code"))?;
-
-                if code != 0 {
-                    error!(
-                        logger,
-                        "hook {} exit status is {}, error message is {}", &path, code, err
-                    );
-                    return Err(anyhow!(nix::Error::UnknownErrno));
-                }
-
-                debug!(logger, "hook {} exit status is 0", &path);
-                Ok(())
-            }
-            Err(e) => Err(anyhow!(
-                "wait child error: {} {}",
-                e,
-                e.raw_os_error().unwrap()
-            )),
-        }
-    });
-
-    match tokio::time::timeout(Duration::new(timeout, 0), join_handle).await {
-        Ok(r) => r.unwrap(),
-        Err(_) => Err(anyhow!(nix::Error::ETIMEDOUT)),
-    }
-}
-
-// valid environment variables according to https://doc.rust-lang.org/std/env/fn.set_var.html#panics
-fn valid_env(e: &str) -> Option<(&str, &str)> {
-    // wherther key or value will contain NULL char.
-    if e.as_bytes().contains(&b'\0') {
-        return None;
-    }
-
-    let v: Vec<&str> = e.splitn(2, '=').collect();
-
-    // key can't hold an `equal` sign, but value can
-    if v.len() != 2 {
-        return None;
-    }
-
-    let (key, value) = (v[0].trim(), v[1].trim());
-
-    // key can't be empty
-    if key.is_empty() {
-        return None;
-    }
-
-    Some((key, value))
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1664,7 +1579,6 @@ mod tests {
     use std::os::unix::io::AsRawFd;
     use tempfile::tempdir;
     use test_utils::skip_if_not_root;
-    use tokio::process::Command;
 
     macro_rules! sl {
         () => {
@@ -1672,113 +1586,6 @@ mod tests {
         };
     }
 
-    async fn which(cmd: &str) -> String {
-        let output: std::process::Output = Command::new("which")
-            .arg(cmd)
-            .output()
-            .await
-            .expect("which command failed to run");
-
-        match String::from_utf8(output.stdout) {
-            Ok(v) => v.trim_end_matches('\n').to_string(),
-            Err(e) => panic!("Invalid UTF-8 sequence: {}", e),
-        }
-    }
-
-    #[tokio::test]
-    async fn test_execute_hook() {
-        let temp_file = "/tmp/test_execute_hook";
-
-        let touch = which("touch").await;
-
-        defer!(fs::remove_file(temp_file).unwrap(););
-        let invalid_str = vec![97, b'\0', 98];
-        let invalid_string = std::str::from_utf8(&invalid_str).unwrap();
-        let invalid_env = format!("{}=value", invalid_string);
-
-        execute_hook(
-            &slog_scope::logger(),
-            &Hook {
-                path: touch,
-                args: vec!["touch".to_string(), temp_file.to_string()],
-                env: vec![invalid_env],
-                timeout: Some(10),
-            },
-            &OCIState {
-                version: "1.2.3".to_string(),
-                id: "321".to_string(),
-                status: ContainerState::Running,
-                pid: 2,
-                bundle: "".to_string(),
-                annotations: Default::default(),
-            },
-        )
-        .await
-        .unwrap();
-
-        assert_eq!(Path::new(&temp_file).exists(), true);
-    }
-
-    #[tokio::test]
-    async fn test_execute_hook_with_error() {
-        let ls = which("ls").await;
-
-        let res = execute_hook(
-            &slog_scope::logger(),
-            &Hook {
-                path: ls,
-                args: vec!["ls".to_string(), "/tmp/not-exist".to_string()],
-                env: vec![],
-                timeout: None,
-            },
-            &OCIState {
-                version: "1.2.3".to_string(),
-                id: "321".to_string(),
-                status: ContainerState::Running,
-                pid: 2,
-                bundle: "".to_string(),
-                annotations: Default::default(),
-            },
-        )
-        .await;
-
-        let expected_err = nix::Error::UnknownErrno;
-        assert_eq!(
-            res.unwrap_err().downcast::<nix::Error>().unwrap(),
-            expected_err
-        );
-    }
-
-    #[tokio::test]
-    async fn test_execute_hook_with_timeout() {
-        let sleep = which("sleep").await;
-
-        let res = execute_hook(
-            &slog_scope::logger(),
-            &Hook {
-                path: sleep,
-                args: vec!["sleep".to_string(), "2".to_string()],
-                env: vec![],
-                timeout: Some(1),
-            },
-            &OCIState {
-                version: "1.2.3".to_string(),
-                id: "321".to_string(),
-                status: ContainerState::Running,
-                pid: 2,
-                bundle: "".to_string(),
-                annotations: Default::default(),
-            },
-        )
-        .await;
-
-        let expected_err = nix::Error::ETIMEDOUT;
-        assert_eq!(
-            res.unwrap_err().downcast::<nix::Error>().unwrap(),
-            expected_err
-        );
-    }
-
     #[test]
     fn test_status_transtition() {
         let mut status = ContainerStatus::new();
@@ -1931,20 +1738,12 @@ mod tests {
         assert!(format!("{:?}", ret).contains("failed to pause container"))
     }
 
-    #[test]
-    fn test_linuxcontainer_pause_cgroupmgr_is_none() {
-        let ret = new_linux_container_and_then(|mut c: LinuxContainer| {
-            c.cgroup_manager = None;
-            c.pause().map_err(|e| anyhow!(e))
-        });
-
-        assert!(ret.is_err(), "Expecting error, Got {:?}", ret);
-    }
-
     #[test]
     fn test_linuxcontainer_pause() {
         let ret = new_linux_container_and_then(|mut c: LinuxContainer| {
-            c.cgroup_manager = FsManager::new("").ok();
+            c.cgroup_manager = Box::new(FsManager::new("").map_err(|e| {
+                anyhow!(format!("fail to create cgroup manager with path: {:}", e))
+            })?);
             c.pause().map_err(|e| anyhow!(e))
         });
 
@@ -1963,21 +1762,12 @@ mod tests {
         assert!(format!("{:?}", ret).contains("not paused"))
     }
 
-    #[test]
-    fn test_linuxcontainer_resume_cgroupmgr_is_none() {
-        let ret = new_linux_container_and_then(|mut c: LinuxContainer| {
-            c.status.transition(ContainerState::Paused);
-            c.cgroup_manager = None;
-            c.resume().map_err(|e| anyhow!(e))
-        });
-
-        assert!(ret.is_err(), "Expecting error, Got {:?}", ret);
-    }
-
     #[test]
     fn test_linuxcontainer_resume() {
         let ret = new_linux_container_and_then(|mut c: LinuxContainer| {
-            c.cgroup_manager = FsManager::new("").ok();
+            c.cgroup_manager = Box::new(FsManager::new("").map_err(|e| {
+                anyhow!(format!("fail to create cgroup manager with path: {:}", e))
+            })?);
             // Change status to paused, this way we can resume it
             c.status.transition(ContainerState::Paused);
             c.resume().map_err(|e| anyhow!(e))
@@ -2110,49 +1900,4 @@ mod tests {
         let ret = do_init_child(std::io::stdin().as_raw_fd());
         assert!(ret.is_err(), "Expecting Err, Got {:?}", ret);
     }
-
-    #[test]
-    fn test_valid_env() {
-        let env = valid_env("a=b=c");
-        assert_eq!(Some(("a", "b=c")), env);
-
-        let env = valid_env("a=b");
-        assert_eq!(Some(("a", "b")), env);
-        let env = valid_env("a =b");
-        assert_eq!(Some(("a", "b")), env);
-
-        let env = valid_env(" a =b");
-        assert_eq!(Some(("a", "b")), env);
-
-        let env = valid_env("a= b");
-        assert_eq!(Some(("a", "b")), env);
-
-        let env = valid_env("a=b ");
-        assert_eq!(Some(("a", "b")), env);
-        let env = valid_env("a=b c ");
-        assert_eq!(Some(("a", "b c")), env);
-
-        let env = valid_env("=b");
-        assert_eq!(None, env);
-
-        let env = valid_env("a=");
-        assert_eq!(Some(("a", "")), env);
-
-        let env = valid_env("a==");
-        assert_eq!(Some(("a", "=")), env);
-
-        let env = valid_env("a");
-        assert_eq!(None, env);
-
-        let invalid_str = vec![97, b'\0', 98];
-        let invalid_string = std::str::from_utf8(&invalid_str).unwrap();
-
-        let invalid_env = format!("{}=value", invalid_string);
-        let env = valid_env(&invalid_env);
-        assert_eq!(None, env);
-
-        let invalid_env = format!("key={}", invalid_string);
-        let env = valid_env(&invalid_env);
-        assert_eq!(None, env);
-    }
 }

src/agent/rustjail/src/lib.rs

@@ -38,6 +38,7 @@ pub mod pipestream;
 pub mod process;
 #[cfg(feature = "seccomp")]
 pub mod seccomp;
+pub mod selinux;
 pub mod specconv;
 pub mod sync;
 pub mod sync_with_async;
@@ -236,6 +237,12 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
     let devices = {
         let mut d = Vec::new();
         for dev in res.Devices.iter() {
+            let dev_type = if dev.Type.is_empty() {
+                None
+            } else {
+                Some(dev.Type.clone())
+            };
+
             let major = if dev.Major == -1 {
                 None
             } else {
@@ -249,7 +256,7 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
             };
             d.push(oci::LinuxDeviceCgroup {
                 allow: dev.Allow,
-                r#type: dev.Type.clone(),
+                r#type: dev_type,
                 major,
                 minor,
                 access: dev.Access.clone(),

src/agent/rustjail/src/mount.rs

@@ -25,6 +25,7 @@ use std::fs::File;
 use std::io::{BufRead, BufReader};
 
 use crate::container::DEFAULT_DEVICES;
+use crate::selinux;
 use crate::sync::write_count;
 use std::string::ToString;
 
@@ -181,6 +182,8 @@ pub fn init_rootfs(
         None => flags |= MsFlags::MS_SLAVE,
     }
 
+    let label = &linux.mount_label;
+
     let root = spec
         .root
         .as_ref()
@@ -244,7 +247,7 @@ pub fn init_rootfs(
                 }
             }
 
-            mount_from(cfd_log, m, rootfs, flags, &data, "")?;
+            mount_from(cfd_log, m, rootfs, flags, &data, label)?;
             // bind mount won't change mount options, we need remount to make mount options
             // effective.
             // first check that we have non-default options required before attempting a
@@ -524,7 +527,6 @@ pub fn pivot_rootfs<P: ?Sized + NixPath + std::fmt::Debug>(path: &P) -> Result<(
 
 fn rootfs_parent_mount_private(path: &str) -> Result<()> {
     let mount_infos = parse_mount_table(MOUNTINFO_PATH)?;
-
     let mut max_len = 0;
     let mut mount_point = String::from("");
     let mut options = String::from("");
@@ -767,9 +769,9 @@ fn mount_from(
     rootfs: &str,
     flags: MsFlags,
     data: &str,
-    _label: &str,
+    label: &str,
 ) -> Result<()> {
-    let d = String::from(data);
+    let mut d = String::from(data);
     let dest = secure_join(rootfs, &m.destination);
 
     let src = if m.r#type.as_str() == "bind" {
@@ -780,7 +782,7 @@ fn mount_from(
             Path::new(&dest).parent().unwrap()
         };
 
-        fs::create_dir_all(&dir).map_err(|e| {
+        fs::create_dir_all(dir).map_err(|e| {
             log_child!(
                 cfd_log,
                 "create dir {}: {}",
@@ -822,6 +824,37 @@ fn mount_from(
         e
     })?;
 
+    // Set the SELinux context for the mounts
+    let mut use_xattr = false;
+    if !label.is_empty() {
+        if selinux::is_enabled()? {
+            let device = Path::new(&m.source)
+                .file_name()
+                .ok_or_else(|| anyhow!("invalid device source path: {}", &m.source))?
+                .to_str()
+                .ok_or_else(|| anyhow!("failed to convert device source path: {}", &m.source))?;
+
+            match device {
+                // SELinux does not support labeling of /proc or /sys
+                "proc" | "sysfs" => (),
+                // SELinux does not support mount labeling against /dev/mqueue,
+                // so we use setxattr instead
+                "mqueue" => {
+                    use_xattr = true;
+                }
+                _ => {
+                    log_child!(cfd_log, "add SELinux mount label to {}", dest.as_str());
+                    selinux::add_mount_label(&mut d, label);
+                }
+            }
+        } else {
+            log_child!(
+                cfd_log,
+                "SELinux label for the mount is provided but SELinux is not enabled on the running kernel"
+            );
+        }
+    }
+
     mount(
         Some(src.as_str()),
         dest.as_str(),
@@ -834,6 +867,10 @@ fn mount_from(
         e
     })?;
 
+    if !label.is_empty() && selinux::is_enabled()? && use_xattr {
+        xattr::set(dest.as_str(), "security.selinux", label.as_bytes())?;
+    }
+
     if flags.contains(MsFlags::MS_BIND)
         && flags.intersects(
             !(MsFlags::MS_REC

src/agent/rustjail/src/seccomp.rs

@@ -63,7 +63,7 @@ pub fn get_unknown_syscalls(scmp: &LinuxSeccomp) -> Option<Vec<String>> {
 // init_seccomp creates a seccomp filter and loads it for the current process
 // including all the child processes.
 pub fn init_seccomp(scmp: &LinuxSeccomp) -> Result<()> {
-    let def_action = ScmpAction::from_str(scmp.default_action.as_str(), Some(libc::EPERM as i32))?;
+    let def_action = ScmpAction::from_str(scmp.default_action.as_str(), Some(libc::EPERM))?;
 
     // Create a new filter context
     let mut filter = ScmpFilterContext::new_filter(def_action)?;

src/agent/rustjail/src/selinux.rs

@@ -0,0 +1,80 @@
+// Copyright 2022 Sony Group Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use anyhow::{Context, Result};
+use nix::unistd::gettid;
+use std::fs::{self, OpenOptions};
+use std::io::prelude::*;
+use std::path::Path;
+
+pub fn is_enabled() -> Result<bool> {
+    let buf = fs::read_to_string("/proc/mounts")?;
+    let enabled = buf.contains("selinuxfs");
+
+    Ok(enabled)
+}
+
+pub fn add_mount_label(data: &mut String, label: &str) {
+    if data.is_empty() {
+        let context = format!("context=\"{}\"", label);
+        data.push_str(&context);
+    } else {
+        let context = format!(",context=\"{}\"", label);
+        data.push_str(&context);
+    }
+}
+
+pub fn set_exec_label(label: &str) -> Result<()> {
+    let mut attr_path = Path::new("/proc/thread-self/attr/exec").to_path_buf();
+    if !attr_path.exists() {
+        // Fall back to the old convention
+        attr_path = Path::new("/proc/self/task")
+            .join(gettid().to_string())
+            .join("attr/exec")
+    }
+
+    let mut file = OpenOptions::new()
+        .write(true)
+        .truncate(true)
+        .open(attr_path)?;
+    file.write_all(label.as_bytes())
+        .with_context(|| "failed to apply SELinux label")?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const TEST_LABEL: &str = "system_u:system_r:unconfined_t:s0";
+
+    #[test]
+    fn test_is_enabled() {
+        let ret = is_enabled();
+        assert!(ret.is_ok(), "Expecting Ok, Got {:?}", ret);
+    }
+
+    #[test]
+    fn test_add_mount_label() {
+        let mut data = String::new();
+        add_mount_label(&mut data, TEST_LABEL);
+        assert_eq!(data, format!("context=\"{}\"", TEST_LABEL));
+
+        let mut data = String::from("defaults");
+        add_mount_label(&mut data, TEST_LABEL);
+        assert_eq!(data, format!("defaults,context=\"{}\"", TEST_LABEL));
+    }
+
+    #[test]
+    fn test_set_exec_label() {
+        let ret = set_exec_label(TEST_LABEL);
+        if is_enabled().unwrap() {
+            assert!(ret.is_ok(), "Expecting Ok, Got {:?}", ret);
+        } else {
+            assert!(ret.is_err(), "Expecting error, Got {:?}", ret);
+        }
+    }
+}

src/agent/rustjail/src/validator.rs

@@ -6,6 +6,7 @@
 use crate::container::Config;
 use anyhow::{anyhow, Context, Result};
 use oci::{Linux, LinuxIdMapping, LinuxNamespace, Spec};
+use regex::Regex;
 use std::collections::HashMap;
 use std::path::{Component, PathBuf};
 
@@ -86,6 +87,23 @@ fn hostname(oci: &Spec) -> Result<()> {
 
 fn security(oci: &Spec) -> Result<()> {
     let linux = get_linux(oci)?;
+    let label_pattern = r".*_u:.*_r:.*_t:s[0-9]|1[0-5].*";
+    let label_regex = Regex::new(label_pattern)?;
+
+    if let Some(ref process) = oci.process {
+        if !process.selinux_label.is_empty() && !label_regex.is_match(&process.selinux_label) {
+            return Err(anyhow!(
+                "SELinux label for the process is invalid format: {}",
+                &process.selinux_label
+            ));
+        }
+    }
+    if !linux.mount_label.is_empty() && !label_regex.is_match(&linux.mount_label) {
+        return Err(anyhow!(
+            "SELinux label for the mount is invalid format: {}",
+            &linux.mount_label
+        ));
+    }
 
     if linux.masked_paths.is_empty() && linux.readonly_paths.is_empty() {
         return Ok(());
@@ -95,8 +113,6 @@ fn security(oci: &Spec) -> Result<()> {
         return Err(anyhow!("Linux namespace does not contain mount"));
     }
 
-    // don't care about selinux at present
-
     Ok(())
 }
 
@@ -285,7 +301,7 @@ pub fn validate(conf: &Config) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use oci::Mount;
+    use oci::{Mount, Process};
 
     #[test]
     fn test_namespace() {
@@ -388,6 +404,29 @@ mod tests {
         ];
         spec.linux = Some(linux);
         security(&spec).unwrap();
+
+        // SELinux
+        let valid_label = "system_u:system_r:container_t:s0:c123,c456";
+        let mut process = Process::default();
+        process.selinux_label = valid_label.to_string();
+        spec.process = Some(process);
+        security(&spec).unwrap();
+
+        let mut linux = Linux::default();
+        linux.mount_label = valid_label.to_string();
+        spec.linux = Some(linux);
+        security(&spec).unwrap();
+
+        let invalid_label = "system_u:system_r:container_t";
+        let mut process = Process::default();
+        process.selinux_label = invalid_label.to_string();
+        spec.process = Some(process);
+        security(&spec).unwrap_err();
+
+        let mut linux = Linux::default();
+        linux.mount_label = invalid_label.to_string();
+        spec.linux = Some(linux);
+        security(&spec).unwrap_err();
     }
 
     #[test]

src/agent/src/config.rs

@@ -11,6 +11,7 @@ use std::fs;
 use std::str::FromStr;
 use std::time;
 use tracing::instrument;
+use url::Url;
 
 use kata_types::config::default::DEFAULT_AGENT_VSOCK_PORT;
 
@@ -25,6 +26,11 @@ const LOG_VPORT_OPTION: &str = "agent.log_vport";
 const CONTAINER_PIPE_SIZE_OPTION: &str = "agent.container_pipe_size";
 const UNIFIED_CGROUP_HIERARCHY_OPTION: &str = "agent.unified_cgroup_hierarchy";
 const CONFIG_FILE: &str = "agent.config_file";
+const AA_KBC_PARAMS: &str = "agent.aa_kbc_params";
+const HTTPS_PROXY: &str = "agent.https_proxy";
+const NO_PROXY: &str = "agent.no_proxy";
+const ENABLE_DATA_INTEGRITY: &str = "agent.data_integrity";
+const ENABLE_SIGNATURE_VERIFICATION: &str = "agent.enable_signature_verification";
 
 const DEFAULT_LOG_LEVEL: slog::Level = slog::Level::Info;
 const DEFAULT_HOTPLUG_TIMEOUT: time::Duration = time::Duration::from_secs(3);
@@ -77,6 +83,12 @@ pub struct AgentConfig {
     pub tracing: bool,
     pub endpoints: AgentEndpoints,
     pub supports_seccomp: bool,
+    pub container_policy_path: String,
+    pub aa_kbc_params: String,
+    pub https_proxy: String,
+    pub no_proxy: String,
+    pub data_integrity: bool,
+    pub enable_signature_verification: bool,
 }
 
 #[derive(Debug, Deserialize)]
@@ -92,6 +104,12 @@ pub struct AgentConfigBuilder {
     pub unified_cgroup_hierarchy: Option<bool>,
     pub tracing: Option<bool>,
     pub endpoints: Option<EndpointsConfig>,
+    pub container_policy_path: Option<String>,
+    pub aa_kbc_params: Option<String>,
+    pub https_proxy: Option<String>,
+    pub no_proxy: Option<String>,
+    pub data_integrity: Option<bool>,
+    pub enable_signature_verification: Option<bool>,
 }
 
 macro_rules! config_override {
@@ -153,6 +171,12 @@ impl Default for AgentConfig {
             tracing: false,
             endpoints: Default::default(),
             supports_seccomp: rpc::have_seccomp(),
+            container_policy_path: String::from(""),
+            aa_kbc_params: String::from(""),
+            https_proxy: String::from(""),
+            no_proxy: String::from(""),
+            data_integrity: false,
+            enable_signature_verification: true,
         }
     }
 }
@@ -181,6 +205,16 @@ impl FromStr for AgentConfig {
         config_override!(agent_config_builder, agent_config, server_addr);
         config_override!(agent_config_builder, agent_config, unified_cgroup_hierarchy);
         config_override!(agent_config_builder, agent_config, tracing);
+        config_override!(agent_config_builder, agent_config, container_policy_path);
+        config_override!(agent_config_builder, agent_config, aa_kbc_params);
+        config_override!(agent_config_builder, agent_config, https_proxy);
+        config_override!(agent_config_builder, agent_config, no_proxy);
+        config_override!(agent_config_builder, agent_config, data_integrity);
+        config_override!(
+            agent_config_builder,
+            agent_config,
+            enable_signature_verification
+        );
 
         // Populate the allowed endpoints hash set, if we got any from the config file.
         if let Some(endpoints) = agent_config_builder.endpoints {
@@ -209,6 +243,10 @@ impl AgentConfig {
         let mut config: AgentConfig = Default::default();
         let cmdline = fs::read_to_string(file)?;
         let params: Vec<&str> = cmdline.split_ascii_whitespace().collect();
+
+        let mut using_config_file = false;
+        // Check if there is config file before parsing params that might
+        // override values from the config file.
         for param in params.iter() {
             // If we get a configuration file path from the command line, we
             // generate our config from it.
@@ -216,9 +254,13 @@ impl AgentConfig {
             // or if it can't be parsed properly.
             if param.starts_with(format!("{}=", CONFIG_FILE).as_str()) {
                 let config_file = get_string_value(param)?;
-                return AgentConfig::from_config_file(&config_file);
+                config = AgentConfig::from_config_file(&config_file)?;
+                using_config_file = true;
+                break;
             }
+        }
 
+        for param in params.iter() {
             // parse cmdline flags
             parse_cmdline_param!(param, DEBUG_CONSOLE_FLAG, config.debug_console);
             parse_cmdline_param!(param, DEV_MODE_FLAG, config.dev_mode);
@@ -278,6 +320,23 @@ impl AgentConfig {
                 config.unified_cgroup_hierarchy,
                 get_bool_value
             );
+
+            parse_cmdline_param!(param, AA_KBC_PARAMS, config.aa_kbc_params, get_string_value);
+            parse_cmdline_param!(param, HTTPS_PROXY, config.https_proxy, get_url_value);
+            parse_cmdline_param!(param, NO_PROXY, config.no_proxy, get_string_value);
+            parse_cmdline_param!(
+                param,
+                ENABLE_DATA_INTEGRITY,
+                config.data_integrity,
+                get_bool_value
+            );
+
+            parse_cmdline_param!(
+                param,
+                ENABLE_SIGNATURE_VERIFICATION,
+                config.enable_signature_verification,
+                get_bool_value
+            );
         }
 
         if let Ok(addr) = env::var(SERVER_ADDR_ENV_VAR) {
@@ -297,7 +356,9 @@ impl AgentConfig {
         }
 
         // We did not get a configuration file: allow all endpoints.
-        config.endpoints.all_allowed = true;
+        if !using_config_file {
+            config.endpoints.all_allowed = true;
+        }
 
         Ok(config)
     }
@@ -431,6 +492,12 @@ fn get_container_pipe_size(param: &str) -> Result<i32> {
     Ok(value)
 }
 
+#[instrument]
+fn get_url_value(param: &str) -> Result<String> {
+    let value = get_string_value(param)?;
+    Ok(Url::parse(&value)?.to_string())
+}
+
 #[cfg(test)]
 mod tests {
     use test_utils::assert_result;
@@ -449,6 +516,8 @@ mod tests {
         assert!(!config.dev_mode);
         assert_eq!(config.log_level, DEFAULT_LOG_LEVEL);
         assert_eq!(config.hotplug_timeout, DEFAULT_HOTPLUG_TIMEOUT);
+        assert_eq!(config.container_policy_path, "");
+        assert!(config.enable_signature_verification);
     }
 
     #[test]
@@ -467,6 +536,12 @@ mod tests {
             server_addr: &'a str,
             unified_cgroup_hierarchy: bool,
             tracing: bool,
+            container_policy_path: &'a str,
+            aa_kbc_params: &'a str,
+            https_proxy: &'a str,
+            no_proxy: &'a str,
+            data_integrity: bool,
+            enable_signature_verification: bool,
         }
 
         impl Default for TestData<'_> {
@@ -482,6 +557,12 @@ mod tests {
                     server_addr: TEST_SERVER_ADDR,
                     unified_cgroup_hierarchy: false,
                     tracing: false,
+                    container_policy_path: "",
+                    aa_kbc_params: "",
+                    https_proxy: "",
+                    no_proxy: "",
+                    data_integrity: false,
+                    enable_signature_verification: true,
                 }
             }
         }
@@ -851,6 +932,81 @@ mod tests {
                 tracing: true,
                 ..Default::default()
             },
+            TestData {
+                contents: "agent.aa_kbc_params=offline_fs_kbc::null",
+                aa_kbc_params: "offline_fs_kbc::null",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.aa_kbc_params=eaa_kbc::127.0.0.1:50000",
+                aa_kbc_params: "eaa_kbc::127.0.0.1:50000",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.https_proxy=http://proxy.url.com:81/",
+                https_proxy: "http://proxy.url.com:81/",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.https_proxy=http://192.168.1.100:81/",
+                https_proxy: "http://192.168.1.100:81/",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.no_proxy=*.internal.url.com",
+                no_proxy: "*.internal.url.com",
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.no_proxy=192.168.1.0/24,172.16.0.0/12",
+                no_proxy: "192.168.1.0/24,172.16.0.0/12",
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                data_integrity: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.data_integrity=true",
+                data_integrity: true,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.data_integrity=false",
+                data_integrity: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.data_integrity=1",
+                data_integrity: true,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.data_integrity=0",
+                data_integrity: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.enable_signature_verification=false",
+                enable_signature_verification: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.enable_signature_verification=0",
+                enable_signature_verification: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.enable_signature_verification=1",
+                enable_signature_verification: true,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.enable_signature_verification=foo",
+                enable_signature_verification: false,
+                ..Default::default()
+            },
         ];
 
         let dir = tempdir().expect("failed to create tmpdir");
@@ -898,6 +1054,20 @@ mod tests {
             assert_eq!(d.container_pipe_size, config.container_pipe_size, "{}", msg);
             assert_eq!(d.server_addr, config.server_addr, "{}", msg);
             assert_eq!(d.tracing, config.tracing, "{}", msg);
+            assert_eq!(
+                d.container_policy_path, config.container_policy_path,
+                "{}",
+                msg
+            );
+            assert_eq!(d.aa_kbc_params, config.aa_kbc_params, "{}", msg);
+            assert_eq!(d.https_proxy, config.https_proxy, "{}", msg);
+            assert_eq!(d.no_proxy, config.no_proxy, "{}", msg);
+            assert_eq!(d.data_integrity, config.data_integrity, "{}", msg);
+            assert_eq!(
+                d.enable_signature_verification, config.enable_signature_verification,
+                "{}",
+                msg
+            );
 
             for v in vars_to_unset {
                 env::remove_var(v);
@@ -1399,4 +1569,50 @@ Caused by:
         // Verify that the default values are valid
         assert_eq!(config.hotplug_timeout, DEFAULT_HOTPLUG_TIMEOUT);
     }
+
+    #[test]
+    fn test_config_from_cmdline_and_config_file() {
+        let dir = tempdir().expect("failed to create tmpdir");
+
+        let agent_config = r#"
+               dev_mode = false
+               server_addr = 'vsock://8:2048'
+
+               [endpoints]
+               allowed = ["CreateContainer", "StartContainer"]
+              "#;
+
+        let config_path = dir.path().join("agent-config.toml");
+        let config_filename = config_path.to_str().expect("failed to get config filename");
+
+        fs::write(config_filename, agent_config).expect("failed to write agen config");
+
+        let cmdline = format!("agent.devmode agent.config_file={}", config_filename);
+
+        let cmdline_path = dir.path().join("cmdline");
+        let cmdline_filename = cmdline_path
+            .to_str()
+            .expect("failed to get cmdline filename");
+
+        fs::write(cmdline_filename, cmdline).expect("failed to write agen config");
+
+        let config = AgentConfig::from_cmdline(cmdline_filename, vec![])
+            .expect("failed to parse command line");
+
+        // Should be overwritten by cmdline
+        assert!(config.dev_mode);
+
+        // Should be from agent config
+        assert_eq!(config.server_addr, "vsock://8:2048");
+
+        // Should be from agent config
+        assert_eq!(
+            config.endpoints.allowed,
+            vec!["CreateContainer".to_string(), "StartContainer".to_string()]
+                .iter()
+                .cloned()
+                .collect()
+        );
+        assert!(!config.endpoints.all_allowed);
+    }
 }

src/agent/src/device.rs

@@ -414,7 +414,7 @@ fn scan_scsi_bus(scsi_addr: &str) -> Result<()> {
 
     // Scan scsi host passing in the channel, SCSI id and LUN.
     // Channel is always 0 because we have only one SCSI controller.
-    let scan_data = format!("0 {} {}", tokens[0], tokens[1]);
+    let scan_data = &format!("0 {} {}", tokens[0], tokens[1]);
 
     for entry in fs::read_dir(SYSFS_SCSI_HOST_PATH)? {
         let host = entry?.file_name();
@@ -428,7 +428,7 @@ fn scan_scsi_bus(scsi_addr: &str) -> Result<()> {
 
         let scan_path = PathBuf::from(&format!("{}/{}/{}", SYSFS_SCSI_HOST_PATH, host_str, "scan"));
 
-        fs::write(scan_path, &scan_data)?;
+        fs::write(scan_path, scan_data)?;
     }
 
     Ok(())
@@ -571,13 +571,15 @@ fn update_spec_devices(spec: &mut Spec, mut updates: HashMap<&str, DevUpdate>) -
 
     if let Some(resources) = linux.resources.as_mut() {
         for r in &mut resources.devices {
-            if let (Some(host_major), Some(host_minor)) = (r.major, r.minor) {
-                if let Some(update) = res_updates.get(&(r.r#type.as_str(), host_major, host_minor))
+            if let (Some(host_type), Some(host_major), Some(host_minor)) =
+                (r.r#type.as_ref(), r.major, r.minor)
+            {
+                if let Some(update) = res_updates.get(&(host_type.as_str(), host_major, host_minor))
                 {
                     info!(
                         sl!(),
                         "update_spec_devices() updating resource";
-                        "type" => &r.r#type,
+                        "type" => &host_type,
                         "host_major" => host_major,
                         "host_minor" => host_minor,
                         "guest_major" => update.guest_major,
@@ -854,7 +856,7 @@ pub fn update_device_cgroup(spec: &mut Spec) -> Result<()> {
         allow: false,
         major: Some(major),
         minor: Some(minor),
-        r#type: String::from("b"),
+        r#type: Some(String::from("b")),
         access: String::from("rw"),
     });
 
@@ -1017,13 +1019,13 @@ mod tests {
                 resources: Some(LinuxResources {
                     devices: vec![
                         oci::LinuxDeviceCgroup {
-                            r#type: "c".to_string(),
+                            r#type: Some("c".to_string()),
                             major: Some(host_major_a),
                             minor: Some(host_minor_a),
                             ..oci::LinuxDeviceCgroup::default()
                         },
                         oci::LinuxDeviceCgroup {
-                            r#type: "c".to_string(),
+                            r#type: Some("c".to_string()),
                             major: Some(host_major_b),
                             minor: Some(host_minor_b),
                             ..oci::LinuxDeviceCgroup::default()
@@ -1116,13 +1118,13 @@ mod tests {
                 resources: Some(LinuxResources {
                     devices: vec![
                         LinuxDeviceCgroup {
-                            r#type: "c".to_string(),
+                            r#type: Some("c".to_string()),
                             major: Some(host_major),
                             minor: Some(host_minor),
                             ..LinuxDeviceCgroup::default()
                         },
                         LinuxDeviceCgroup {
-                            r#type: "b".to_string(),
+                            r#type: Some("b".to_string()),
                             major: Some(host_major),
                             minor: Some(host_minor),
                             ..LinuxDeviceCgroup::default()
@@ -1531,7 +1533,7 @@ mod tests {
         pci_driver_override(syspci, dev0, "drv_b").unwrap();
         assert_eq!(fs::read_to_string(&dev0override).unwrap(), "drv_b");
         assert_eq!(fs::read_to_string(&probepath).unwrap(), dev0.to_string());
-        assert_eq!(fs::read_to_string(&drvaunbind).unwrap(), dev0.to_string());
+        assert_eq!(fs::read_to_string(drvaunbind).unwrap(), dev0.to_string());
     }
 
     #[test]
@@ -1543,7 +1545,7 @@ mod tests {
         let dev0 = pci::Address::new(0, 0, pci::SlotFn::new(0, 0).unwrap());
         let dev0path = syspci.join("devices").join(dev0.to_string());
 
-        fs::create_dir_all(&dev0path).unwrap();
+        fs::create_dir_all(dev0path).unwrap();
 
         // Test dev0
         assert!(pci_iommu_group(&syspci, dev0).unwrap().is_none());
@@ -1554,7 +1556,7 @@ mod tests {
         let dev1group = dev1path.join("iommu_group");
 
         fs::create_dir_all(&dev1path).unwrap();
-        std::os::unix::fs::symlink("../../../kernel/iommu_groups/12", &dev1group).unwrap();
+        std::os::unix::fs::symlink("../../../kernel/iommu_groups/12", dev1group).unwrap();
 
         // Test dev1
         assert_eq!(
@@ -1567,7 +1569,7 @@ mod tests {
         let dev2path = syspci.join("devices").join(dev2.to_string());
         let dev2group = dev2path.join("iommu_group");
 
-        fs::create_dir_all(&dev2group).unwrap();
+        fs::create_dir_all(dev2group).unwrap();
 
         // Test dev2
         assert!(pci_iommu_group(&syspci, dev2).is_err());

src/agent/src/image_rpc.rs

@@ -0,0 +1,337 @@
+// Copyright (c) 2021 Alibaba Cloud
+// Copyright (c) 2021, 2023 IBM Corporation
+// Copyright (c) 2022 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::env;
+use std::fs;
+use std::path::Path;
+use std::process::Command;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+
+use anyhow::{anyhow, Result};
+use async_trait::async_trait;
+use protocols::image;
+use tokio::sync::Mutex;
+use ttrpc::{self, error::get_rpc_status as ttrpc_error};
+
+use crate::rpc::{verify_cid, CONTAINER_BASE};
+use crate::sandbox::Sandbox;
+use crate::AGENT_CONFIG;
+
+use image_rs::image::ImageClient;
+use std::io::Write;
+
+const AA_PATH: &str = "/usr/local/bin/attestation-agent";
+const AA_KEYPROVIDER_PORT: &str = "127.0.0.1:50000";
+const AA_GETRESOURCE_PORT: &str = "127.0.0.1:50001";
+const OCICRYPT_CONFIG_PATH: &str = "/tmp/ocicrypt_config.json";
+// kata rootfs is readonly, use tmpfs before CC storage is implemented.
+const KATA_CC_IMAGE_WORK_DIR: &str = "/run/image/";
+const KATA_CC_PAUSE_BUNDLE: &str = "/pause_bundle";
+const CONFIG_JSON: &str = "config.json";
+
+// Convenience macro to obtain the scope logger
+macro_rules! sl {
+    () => {
+        slog_scope::logger()
+    };
+}
+
+pub struct ImageService {
+    sandbox: Arc<Mutex<Sandbox>>,
+    attestation_agent_started: AtomicBool,
+    image_client: Arc<Mutex<ImageClient>>,
+}
+
+impl ImageService {
+    pub fn new(sandbox: Arc<Mutex<Sandbox>>) -> Self {
+        env::set_var("CC_IMAGE_WORK_DIR", KATA_CC_IMAGE_WORK_DIR);
+        Self {
+            sandbox,
+            attestation_agent_started: AtomicBool::new(false),
+            image_client: Arc::new(Mutex::new(ImageClient::default())),
+        }
+    }
+
+    // pause image is packaged in rootfs for CC
+    fn unpack_pause_image(cid: &str) -> Result<()> {
+        let cc_pause_bundle = Path::new(KATA_CC_PAUSE_BUNDLE);
+        if !cc_pause_bundle.exists() {
+            return Err(anyhow!("Pause image not present in rootfs"));
+        }
+
+        info!(sl!(), "use guest pause image cid {:?}", cid);
+        let pause_bundle = Path::new(CONTAINER_BASE).join(cid);
+        let pause_rootfs = pause_bundle.join("rootfs");
+        let pause_config = pause_bundle.join(CONFIG_JSON);
+        let pause_binary = pause_rootfs.join("pause");
+        fs::create_dir_all(&pause_rootfs)?;
+        if !pause_config.exists() {
+            fs::copy(
+                cc_pause_bundle.join(CONFIG_JSON),
+                pause_bundle.join(CONFIG_JSON),
+            )?;
+        }
+        if !pause_binary.exists() {
+            fs::copy(cc_pause_bundle.join("rootfs").join("pause"), pause_binary)?;
+        }
+
+        Ok(())
+    }
+
+    // If we fail to start the AA, ocicrypt won't be able to unwrap keys
+    // and container decryption will fail.
+    fn init_attestation_agent() -> Result<()> {
+        let config_path = OCICRYPT_CONFIG_PATH;
+
+        // The image will need to be encrypted using a keyprovider
+        // that has the same name (at least according to the config).
+        let ocicrypt_config = serde_json::json!({
+            "key-providers": {
+                "attestation-agent":{
+                    "grpc":AA_KEYPROVIDER_PORT
+                }
+            }
+        });
+
+        let mut config_file = fs::File::create(config_path)?;
+        config_file.write_all(ocicrypt_config.to_string().as_bytes())?;
+
+        // The Attestation Agent will run for the duration of the guest.
+        Command::new(AA_PATH)
+            .arg("--keyprovider_sock")
+            .arg(AA_KEYPROVIDER_PORT)
+            .arg("--getresource_sock")
+            .arg(AA_GETRESOURCE_PORT)
+            .spawn()?;
+        Ok(())
+    }
+
+    /// Determines the container id (cid) to use for a given request.
+    ///
+    /// If the request specifies a non-empty id, use it; otherwise derive it from the image path.
+    /// In either case, verify that the chosen id is valid.
+    fn cid_from_request(req: &image::PullImageRequest) -> Result<String> {
+        let req_cid = req.get_container_id();
+        let cid = if !req_cid.is_empty() {
+            req_cid.to_string()
+        } else if let Some(last) = req.get_image().rsplit('/').next() {
+            // ':' not valid for container id
+            last.replace(':', "_")
+        } else {
+            return Err(anyhow!("Invalid image name. {}", req.get_image()));
+        };
+        verify_cid(&cid)?;
+        Ok(cid)
+    }
+
+    async fn pull_image(&self, req: &image::PullImageRequest) -> Result<String> {
+        env::set_var("OCICRYPT_KEYPROVIDER_CONFIG", OCICRYPT_CONFIG_PATH);
+
+        let https_proxy = &AGENT_CONFIG.read().await.https_proxy;
+        if !https_proxy.is_empty() {
+            env::set_var("HTTPS_PROXY", https_proxy);
+        }
+
+        let no_proxy = &AGENT_CONFIG.read().await.no_proxy;
+        if !no_proxy.is_empty() {
+            env::set_var("NO_PROXY", no_proxy);
+        }
+
+        let cid = Self::cid_from_request(req)?;
+        let image = req.get_image();
+        if cid.starts_with("pause") {
+            Self::unpack_pause_image(&cid)?;
+
+            let mut sandbox = self.sandbox.lock().await;
+            sandbox.images.insert(String::from(image), cid);
+            return Ok(image.to_owned());
+        }
+
+        let aa_kbc_params = &AGENT_CONFIG.read().await.aa_kbc_params;
+        if !aa_kbc_params.is_empty() {
+            match self.attestation_agent_started.compare_exchange_weak(
+                false,
+                true,
+                Ordering::SeqCst,
+                Ordering::SeqCst,
+            ) {
+                Ok(_) => Self::init_attestation_agent()?,
+                Err(_) => info!(sl!(), "Attestation Agent already running"),
+            }
+        }
+        // If the attestation-agent is being used, then enable the authenticated credentials support
+        info!(
+            sl!(),
+            "image_client.config.auth set to: {}",
+            !aa_kbc_params.is_empty()
+        );
+        self.image_client.lock().await.config.auth = !aa_kbc_params.is_empty();
+
+        // Read enable signature verification from the agent config and set it in the image_client
+        let enable_signature_verification =
+            &AGENT_CONFIG.read().await.enable_signature_verification;
+        info!(
+            sl!(),
+            "enable_signature_verification set to: {}", enable_signature_verification
+        );
+        self.image_client.lock().await.config.security_validate = *enable_signature_verification;
+
+        let source_creds = (!req.get_source_creds().is_empty()).then(|| req.get_source_creds());
+
+        let bundle_path = Path::new(CONTAINER_BASE).join(&cid);
+        fs::create_dir_all(&bundle_path)?;
+
+        let decrypt_config = format!("provider:attestation-agent:{}", aa_kbc_params);
+
+        info!(sl!(), "pull image {:?}, bundle path {:?}", cid, bundle_path);
+        // Image layers will store at KATA_CC_IMAGE_WORK_DIR, generated bundles
+        // with rootfs and config.json will store under CONTAINER_BASE/cid.
+        let res = self
+            .image_client
+            .lock()
+            .await
+            .pull_image(image, &bundle_path, &source_creds, &Some(&decrypt_config))
+            .await;
+
+        match res {
+            Ok(image) => {
+                info!(
+                    sl!(),
+                    "pull and unpack image {:?}, cid: {:?}, with image-rs succeed. ", image, cid
+                );
+            }
+            Err(e) => {
+                error!(
+                    sl!(),
+                    "pull and unpack image {:?}, cid: {:?}, with image-rs failed with {:?}. ",
+                    image,
+                    cid,
+                    e.to_string()
+                );
+                return Err(e);
+            }
+        };
+
+        let mut sandbox = self.sandbox.lock().await;
+        sandbox.images.insert(String::from(image), cid);
+        Ok(image.to_owned())
+    }
+}
+
+#[async_trait]
+impl protocols::image_ttrpc_async::Image for ImageService {
+    async fn pull_image(
+        &self,
+        _ctx: &ttrpc::r#async::TtrpcContext,
+        req: image::PullImageRequest,
+    ) -> ttrpc::Result<image::PullImageResponse> {
+        match self.pull_image(&req).await {
+            Ok(r) => {
+                let mut resp = image::PullImageResponse::new();
+                resp.image_ref = r;
+                return Ok(resp);
+            }
+            Err(e) => {
+                return Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()));
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::ImageService;
+    use protocols::image;
+
+    #[test]
+    fn test_cid_from_request() {
+        struct Case {
+            cid: &'static str,
+            image: &'static str,
+            result: Option<&'static str>,
+        }
+
+        let cases = [
+            Case {
+                cid: "",
+                image: "",
+                result: None,
+            },
+            Case {
+                cid: "..",
+                image: "",
+                result: None,
+            },
+            Case {
+                cid: "",
+                image: "..",
+                result: None,
+            },
+            Case {
+                cid: "",
+                image: "abc/..",
+                result: None,
+            },
+            Case {
+                cid: "",
+                image: "abc/",
+                result: None,
+            },
+            Case {
+                cid: "",
+                image: "../abc",
+                result: Some("abc"),
+            },
+            Case {
+                cid: "",
+                image: "../9abc",
+                result: Some("9abc"),
+            },
+            Case {
+                cid: "some-string.1_2",
+                image: "",
+                result: Some("some-string.1_2"),
+            },
+            Case {
+                cid: "0some-string.1_2",
+                image: "",
+                result: Some("0some-string.1_2"),
+            },
+            Case {
+                cid: "a:b",
+                image: "",
+                result: None,
+            },
+            Case {
+                cid: "",
+                image: "prefix/a:b",
+                result: Some("a_b"),
+            },
+            Case {
+                cid: "",
+                image: "/a/b/c/d:e",
+                result: Some("d_e"),
+            },
+        ];
+
+        for case in &cases {
+            let mut req = image::PullImageRequest::new();
+            req.set_image(case.image.to_string());
+            req.set_container_id(case.cid.to_string());
+            let ret = ImageService::cid_from_request(&req);
+            match (case.result, ret) {
+                (Some(expected), Ok(actual)) => assert_eq!(expected, actual),
+                (None, Err(_)) => (),
+                (None, Ok(r)) => panic!("Expected an error, got {}", r),
+                (Some(expected), Err(e)) => {
+                    panic!("Expected {} but got an error ({})", expected, e)
+                }
+            }
+        }
+    }
+}

src/agent/src/main.rs

@@ -71,6 +71,7 @@ use tokio::{
     task::JoinHandle,
 };
 
+mod image_rpc;
 mod rpc;
 mod tracer;
 

src/agent/src/metrics.rs

@@ -5,10 +5,11 @@
 
 extern crate procfs;
 
-use prometheus::{Encoder, Gauge, GaugeVec, IntCounter, TextEncoder};
+use prometheus::{Encoder, Gauge, GaugeVec, IntCounter, Opts, Registry, TextEncoder};
 
-use anyhow::Result;
+use anyhow::{anyhow, Result};
 use slog::warn;
+use std::sync::Mutex;
 use tracing::instrument;
 
 const NAMESPACE_KATA_AGENT: &str = "kata_agent";
@@ -23,55 +24,70 @@ macro_rules! sl {
 
 lazy_static! {
 
-    static ref     AGENT_SCRAPE_COUNT: IntCounter =
-    prometheus::register_int_counter!(format!("{}_{}",NAMESPACE_KATA_AGENT,"scrape_count"), "Metrics scrape count").unwrap();
+    static ref REGISTERED: Mutex<bool> = Mutex::new(false);
 
-    static ref     AGENT_THREADS: Gauge =
-    prometheus::register_gauge!(format!("{}_{}",NAMESPACE_KATA_AGENT,"threads"), "Agent process threads").unwrap();
+    // custom registry
+    static ref REGISTRY: Registry = Registry::new();
 
-    static ref     AGENT_TOTAL_TIME: Gauge =
-    prometheus::register_gauge!(format!("{}_{}",NAMESPACE_KATA_AGENT,"total_time"), "Agent process total time").unwrap();
+    static ref AGENT_SCRAPE_COUNT: IntCounter =
+    IntCounter::new(format!("{}_{}",NAMESPACE_KATA_AGENT,"scrape_count"), "Metrics scrape count").unwrap();
 
-    static ref     AGENT_TOTAL_VM: Gauge =
-    prometheus::register_gauge!(format!("{}_{}",NAMESPACE_KATA_AGENT,"total_vm"), "Agent process total VM size").unwrap();
+    // agent metrics
+    static ref AGENT_THREADS: Gauge =
+    Gauge::new(format!("{}_{}",NAMESPACE_KATA_AGENT,"threads"), "Agent process threads").unwrap();
 
-    static ref     AGENT_TOTAL_RSS: Gauge =
-    prometheus::register_gauge!(format!("{}_{}",NAMESPACE_KATA_AGENT,"total_rss"), "Agent process total RSS size").unwrap();
+    static ref AGENT_TOTAL_TIME: Gauge =
+    Gauge::new(format!("{}_{}",NAMESPACE_KATA_AGENT,"total_time"), "Agent process total time").unwrap();
 
-    static ref     AGENT_PROC_STATUS: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_AGENT,"proc_status"), "Agent process status.", &["item"]).unwrap();
+    static ref AGENT_TOTAL_VM: Gauge =
+    Gauge::new(format!("{}_{}",NAMESPACE_KATA_AGENT,"total_vm"), "Agent process total VM size").unwrap() ;
 
-    static ref     AGENT_IO_STAT: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_AGENT,"io_stat"), "Agent process IO statistics.", &["item"]).unwrap();
+    static ref AGENT_TOTAL_RSS: Gauge =
+    Gauge::new(format!("{}_{}",NAMESPACE_KATA_AGENT,"total_rss"), "Agent process total RSS size").unwrap();
 
-    static ref     AGENT_PROC_STAT: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_AGENT,"proc_stat"), "Agent process statistics.", &["item"]).unwrap();
+    static ref AGENT_PROC_STATUS: GaugeVec =
+    GaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_AGENT,"proc_status"), "Agent process status."), &["item"]).unwrap();
+
+    static ref AGENT_IO_STAT: GaugeVec =
+    GaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_AGENT,"io_stat"), "Agent process IO statistics."), &["item"]).unwrap();
+
+    static ref AGENT_PROC_STAT: GaugeVec =
+    GaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_AGENT,"proc_stat"), "Agent process statistics."), &["item"]).unwrap();
 
     // guest os metrics
-    static ref     GUEST_LOAD: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"load") , "Guest system load.", &["item"]).unwrap();
+    static ref GUEST_LOAD: GaugeVec =
+    GaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_GUEST,"load"), "Guest system load."), &["item"]).unwrap();
 
-    static ref     GUEST_TASKS: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"tasks") , "Guest system load.", &["item"]).unwrap();
+    static ref GUEST_TASKS: GaugeVec =
+    GaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_GUEST,"tasks"), "Guest system load."), &["item"]).unwrap();
 
-    static ref     GUEST_CPU_TIME: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"cpu_time") , "Guest CPU statistics.", &["cpu","item"]).unwrap();
+    static ref GUEST_CPU_TIME: GaugeVec =
+    GaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_GUEST,"cpu_time"), "Guest CPU statistics."), &["cpu","item"]).unwrap();
 
-    static ref     GUEST_VM_STAT: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"vm_stat") , "Guest virtual memory statistics.", &["item"]).unwrap();
+    static ref GUEST_VM_STAT: GaugeVec =
+    GaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_GUEST,"vm_stat"), "Guest virtual memory statistics."), &["item"]).unwrap();
 
-    static ref     GUEST_NETDEV_STAT: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"netdev_stat") , "Guest net devices statistics.", &["interface","item"]).unwrap();
+    static ref GUEST_NETDEV_STAT: GaugeVec =
+    GaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_GUEST,"netdev_stat"), "Guest net devices statistics."), &["interface","item"]).unwrap();
 
-    static ref     GUEST_DISKSTAT: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"diskstat") , "Disks statistics in system.", &["disk","item"]).unwrap();
+    static ref GUEST_DISKSTAT: GaugeVec =
+    GaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_GUEST,"diskstat"), "Disks statistics in system."), &["disk","item"]).unwrap();
 
-    static ref     GUEST_MEMINFO: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"meminfo") , "Statistics about memory usage in the system.", &["item"]).unwrap();
+    static ref GUEST_MEMINFO: GaugeVec =
+    GaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_GUEST,"meminfo"), "Statistics about memory usage in the system."), &["item"]).unwrap();
 }
 
 #[instrument]
 pub fn get_metrics(_: &protocols::agent::GetMetricsRequest) -> Result<String> {
+    let mut registered = REGISTERED
+        .lock()
+        .map_err(|e| anyhow!("failed to check agent metrics register status {:?}", e))?;
+
+    if !(*registered) {
+        register_metrics()?;
+        *registered = true;
+    }
+
     AGENT_SCRAPE_COUNT.inc();
 
     // update agent process metrics
@@ -81,7 +97,7 @@ pub fn get_metrics(_: &protocols::agent::GetMetricsRequest) -> Result<String> {
     update_guest_metrics();
 
     // gather all metrics and return as a String
-    let metric_families = prometheus::gather();
+    let metric_families = REGISTRY.gather();
 
     let mut buffer = Vec::new();
     let encoder = TextEncoder::new();
@@ -90,6 +106,31 @@ pub fn get_metrics(_: &protocols::agent::GetMetricsRequest) -> Result<String> {
     Ok(String::from_utf8(buffer)?)
 }
 
+#[instrument]
+fn register_metrics() -> Result<()> {
+    REGISTRY.register(Box::new(AGENT_SCRAPE_COUNT.clone()))?;
+
+    // agent metrics
+    REGISTRY.register(Box::new(AGENT_THREADS.clone()))?;
+    REGISTRY.register(Box::new(AGENT_TOTAL_TIME.clone()))?;
+    REGISTRY.register(Box::new(AGENT_TOTAL_VM.clone()))?;
+    REGISTRY.register(Box::new(AGENT_TOTAL_RSS.clone()))?;
+    REGISTRY.register(Box::new(AGENT_PROC_STATUS.clone()))?;
+    REGISTRY.register(Box::new(AGENT_IO_STAT.clone()))?;
+    REGISTRY.register(Box::new(AGENT_PROC_STAT.clone()))?;
+
+    // guest metrics
+    REGISTRY.register(Box::new(GUEST_LOAD.clone()))?;
+    REGISTRY.register(Box::new(GUEST_TASKS.clone()))?;
+    REGISTRY.register(Box::new(GUEST_CPU_TIME.clone()))?;
+    REGISTRY.register(Box::new(GUEST_VM_STAT.clone()))?;
+    REGISTRY.register(Box::new(GUEST_NETDEV_STAT.clone()))?;
+    REGISTRY.register(Box::new(GUEST_DISKSTAT.clone()))?;
+    REGISTRY.register(Box::new(GUEST_MEMINFO.clone()))?;
+
+    Ok(())
+}
+
 #[instrument]
 fn update_agent_metrics() -> Result<()> {
     let me = procfs::process::Process::myself();

src/agent/src/mount.rs

@@ -648,7 +648,7 @@ pub fn recursive_ownership_change(
 ) -> Result<()> {
     let mut mask = if read_only { RO_MASK } else { RW_MASK };
     if path.is_dir() {
-        for entry in fs::read_dir(&path)? {
+        for entry in fs::read_dir(path)? {
             recursive_ownership_change(entry?.path().as_path(), uid, gid, read_only)?;
         }
         mask |= EXEC_MASK;
@@ -894,7 +894,7 @@ pub fn get_cgroup_mounts(
         }]);
     }
 
-    let file = File::open(&cg_path)?;
+    let file = File::open(cg_path)?;
     let reader = BufReader::new(file);
 
     let mut has_device_cgroup = false;
@@ -1777,7 +1777,7 @@ mod tests {
             let tempdir = tempdir().unwrap();
 
             let src = if d.mask_src {
-                tempdir.path().join(&d.src)
+                tempdir.path().join(d.src)
             } else {
                 Path::new(d.src).to_path_buf()
             };

src/agent/src/namespace.rs

@@ -78,6 +78,7 @@ impl Namespace {
     // setup creates persistent namespace without switching to it.
     // Note, pid namespaces cannot be persisted.
     #[instrument]
+    #[allow(clippy::question_mark)]
     pub async fn setup(mut self) -> Result<Self> {
         fs::create_dir_all(&self.persistent_ns_dir)?;
 
@@ -88,7 +89,7 @@ impl Namespace {
         }
         let logger = self.logger.clone();
 
-        let new_ns_path = ns_path.join(&ns_type.get());
+        let new_ns_path = ns_path.join(ns_type.get());
 
         File::create(new_ns_path.as_path())?;
 
@@ -102,7 +103,7 @@ impl Namespace {
                 let source = Path::new(&origin_ns_path);
                 let destination = new_ns_path.as_path();
 
-                File::open(&source)?;
+                File::open(source)?;
 
                 // Create a new netns on the current thread.
                 let cf = ns_type.get_flags();

src/agent/src/netlink.rs

@@ -529,7 +529,9 @@ impl Handle {
             .map_err(|e| anyhow!("Failed to parse IP {}: {:?}", ip_address, e))?;
 
         // Import rtnetlink objects that make sense only for this function
-        use packet::constants::{NDA_UNSPEC, NLM_F_ACK, NLM_F_CREATE, NLM_F_EXCL, NLM_F_REQUEST};
+        use packet::constants::{
+            NDA_UNSPEC, NLM_F_ACK, NLM_F_CREATE, NLM_F_REPLACE, NLM_F_REQUEST,
+        };
         use packet::neighbour::{NeighbourHeader, NeighbourMessage};
         use packet::nlas::neighbour::Nla;
         use packet::{NetlinkMessage, NetlinkPayload, RtnlMessage};
@@ -572,7 +574,7 @@ impl Handle {
 
         // Send request and ACK
         let mut req = NetlinkMessage::from(RtnlMessage::NewNeighbour(message));
-        req.header.flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE;
+        req.header.flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_CREATE | NLM_F_REPLACE;
 
         let mut response = self.handle.request(req)?;
         while let Some(message) = response.next().await {
@@ -944,13 +946,13 @@ mod tests {
     fn clean_env_for_test_add_one_arp_neighbor(dummy_name: &str, ip: &str) {
         // ip link delete dummy
         Command::new("ip")
-            .args(&["link", "delete", dummy_name])
+            .args(["link", "delete", dummy_name])
             .output()
             .expect("prepare: failed to delete dummy");
 
         // ip neigh del dev dummy ip
         Command::new("ip")
-            .args(&["neigh", "del", dummy_name, ip])
+            .args(["neigh", "del", dummy_name, ip])
             .output()
             .expect("prepare: failed to delete neigh");
     }
@@ -965,19 +967,19 @@ mod tests {
 
         // ip link add dummy type dummy
         Command::new("ip")
-            .args(&["link", "add", dummy_name, "type", "dummy"])
+            .args(["link", "add", dummy_name, "type", "dummy"])
             .output()
             .expect("failed to add dummy interface");
 
         // ip addr add 192.168.0.2/16 dev dummy
         Command::new("ip")
-            .args(&["addr", "add", "192.168.0.2/16", "dev", dummy_name])
+            .args(["addr", "add", "192.168.0.2/16", "dev", dummy_name])
             .output()
             .expect("failed to add ip for dummy");
 
         // ip link set dummy up;
         Command::new("ip")
-            .args(&["link", "set", dummy_name, "up"])
+            .args(["link", "set", dummy_name, "up"])
             .output()
             .expect("failed to up dummy");
     }
@@ -1009,7 +1011,7 @@ mod tests {
 
         // ip neigh show dev dummy ip
         let stdout = Command::new("ip")
-            .args(&["neigh", "show", "dev", dummy_name, to_ip])
+            .args(["neigh", "show", "dev", dummy_name, to_ip])
             .output()
             .expect("failed to show neigh")
             .stdout;

src/agent/src/network.rs

@@ -7,6 +7,7 @@ use anyhow::{anyhow, Result};
 use nix::mount::{self, MsFlags};
 use slog::Logger;
 use std::fs;
+use std::path;
 
 const KATA_GUEST_SANDBOX_DNS_FILE: &str = "/run/kata-containers/sandbox/resolv.conf";
 const GUEST_DNS_FILE: &str = "/etc/resolv.conf";
@@ -64,7 +65,13 @@ fn do_setup_guest_dns(logger: Logger, dns_list: Vec<String>, src: &str, dst: &st
         .map(|x| x.trim())
         .collect::<Vec<&str>>()
         .join("\n");
-    fs::write(src, &content)?;
+
+    // make sure the src file's parent path exist.
+    let file_path = path::Path::new(src);
+    if let Some(p) = file_path.parent() {
+        fs::create_dir_all(p)?;
+    }
+    fs::write(src, content)?;
 
     // bind mount to /etc/resolv.conf
     mount::mount(Some(src), dst, Some("bind"), MsFlags::MS_BIND, None::<&str>)

src/agent/src/rpc.rs

@@ -34,7 +34,10 @@ use protocols::health::{
     HealthCheckResponse, HealthCheckResponse_ServingStatus, VersionCheckResponse,
 };
 use protocols::types::Interface;
-use protocols::{agent_ttrpc_async as agent_ttrpc, health_ttrpc_async as health_ttrpc};
+use protocols::{
+    agent_ttrpc_async as agent_ttrpc, health_ttrpc_async as health_ttrpc,
+    image_ttrpc_async as image_ttrpc,
+};
 use rustjail::cgroups::notifier;
 use rustjail::container::{BaseContainer, Container, LinuxContainer};
 use rustjail::process::Process;
@@ -44,12 +47,12 @@ use nix::errno::Errno;
 use nix::mount::MsFlags;
 use nix::sys::{stat, statfs};
 use nix::unistd::{self, Pid};
-use rustjail::cgroups::Manager;
 use rustjail::process::ProcessOperations;
 
 use crate::device::{
     add_devices, get_virtio_blk_pci_device_name, update_device_cgroup, update_env_pci,
 };
+use crate::image_rpc;
 use crate::linux_abi::*;
 use crate::metrics::get_metrics;
 use crate::mount::{add_storages, baremount, STORAGE_HANDLER_LIST};
@@ -81,12 +84,22 @@ use std::io::{BufRead, BufReader, Write};
 use std::os::unix::fs::FileExt;
 use std::path::PathBuf;
 
-const CONTAINER_BASE: &str = "/run/kata-containers";
+pub const CONTAINER_BASE: &str = "/run/kata-containers";
 const MODPROBE_PATH: &str = "/sbin/modprobe";
+const ANNO_K8S_IMAGE_NAME: &str = "io.kubernetes.cri.image-name";
+const CONFIG_JSON: &str = "config.json";
+const INIT_TRUSTED_STORAGE: &str = "/usr/bin/kata-init-trusted-storage";
+const TRUSTED_STORAGE_DEVICE: &str = "/dev/trusted_store";
 
+/// the iptables seriers binaries could appear either in /sbin
+/// or /usr/sbin, we need to check both of them
+const USR_IPTABLES_SAVE: &str = "/usr/sbin/iptables-save";
 const IPTABLES_SAVE: &str = "/sbin/iptables-save";
+const USR_IPTABLES_RESTORE: &str = "/usr/sbin/iptables-store";
 const IPTABLES_RESTORE: &str = "/sbin/iptables-restore";
+const USR_IP6TABLES_SAVE: &str = "/usr/sbin/ip6tables-save";
 const IP6TABLES_SAVE: &str = "/sbin/ip6tables-save";
+const USR_IP6TABLES_RESTORE: &str = "/usr/sbin/ip6tables-save";
 const IP6TABLES_RESTORE: &str = "/sbin/ip6tables-restore";
 
 const ERR_CANNOT_GET_WRITER: &str = "Cannot get writer";
@@ -134,6 +147,41 @@ pub struct AgentService {
     sandbox: Arc<Mutex<Sandbox>>,
 }
 
+// A container ID must match this regex:
+//
+//     ^[a-zA-Z0-9][a-zA-Z0-9_.-]+$
+//
+pub fn verify_cid(id: &str) -> Result<()> {
+    let mut chars = id.chars();
+
+    let valid = matches!(chars.next(), Some(first) if first.is_alphanumeric()
+                && id.len() > 1
+                && chars.all(|c| c.is_alphanumeric() || ['.', '-', '_'].contains(&c)));
+
+    match valid {
+        true => Ok(()),
+        false => Err(anyhow!("invalid container ID: {:?}", id)),
+    }
+}
+
+// Partially merge an OCI process specification into another one.
+fn merge_oci_process(target: &mut oci::Process, source: &oci::Process) {
+    if target.args.is_empty() && !source.args.is_empty() {
+        target.args.append(&mut source.args.clone());
+    }
+
+    if target.cwd == "/" && source.cwd != "/" {
+        target.cwd = String::from(&source.cwd);
+    }
+
+    for source_env in &source.env {
+        let variable_name: Vec<&str> = source_env.split('=').collect();
+        if !target.env.iter().any(|i| i.contains(variable_name[0])) {
+            target.env.push(source_env.to_string());
+        }
+    }
+}
+
 impl AgentService {
     #[instrument]
     async fn do_create_container(
@@ -164,6 +212,9 @@ impl AgentService {
             "receive createcontainer, storages: {:?}", &req.storages
         );
 
+        // Merge the image bundle OCI spec into the container creation request OCI spec.
+        self.merge_bundle_oci(&mut oci).await?;
+
         // Some devices need some extra processing (the ones invoked with
         // --device for instance), and that's what this call is doing. It
         // updates the devices listed in the OCI spec, so that they actually
@@ -171,6 +222,30 @@ impl AgentService {
         // cannot predict everything from the caller.
         add_devices(&req.devices.to_vec(), &mut oci, &self.sandbox).await?;
 
+        let linux = oci
+            .linux
+            .as_mut()
+            .ok_or_else(|| anyhow!("Spec didn't contain linux field"))?;
+
+        for specdev in &mut linux.devices {
+            let dev_major_minor = format!("{}:{}", specdev.major, specdev.minor);
+
+            if specdev.path == TRUSTED_STORAGE_DEVICE {
+                let data_integrity = AGENT_CONFIG.read().await.data_integrity;
+                info!(
+                    sl!(),
+                    "trusted_store device major:min {}, enable data integrity {}",
+                    dev_major_minor,
+                    data_integrity.to_string()
+                );
+
+                Command::new(INIT_TRUSTED_STORAGE)
+                    .args([&dev_major_minor, &data_integrity.to_string()])
+                    .output()
+                    .expect("Failed to initialize confidential storage");
+            }
+        }
+
         // Both rootfs and volumes (invoked with --volume for instance) will
         // be processed the same way. The idea is to always mount any provided
         // storage to the specified MountPoint, so that it will match what's
@@ -266,14 +341,13 @@ impl AgentService {
         }
 
         // start oom event loop
-        if let Some(ref ctr) = ctr.cgroup_manager {
-            let cg_path = ctr.get_cg_path("memory");
 
-            if let Some(cg_path) = cg_path {
-                let rx = notifier::notify_oom(cid.as_str(), cg_path.to_string()).await?;
+        let cg_path = ctr.cgroup_manager.as_ref().get_cgroup_path("memory");
 
-                s.run_oom_event_monitor(rx, cid.clone()).await;
-            }
+        if let Ok(cg_path) = cg_path {
+            let rx = notifier::notify_oom(cid.as_str(), cg_path.to_string()).await?;
+
+            s.run_oom_event_monitor(rx, cid.clone()).await;
         }
 
         Ok(())
@@ -377,6 +451,7 @@ impl AgentService {
             "signal process";
             "container-id" => cid.clone(),
             "exec-id" => eid.clone(),
+            "signal" => req.signal,
         );
 
         let mut sig: libc::c_int = req.signal as libc::c_int;
@@ -459,11 +534,7 @@ impl AgentService {
         let ctr = sandbox
             .get_container(cid)
             .ok_or_else(|| anyhow!("Invalid container id {}", cid))?;
-        let cm = ctr
-            .cgroup_manager
-            .as_ref()
-            .ok_or_else(|| anyhow!("cgroup manager not exist"))?;
-        cm.freeze(state)?;
+        ctr.cgroup_manager.as_ref().freeze(state)?;
         Ok(())
     }
 
@@ -473,11 +544,7 @@ impl AgentService {
         let ctr = sandbox
             .get_container(cid)
             .ok_or_else(|| anyhow!("Invalid container id {}", cid))?;
-        let cm = ctr
-            .cgroup_manager
-            .as_ref()
-            .ok_or_else(|| anyhow!("cgroup manager not exist"))?;
-        let pids = cm.get_pids()?;
+        let pids = ctr.cgroup_manager.as_ref().get_pids()?;
         Ok(pids)
     }
 
@@ -629,6 +696,54 @@ impl AgentService {
             }
         }
     }
+
+    // When being passed an image name through a container annotation, merge its
+    // corresponding bundle OCI specification into the passed container creation one.
+    async fn merge_bundle_oci(&self, container_oci: &mut oci::Spec) -> Result<()> {
+        if let Some(image_name) = container_oci
+            .annotations
+            .get(&ANNO_K8S_IMAGE_NAME.to_string())
+        {
+            if let Some(container_id) = self.sandbox.clone().lock().await.images.get(image_name) {
+                let image_oci_config_path = Path::new(CONTAINER_BASE)
+                    .join(container_id)
+                    .join(CONFIG_JSON);
+                debug!(
+                    sl!(),
+                    "Image bundle config path: {:?}", image_oci_config_path
+                );
+
+                let image_oci =
+                    oci::Spec::load(image_oci_config_path.to_str().ok_or_else(|| {
+                        anyhow!(
+                            "Invalid container image OCI config path {:?}",
+                            image_oci_config_path
+                        )
+                    })?)
+                    .context("load image bundle")?;
+
+                if let Some(container_root) = container_oci.root.as_mut() {
+                    if let Some(image_root) = image_oci.root.as_ref() {
+                        let root_path = Path::new(CONTAINER_BASE)
+                            .join(container_id)
+                            .join(image_root.path.clone());
+                        container_root.path =
+                            String::from(root_path.to_str().ok_or_else(|| {
+                                anyhow!("Invalid container image root path {:?}", root_path)
+                            })?);
+                    }
+                }
+
+                if let Some(container_process) = container_oci.process.as_mut() {
+                    if let Some(image_process) = image_oci.process.as_ref() {
+                        merge_oci_process(container_process, image_process);
+                    }
+                }
+            }
+        }
+
+        Ok(())
+    }
 }
 
 #[async_trait]
@@ -998,8 +1113,18 @@ impl agent_ttrpc::AgentService for AgentService {
 
         info!(sl!(), "get_ip_tables: request received");
 
+        // the binary could exists in either /usr/sbin or /sbin
+        // here check both of the places and return the one exists
+        // if none exists, return the /sbin one, and the rpc will
+        // returns an internal error
         let cmd = if req.is_ipv6 {
-            IP6TABLES_SAVE
+            if Path::new(USR_IP6TABLES_SAVE).exists() {
+                USR_IP6TABLES_SAVE
+            } else {
+                IP6TABLES_SAVE
+            }
+        } else if Path::new(USR_IPTABLES_SAVE).exists() {
+            USR_IPTABLES_SAVE
         } else {
             IPTABLES_SAVE
         }
@@ -1027,8 +1152,18 @@ impl agent_ttrpc::AgentService for AgentService {
 
         info!(sl!(), "set_ip_tables request received");
 
+        // the binary could exists in both /usr/sbin and /sbin
+        // here check both of the places and return the one exists
+        // if none exists, return the /sbin one, and the rpc will
+        // returns an internal error
         let cmd = if req.is_ipv6 {
-            IP6TABLES_RESTORE
+            if Path::new(USR_IP6TABLES_RESTORE).exists() {
+                USR_IP6TABLES_RESTORE
+            } else {
+                IP6TABLES_RESTORE
+            }
+        } else if Path::new(USR_IPTABLES_RESTORE).exists() {
+            USR_IPTABLES_RESTORE
         } else {
             IPTABLES_RESTORE
         }
@@ -1657,22 +1792,27 @@ async fn read_stream(reader: Arc<Mutex<ReadHalf<PipeStream>>>, l: usize) -> Resu
 }
 
 pub fn start(s: Arc<Mutex<Sandbox>>, server_address: &str) -> Result<TtrpcServer> {
-    let agent_service =
-        Box::new(AgentService { sandbox: s }) as Box<dyn agent_ttrpc::AgentService + Send + Sync>;
-
+    let agent_service = Box::new(AgentService { sandbox: s.clone() })
+        as Box<dyn agent_ttrpc::AgentService + Send + Sync>;
     let agent_worker = Arc::new(agent_service);
 
     let health_service = Box::new(HealthService {}) as Box<dyn health_ttrpc::Health + Send + Sync>;
     let health_worker = Arc::new(health_service);
 
+    let image_service =
+        Box::new(image_rpc::ImageService::new(s)) as Box<dyn image_ttrpc::Image + Send + Sync>;
+
     let aservice = agent_ttrpc::create_agent_service(agent_worker);
 
     let hservice = health_ttrpc::create_health(health_worker);
 
+    let iservice = image_ttrpc::create_image(Arc::new(image_service));
+
     let server = TtrpcServer::new()
         .bind(server_address)?
         .register_service(aservice)
-        .register_service(hservice);
+        .register_service(hservice)
+        .register_service(iservice);
 
     info!(sl!(), "ttRPC server started"; "address" => server_address);
 
@@ -1878,6 +2018,38 @@ fn do_copy_file(req: &CopyFileRequest) -> Result<()> {
 
     std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(req.dir_mode))?;
 
+    let sflag = stat::SFlag::from_bits_truncate(req.file_mode);
+
+    if sflag.contains(stat::SFlag::S_IFDIR) {
+        fs::create_dir(path.clone()).or_else(|e| {
+            if e.kind() != std::io::ErrorKind::AlreadyExists {
+                return Err(e);
+            }
+            Ok(())
+        })?;
+
+        std::fs::set_permissions(path.clone(), std::fs::Permissions::from_mode(req.file_mode))?;
+
+        unistd::chown(
+            &path,
+            Some(Uid::from_raw(req.uid as u32)),
+            Some(Gid::from_raw(req.gid as u32)),
+        )?;
+
+        return Ok(());
+    }
+
+    if sflag.contains(stat::SFlag::S_IFLNK) {
+        let src = PathBuf::from(String::from_utf8(req.data.clone()).unwrap());
+
+        unistd::symlinkat(&src, None, &path)?;
+        let path_str = CString::new(path.to_str().unwrap())?;
+        let ret = unsafe { libc::lchown(path_str.as_ptr(), req.uid as u32, req.gid as u32) };
+        Errno::result(ret).map(drop)?;
+
+        return Ok(());
+    }
+
     let mut tmpfile = path.clone();
     tmpfile.set_extension("tmp");
 
@@ -1943,18 +2115,26 @@ pub fn setup_bundle(cid: &str, spec: &mut Spec) -> Result<PathBuf> {
     let spec_root_path = Path::new(&spec_root.path);
 
     let bundle_path = Path::new(CONTAINER_BASE).join(cid);
-    let config_path = bundle_path.join("config.json");
+    let config_path = bundle_path.join(CONFIG_JSON);
     let rootfs_path = bundle_path.join("rootfs");
 
-    fs::create_dir_all(&rootfs_path)?;
-    baremount(
-        spec_root_path,
-        &rootfs_path,
-        "bind",
-        MsFlags::MS_BIND,
-        "",
-        &sl!(),
-    )?;
+    let rootfs_exists = Path::new(&rootfs_path).exists();
+    info!(
+        sl!(),
+        "The rootfs_path is {:?} and exists: {}", rootfs_path, rootfs_exists
+    );
+
+    if !rootfs_exists {
+        fs::create_dir_all(&rootfs_path)?;
+        baremount(
+            spec_root_path,
+            &rootfs_path,
+            "bind",
+            MsFlags::MS_BIND,
+            "",
+            &sl!(),
+        )?;
+    }
 
     let rootfs_path_name = rootfs_path
         .to_str()
@@ -2756,16 +2936,25 @@ OtherField:other
     async fn test_ip_tables() {
         skip_if_not_root!();
 
-        if !check_command(IPTABLES_SAVE)
-            || !check_command(IPTABLES_RESTORE)
-            || !check_command(IP6TABLES_SAVE)
-            || !check_command(IP6TABLES_RESTORE)
-        {
-            warn!(
-                sl!(),
-                "one or more commands for ip tables test are missing, skip it"
-            );
-            return;
+        let iptables_cmd_list = [
+            USR_IPTABLES_SAVE,
+            USR_IP6TABLES_SAVE,
+            USR_IPTABLES_RESTORE,
+            USR_IP6TABLES_RESTORE,
+            IPTABLES_SAVE,
+            IP6TABLES_SAVE,
+            IPTABLES_RESTORE,
+            IP6TABLES_RESTORE,
+        ];
+
+        for cmd in iptables_cmd_list {
+            if !check_command(cmd) {
+                warn!(
+                    sl!(),
+                    "one or more commands for ip tables test are missing, skip it"
+                );
+                return;
+            }
         }
 
         let logger = slog::Logger::root(slog::Discard, o!());
@@ -2877,7 +3066,7 @@ COMMIT
             .unwrap();
         assert!(!result.data.is_empty(), "we should have non-zero output:");
         assert!(
-            std::str::from_utf8(&*result.data).unwrap().contains(
+            std::str::from_utf8(&result.data).unwrap().contains(
                 "PREROUTING -d 192.168.103.153/32 -j DNAT --to-destination 192.168.188.153"
             ),
             "We should see the resulting rule"
@@ -2915,10 +3104,141 @@ COMMIT
             .unwrap();
         assert!(!result.data.is_empty(), "we should have non-zero output:");
         assert!(
-            std::str::from_utf8(&*result.data)
+            std::str::from_utf8(&result.data)
                 .unwrap()
                 .contains("INPUT -s 2001:db8:100::1/128 -i sit+ -p tcp -m tcp --sport 512:65535"),
             "We should see the resulting rule"
         );
     }
+
+    #[tokio::test]
+    async fn test_merge_cwd() {
+        #[derive(Debug)]
+        struct TestData<'a> {
+            container_process_cwd: &'a str,
+            image_process_cwd: &'a str,
+            expected: &'a str,
+        }
+
+        let tests = &[
+            // Image cwd should override blank container cwd
+            // TODO - how can we tell the user didn't specifically set it to `/` vs not setting at all? Is that scenario valid?
+            TestData {
+                container_process_cwd: "/",
+                image_process_cwd: "/imageDir",
+                expected: "/imageDir",
+            },
+            // Container cwd should override image cwd
+            TestData {
+                container_process_cwd: "/containerDir",
+                image_process_cwd: "/imageDir",
+                expected: "/containerDir",
+            },
+            // Container cwd should override blank image cwd
+            TestData {
+                container_process_cwd: "/containerDir",
+                image_process_cwd: "/",
+                expected: "/containerDir",
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let mut container_process = oci::Process {
+                cwd: d.container_process_cwd.to_string(),
+                ..Default::default()
+            };
+
+            let image_process = oci::Process {
+                cwd: d.image_process_cwd.to_string(),
+                ..Default::default()
+            };
+
+            merge_oci_process(&mut container_process, &image_process);
+
+            assert_eq!(d.expected, container_process.cwd, "{}", msg);
+        }
+    }
+
+    #[tokio::test]
+    async fn test_merge_env() {
+        #[derive(Debug)]
+        struct TestData {
+            container_process_env: Vec<String>,
+            image_process_env: Vec<String>,
+            expected: Vec<String>,
+        }
+
+        let tests = &[
+            // Test that the pods environment overrides the images
+            TestData {
+                container_process_env: vec!["ISPRODUCTION=true".to_string()],
+                image_process_env: vec!["ISPRODUCTION=false".to_string()],
+                expected: vec!["ISPRODUCTION=true".to_string()],
+            },
+            // Test that multiple environment variables can be overrided
+            TestData {
+                container_process_env: vec![
+                    "ISPRODUCTION=true".to_string(),
+                    "ISDEVELOPMENT=false".to_string(),
+                ],
+                image_process_env: vec![
+                    "ISPRODUCTION=false".to_string(),
+                    "ISDEVELOPMENT=true".to_string(),
+                ],
+                expected: vec![
+                    "ISPRODUCTION=true".to_string(),
+                    "ISDEVELOPMENT=false".to_string(),
+                ],
+            },
+            // Test that when none of the variables match do not override them
+            TestData {
+                container_process_env: vec!["ANOTHERENV=TEST".to_string()],
+                image_process_env: vec![
+                    "ISPRODUCTION=false".to_string(),
+                    "ISDEVELOPMENT=true".to_string(),
+                ],
+                expected: vec![
+                    "ANOTHERENV=TEST".to_string(),
+                    "ISPRODUCTION=false".to_string(),
+                    "ISDEVELOPMENT=true".to_string(),
+                ],
+            },
+            // Test a mix of both overriding and not
+            TestData {
+                container_process_env: vec![
+                    "ANOTHERENV=TEST".to_string(),
+                    "ISPRODUCTION=true".to_string(),
+                ],
+                image_process_env: vec![
+                    "ISPRODUCTION=false".to_string(),
+                    "ISDEVELOPMENT=true".to_string(),
+                ],
+                expected: vec![
+                    "ANOTHERENV=TEST".to_string(),
+                    "ISPRODUCTION=true".to_string(),
+                    "ISDEVELOPMENT=true".to_string(),
+                ],
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let mut container_process = oci::Process {
+                env: d.container_process_env.clone(),
+                ..Default::default()
+            };
+
+            let image_process = oci::Process {
+                env: d.image_process_env.clone(),
+                ..Default::default()
+            };
+
+            merge_oci_process(&mut container_process, &image_process);
+
+            assert_eq!(d.expected, container_process.env, "{}", msg);
+        }
+    }
 }

src/agent/src/sandbox.rs

@@ -60,6 +60,7 @@ pub struct Sandbox {
     pub event_tx: Option<Sender<String>>,
     pub bind_watcher: BindWatcher,
     pub pcimap: HashMap<pci::Address, pci::Address>,
+    pub images: HashMap<String, String>,
 }
 
 impl Sandbox {
@@ -93,6 +94,7 @@ impl Sandbox {
             event_tx: Some(tx),
             bind_watcher: BindWatcher::new(),
             pcimap: HashMap::new(),
+            images: HashMap::new(),
         })
     }
 
@@ -296,7 +298,6 @@ impl Sandbox {
             info!(self.logger, "updating {}", ctr.id.as_str());
             ctr.cgroup_manager
                 .as_ref()
-                .unwrap()
                 .update_cpuset_path(guest_cpuset.as_str(), container_cpust)?;
         }
 
@@ -327,7 +328,7 @@ impl Sandbox {
             // Reject non-file, symlinks and non-executable files
             if !entry.file_type()?.is_file()
                 || entry.file_type()?.is_symlink()
-                || entry.metadata()?.permissions().mode() & 0o777 & 0o111 == 0
+                || entry.metadata()?.permissions().mode() & 0o111 == 0
             {
                 continue;
             }
@@ -1073,7 +1074,7 @@ mod tests {
                 fs::create_dir(&subdir_path).unwrap();
                 for file in j.files {
                     let subfile_path = format!("{}/{}", subdir_path, file.name);
-                    let mut subfile = File::create(&subfile_path).unwrap();
+                    let mut subfile = File::create(subfile_path).unwrap();
                     subfile.write_all(file.content.as_bytes()).unwrap();
                 }
             }

src/agent/src/signal.rs

@@ -24,7 +24,7 @@ async fn handle_sigchild(logger: Logger, sandbox: Arc<Mutex<Sandbox>>) -> Result
     loop {
         // Avoid reaping the undesirable child's signal, e.g., execute_hook's
         // The lock should be released immediately.
-        rustjail::container::WAIT_PID_LOCKER.lock().await;
+        let _ = rustjail::container::WAIT_PID_LOCKER.lock().await;
         let result = wait::waitpid(
             Some(Pid::from_raw(-1)),
             Some(WaitPidFlag::WNOHANG | WaitPidFlag::__WALL),

src/agent/src/watcher.rs

@@ -11,7 +11,7 @@ use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::SystemTime;
 
-use anyhow::{ensure, Context, Result};
+use anyhow::{anyhow, ensure, Context, Result};
 use async_recursion::async_recursion;
 use nix::mount::{umount, MsFlags};
 use nix::unistd::{Gid, Uid};
@@ -34,9 +34,13 @@ const MAX_SIZE_PER_WATCHABLE_MOUNT: u64 = 1024 * 1024;
 /// How often to check for modified files.
 const WATCH_INTERVAL_SECS: u64 = 2;
 
-/// Destination path for tmpfs
+/// Destination path for tmpfs, which used by the golang runtime
 const WATCH_MOUNT_POINT_PATH: &str = "/run/kata-containers/shared/containers/watchable/";
 
+/// Destination path for tmpfs for runtime-rs passthrough file sharing
+const WATCH_MOUNT_POINT_PATH_PASSTHROUGH: &str =
+    "/run/kata-containers/shared/containers/passthrough/watchable/";
+
 /// Represents a single watched storage entry which may have multiple files to watch.
 #[derive(Default, Debug, Clone)]
 struct Storage {
@@ -120,7 +124,7 @@ impl Storage {
 
         // if we are creating a directory: just create it, nothing more to do
         if metadata.file_type().is_dir() {
-            let dest_file_path = self.make_target_path(&source_file_path)?;
+            let dest_file_path = self.make_target_path(source_file_path)?;
 
             fs::create_dir_all(&dest_file_path)
                 .await
@@ -148,7 +152,7 @@ impl Storage {
             // Assume target mount is a file path
             self.target_mount_point.clone()
         } else {
-            let dest_file_path = self.make_target_path(&source_file_path)?;
+            let dest_file_path = self.make_target_path(source_file_path)?;
 
             if let Some(path) = dest_file_path.parent() {
                 debug!(logger, "Creating destination directory: {}", path.display());
@@ -451,7 +455,7 @@ impl BindWatcher {
     ) -> Result<()> {
         if self.watch_thread.is_none() {
             // Virtio-fs shared path is RO by default, so we back the target-mounts by tmpfs.
-            self.mount(logger).await?;
+            self.mount(logger).await.context("mount watch directory")?;
 
             // Spawn background thread to monitor changes
             self.watch_thread = Some(Self::spawn_watcher(
@@ -500,16 +504,28 @@ impl BindWatcher {
     }
 
     async fn mount(&self, logger: &Logger) -> Result<()> {
-        fs::create_dir_all(WATCH_MOUNT_POINT_PATH).await?;
+        // the watchable directory is created on the host side.
+        // here we can only check if it exist.
+        // first we will check the default WATCH_MOUNT_POINT_PATH,
+        // and then check WATCH_MOUNT_POINT_PATH_PASSTHROUGH
+        // in turn which are introduced by runtime-rs file sharing.
+        let watchable_dir = if Path::new(WATCH_MOUNT_POINT_PATH).is_dir() {
+            WATCH_MOUNT_POINT_PATH
+        } else if Path::new(WATCH_MOUNT_POINT_PATH_PASSTHROUGH).is_dir() {
+            WATCH_MOUNT_POINT_PATH_PASSTHROUGH
+        } else {
+            return Err(anyhow!("watchable mount source not found"));
+        };
 
         baremount(
             Path::new("tmpfs"),
-            Path::new(WATCH_MOUNT_POINT_PATH),
+            Path::new(watchable_dir),
             "tmpfs",
             MsFlags::empty(),
             "",
             logger,
-        )?;
+        )
+        .context("baremount watchable mount path")?;
 
         Ok(())
     }
@@ -520,7 +536,12 @@ impl BindWatcher {
             handle.abort();
         }
 
-        let _ = umount(WATCH_MOUNT_POINT_PATH);
+        // try umount watchable mount path in turn
+        if Path::new(WATCH_MOUNT_POINT_PATH).is_dir() {
+            let _ = umount(WATCH_MOUNT_POINT_PATH);
+        } else if Path::new(WATCH_MOUNT_POINT_PATH_PASSTHROUGH).is_dir() {
+            let _ = umount(WATCH_MOUNT_POINT_PATH_PASSTHROUGH);
+        }
     }
 }
 
@@ -529,6 +550,7 @@ mod tests {
     use super::*;
     use crate::mount::is_mounted;
     use nix::unistd::{Gid, Uid};
+    use scopeguard::defer;
     use std::fs;
     use std::thread;
     use test_utils::skip_if_not_root;
@@ -756,7 +778,7 @@ mod tests {
             22
         );
         assert_eq!(
-            fs::read_to_string(&entries.0[0].target_mount_point.as_path().join("1.txt")).unwrap(),
+            fs::read_to_string(entries.0[0].target_mount_point.as_path().join("1.txt")).unwrap(),
             "updated"
         );
 
@@ -801,7 +823,7 @@ mod tests {
             2
         );
         assert_eq!(
-            fs::read_to_string(&entries.0[1].target_mount_point.as_path().join("foo.txt")).unwrap(),
+            fs::read_to_string(entries.0[1].target_mount_point.as_path().join("foo.txt")).unwrap(),
             "updated"
         );
 
@@ -978,7 +1000,7 @@ mod tests {
 
         // create a path we'll remove later
         fs::create_dir_all(source_dir.path().join("tmp")).unwrap();
-        fs::write(&source_dir.path().join("tmp/test-file"), "foo").unwrap();
+        fs::write(source_dir.path().join("tmp/test-file"), "foo").unwrap();
         assert_eq!(entry.scan(&logger).await.unwrap(), 3); // root, ./tmp, test-file
 
         // Verify expected directory, file:
@@ -1269,19 +1291,26 @@ mod tests {
 
     #[tokio::test]
     #[serial]
+    #[cfg(not(target_arch = "aarch64"))]
     async fn create_tmpfs() {
         skip_if_not_root!();
 
         let logger = slog::Logger::root(slog::Discard, o!());
         let mut watcher = BindWatcher::default();
 
-        watcher.mount(&logger).await.unwrap();
-        assert!(is_mounted(WATCH_MOUNT_POINT_PATH).unwrap());
+        for mount_point in [WATCH_MOUNT_POINT_PATH, WATCH_MOUNT_POINT_PATH_PASSTHROUGH] {
+            fs::create_dir_all(mount_point).unwrap();
+            // ensure the watchable directory is deleted.
+            defer!(fs::remove_dir_all(mount_point).unwrap());
 
-        thread::sleep(Duration::from_millis(20));
+            watcher.mount(&logger).await.unwrap();
+            assert!(is_mounted(mount_point).unwrap());
 
-        watcher.cleanup();
-        assert!(!is_mounted(WATCH_MOUNT_POINT_PATH).unwrap());
+            thread::sleep(Duration::from_millis(20));
+
+            watcher.cleanup();
+            assert!(!is_mounted(mount_point).unwrap());
+        }
     }
 
     #[tokio::test]
@@ -1289,6 +1318,10 @@ mod tests {
     async fn spawn_thread() {
         skip_if_not_root!();
 
+        fs::create_dir_all(WATCH_MOUNT_POINT_PATH).unwrap();
+        // ensure the watchable directory is deleted.
+        defer!(fs::remove_dir_all(WATCH_MOUNT_POINT_PATH).unwrap());
+
         let source_dir = tempfile::tempdir().unwrap();
         fs::write(source_dir.path().join("1.txt"), "one").unwrap();
 
@@ -1319,6 +1352,10 @@ mod tests {
     async fn verify_container_cleanup_watching() {
         skip_if_not_root!();
 
+        fs::create_dir_all(WATCH_MOUNT_POINT_PATH).unwrap();
+        // ensure the watchable directory is deleted.
+        defer!(fs::remove_dir_all(WATCH_MOUNT_POINT_PATH).unwrap());
+
         let source_dir = tempfile::tempdir().unwrap();
         fs::write(source_dir.path().join("1.txt"), "one").unwrap();
 

src/dragonball/Cargo.toml

@@ -12,21 +12,21 @@ edition = "2018"
 [dependencies]
 arc-swap = "1.5.0"
 bytes = "1.1.0"
-dbs-address-space = "0.1.0"
+dbs-address-space = "0.2.0"
 dbs-allocator = "0.1.0"
-dbs-arch = "0.1.0"
-dbs-boot = "0.2.0"
-dbs-device = "0.1.0"
-dbs-interrupt = { version = "0.1.0", features = ["kvm-irq"] }
+dbs-arch = "0.2.0"
+dbs-boot = "0.3.0"
+dbs-device = "0.2.0"
+dbs-interrupt = { version = "0.2.0", features = ["kvm-irq"] }
 dbs-legacy-devices = "0.1.0"
 dbs-upcall = { version = "0.1.0", optional = true }
-dbs-utils = "0.1.0"
+dbs-utils = "0.2.0"
 dbs-virtio-devices = { version = "0.1.0", optional = true, features = ["virtio-mmio"] }
 kvm-bindings = "0.5.0"
 kvm-ioctls = "0.11.0"
 lazy_static = "1.2"
 libc = "0.2.39"
-linux-loader = "0.4.0"
+linux-loader = "0.6.0"
 log = "0.4.14"
 nix = "0.24.2"
 seccompiler = "0.2.0"
@@ -36,30 +36,21 @@ serde_json = "1.0.9"
 slog = "2.5.2"
 slog-scope = "4.4.0"
 thiserror = "1"
-vmm-sys-util = "0.9.0"
-virtio-queue = { version = "0.1.0", optional = true }
-vm-memory = { version = "0.7.0", features = ["backend-mmap"] }
+vmm-sys-util = "0.11.0"
+virtio-queue = { version = "0.4.0", optional = true }
+vm-memory = { version = "0.9.0", features = ["backend-mmap"] }
 
 [dev-dependencies]
 slog-term = "2.9.0"
 slog-async = "2.7.0"
+test-utils = { path = "../libs/test-utils" }
 
 [features]
 acpi = []
-atomic-guest-memory = []
+atomic-guest-memory = [ "vm-memory/backend-atomic" ]
 hotplug = ["virtio-vsock"]
 virtio-vsock = ["dbs-virtio-devices/virtio-vsock", "virtio-queue"]
 virtio-blk = ["dbs-virtio-devices/virtio-blk", "virtio-queue"]
 virtio-net = ["dbs-virtio-devices/virtio-net", "virtio-queue"]
 # virtio-fs only work on atomic-guest-memory
 virtio-fs = ["dbs-virtio-devices/virtio-fs", "virtio-queue", "atomic-guest-memory"]
-
-[patch.'crates-io']
-dbs-device = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" }
-dbs-interrupt = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" }
-dbs-legacy-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" }
-dbs-upcall = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" }
-dbs-utils = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" }
-dbs-virtio-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" }
-dbs-boot = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" }
-dbs-arch = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" }

src/dragonball/Makefile

@@ -2,12 +2,22 @@
 # Copyright (c) 2019-2022 Ant Group. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+include ../../utils.mk
+
+ifeq ($(ARCH), s390x)
+default build check test clippy:
+	@echo "s390x not support currently"
+	exit 0
+else
+
 default: build
 
 build:
-	# FIXME: This line will be removed when we solve the vm-memory dependency problem in Dragonball Sandbox
-	cargo update -p vm-memory:0.8.0 --precise 0.7.0
-	cargo build --all-features
+	@echo "INFO: cargo build..."
+	cargo build --all-features --target $(TRIPLE)
+
+static-checks-build:
+	@echo "INFO: static-checks-build do nothing.."
 
 check: clippy format
 
@@ -17,6 +27,9 @@ clippy:
 		-- \
 		-D warnings
 
+vendor:
+	@echo "INFO: vendor do nothing.."
+
 format:
 	@echo "INFO: cargo fmt..."
 	cargo fmt -- --check
@@ -25,5 +38,13 @@ clean:
 	cargo clean
 
 test:
-	@echo "INFO: testing dragonball for development build"
-	cargo test --all-features -- --nocapture
+ifdef SUPPORT_VIRTUALIZATION
+	cargo test --all-features --target $(TRIPLE) -- --nocapture
+else
+	@echo "INFO: skip testing dragonball, it need virtualization support."
+	exit 0
+endif
+
+endif # ifeq ($(ARCH), s390x)
+
+.DEFAULT_GOAL := default

src/dragonball/README.md

@@ -19,6 +19,7 @@ and configuration process.
 Device: [Device Document](docs/device.md)
 vCPU: [vCPU Document](docs/vcpu.md)
 API: [API Document](docs/api.md)
+`Upcall`: [`Upcall` Document](docs/upcall.md)
 
 Currently, the documents are still actively adding.
 You could see the [official documentation](docs/) page for more details.

src/dragonball/docs/images/upcall-architecture.svg

@@ -0,0 +1,177 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="51 242 818 479" width="818" height="479">
+  <defs>
+    <marker orient="auto" overflow="visible" markerUnits="strokeWidth" id="FilledArrow_Marker" stroke-linejoin="miter" stroke-miterlimit="10" viewBox="-1 -4 10 8" markerWidth="10" markerHeight="8" color="black">
+      <g>
+        <path d="M 8 0 L 0 -3 L 0 3 Z" fill="currentColor" stroke="currentColor" stroke-width="1"/>
+      </g>
+    </marker>
+    <marker orient="auto" overflow="visible" markerUnits="strokeWidth" id="FilledArrow_Marker_2" stroke-linejoin="miter" stroke-miterlimit="10" viewBox="-9 -4 10 8" markerWidth="10" markerHeight="8" color="black">
+      <g>
+        <path d="M -8 0 L 0 3 L 0 -3 Z" fill="currentColor" stroke="currentColor" stroke-width="1"/>
+      </g>
+    </marker>
+  </defs>
+  <g id="Canvas_1" fill="none" fill-opacity="1" stroke="none" stroke-opacity="1" stroke-dasharray="none">
+    <title>Canvas 1</title>
+    <rect fill="white" x="51" y="242" width="818" height="479"/>
+    <g id="Canvas_1_Layer_1">
+      <title>Layer 1</title>
+      <g id="Line_4">
+        <line x1="153" y1="279.5" x2="856.1097" y2="279.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
+      </g>
+      <g id="Graphic_5">
+        <text transform="translate(56 247.5)" fill="black">
+          <tspan font-family="Alibaba PuHuiTi" font-size="16" fill="black" x="0" y="17">Guest User</tspan>
+        </text>
+      </g>
+      <g id="Graphic_6">
+        <text transform="translate(56 286)" fill="black">
+          <tspan font-family="Alibaba PuHuiTi" font-size="16" fill="black" x="0" y="17">Guest Kernel</tspan>
+        </text>
+      </g>
+      <g id="Line_7">
+        <line x1="153" y1="592" x2="856.1097" y2="592" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
+      </g>
+      <g id="Graphic_8">
+        <text transform="translate(62.76 597.5)" fill="black">
+          <tspan font-family="Alibaba PuHuiTi" font-size="16" fill="black" x="7531753e-19" y="17">Hypervisor</tspan>
+        </text>
+      </g>
+      <g id="Graphic_10">
+        <path d="M 264 328 L 347.456 328 C 354.0834 328 359.456 333.3726 359.456 340 L 359.456 524.5 C 359.456 531.1274 354.0834 536.5 347.456 536.5 L 264 536.5 C 257.37258 536.5 252 531.1274 252 524.5 L 252 340 C 252 333.3726 257.37258 328 264 328 Z" stroke="gray" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
+      </g>
+      <g id="Graphic_11">
+        <text transform="translate(276.776 333)" fill="black">
+          <tspan font-family="Alibaba PuHuiTi" font-size="16" fill="black" x="0" y="17">socket</tspan>
+        </text>
+      </g>
+      <g id="Graphic_12">
+        <path d="M 582 294.5 L 672 294.5 C 678.6274 294.5 684 299.8726 684 306.5 L 684 354 C 684 360.6274 678.6274 366 672 366 L 582 366 C 575.3726 366 570 360.6274 570 354 L 570 306.5 C 570 299.8726 575.3726 294.5 582 294.5 Z" stroke="gray" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
+        <text transform="translate(575 302.578)" fill="black">
+          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="27.704" y="15">Device </tspan>
+          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="20.44" y="33.448">Manager</tspan>
+          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="25.488" y="51.895996">Service</tspan>
+        </text>
+      </g>
+      <g id="Graphic_13">
+        <text transform="translate(284.824 374)" fill="black">
+          <tspan font-family="Alibaba PuHuiTi" font-size="16" fill="black" x="8135714e-19" y="17">bind</tspan>
+        </text>
+      </g>
+      <g id="Graphic_14">
+        <text transform="translate(280.528 416.25)" fill="black">
+          <tspan font-family="Alibaba PuHuiTi" font-size="16" fill="black" x="0" y="17">listen</tspan>
+        </text>
+      </g>
+      <g id="Graphic_15">
+        <text transform="translate(274.92 459.5)" fill="black">
+          <tspan font-family="Alibaba PuHuiTi" font-size="16" fill="black" x="0" y="17">accept</tspan>
+        </text>
+      </g>
+      <g id="Graphic_16">
+        <text transform="translate(256.372 503.5)" fill="black">
+          <tspan font-family="Alibaba PuHuiTi" font-size="16" fill="black" x="8668621e-19" y="17">new kthread</tspan>
+        </text>
+      </g>
+      <g id="Graphic_17">
+        <path d="M 268 566.5 L 807.5 566.5 C 813.0228 566.5 817.5 570.97715 817.5 576.5 L 817.5 576.5 C 817.5 582.02285 813.0228 586.5 807.5 586.5 L 268 586.5 C 262.47715 586.5 258 582.02285 258 576.5 L 258 576.5 C 258 570.97715 262.47715 566.5 268 566.5 Z" stroke="gray" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
+        <text transform="translate(263 567.276)" fill="black">
+          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="226.454" y="15">virtio-vsocket</tspan>
+        </text>
+      </g>
+      <g id="Graphic_18">
+        <path d="M 268 598.5 L 807.5 598.5 C 813.0228 598.5 817.5 602.97715 817.5 608.5 L 817.5 608.5 C 817.5 614.02285 813.0228 618.5 807.5 618.5 L 268 618.5 C 262.47715 618.5 258 614.02285 258 608.5 L 258 608.5 C 258 602.97715 262.47715 598.5 268 598.5 Z" stroke="gray" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
+        <text transform="translate(263 599.276)" fill="black">
+          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="193.254" y="15">virtio-vsocket backend</tspan>
+        </text>
+      </g>
+      <g id="Line_20">
+        <line x1="301.9" y1="352" x2="301.9" y2="369.84976" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
+      </g>
+      <g id="Line_21">
+        <line x1="300.828" y1="394.6251" x2="300.828" y2="412.4749" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
+      </g>
+      <g id="Line_22">
+        <line x1="300.828" y1="437.56256" x2="300.828" y2="455.4123" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
+      </g>
+      <g id="Line_23">
+        <line x1="299.9" y1="480.1251" x2="299.9" y2="497.9749" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
+      </g>
+      <g id="Graphic_24">
+        <rect x="266.5" y="541.5" width="71.188" height="20" stroke="gray" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
+        <text transform="translate(271.5 540.5)" fill="black">
+          <tspan font-family="Alibaba PuHuiTi" font-size="16" fill="black" x="13.858" y="17">Port</tspan>
+        </text>
+      </g>
+      <g id="Graphic_27">
+        <path d="M 582 648.5 L 672 648.5 C 678.6274 648.5 684 653.8726 684 660.5 L 684 708 C 684 714.6274 678.6274 720 672 720 L 582 720 C 575.3726 720 570 714.6274 570 708 L 570 660.5 C 570 653.8726 575.3726 648.5 582 648.5 Z" stroke="gray" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
+        <text transform="translate(575 656.578)" fill="black">
+          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="27.704" y="15">Device </tspan>
+          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="20.44" y="33.448">Manager</tspan>
+          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="20.288" y="51.895996">Backend</tspan>
+        </text>
+      </g>
+      <g id="Line_28">
+        <line x1="627" y1="375.9" x2="627" y2="638.6" marker-end="url(#FilledArrow_Marker)" marker-start="url(#FilledArrow_Marker_2)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
+      </g>
+      <g id="Graphic_31">
+        <path d="M 711 294.5 L 801 294.5 C 807.6274 294.5 813 299.8726 813 306.5 L 813 354 C 813 360.6274 807.6274 366 801 366 L 711 366 C 704.3726 366 699 360.6274 699 354 L 699 306.5 C 699 299.8726 704.3726 294.5 711 294.5 Z" stroke="gray" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
+        <text transform="translate(704 321.026)" fill="black">
+          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="17.784" y="15">Service B</tspan>
+        </text>
+      </g>
+      <g id="Graphic_30">
+        <path d="M 711 648.5 L 801 648.5 C 807.6274 648.5 813 653.8726 813 660.5 L 813 708 C 813 714.6274 807.6274 720 801 720 L 711 720 C 704.3726 720 699 714.6274 699 708 L 699 660.5 C 699 653.8726 704.3726 648.5 711 648.5 Z" stroke="gray" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
+        <text transform="translate(704 675.026)" fill="black">
+          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="12.584" y="15">Backend B</tspan>
+        </text>
+      </g>
+      <g id="Line_29">
+        <line x1="756" y1="375.9" x2="756" y2="638.6" marker-end="url(#FilledArrow_Marker)" marker-start="url(#FilledArrow_Marker_2)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
+      </g>
+      <g id="Graphic_32">
+        <text transform="translate(833 319.25)" fill="black">
+          <tspan font-family="Alibaba PuHuiTi" font-size="16" fill="black" x="58264504e-20" y="17">……</tspan>
+        </text>
+      </g>
+      <g id="Graphic_33">
+        <text transform="translate(833 673.25)" fill="black">
+          <tspan font-family="Alibaba PuHuiTi" font-size="16" fill="black" x="58264504e-20" y="17">……</tspan>
+        </text>
+      </g>
+      <g id="Graphic_34">
+        <text transform="translate(252.616 296)" fill="black">
+          <tspan font-family="Alibaba PuHuiTi" font-size="16" fill="black" x="0" y="17">Upcall Server</tspan>
+        </text>
+      </g>
+      <g id="Line_39">
+        <path d="M 251.372 514.94444 L 196.16455 515.40173 L 196.2135 443.25 L 290.92825 443.92903" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
+      </g>
+      <g id="Graphic_41">
+        <text transform="translate(417 503.5)" fill="black">
+          <tspan font-family="Alibaba PuHuiTi" font-size="16" fill="black" x="0" y="17">Service handler</tspan>
+        </text>
+      </g>
+      <g id="Graphic_42">
+        <rect x="591.406" y="540.4723" width="71.188" height="20" stroke="gray" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
+        <text transform="translate(596.406 539.4723)" fill="black">
+          <tspan font-family="Alibaba PuHuiTi" font-size="16" fill="black" x="10.386" y="17">Conn</tspan>
+        </text>
+      </g>
+      <g id="Graphic_43">
+        <rect x="720.406" y="541.4723" width="71.188" height="20" stroke="gray" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
+        <text transform="translate(725.406 540.4723)" fill="black">
+          <tspan font-family="Alibaba PuHuiTi" font-size="16" fill="black" x="10.386" y="17">Conn</tspan>
+        </text>
+      </g>
+      <g id="Line_44">
+        <line x1="358.684" y1="514.5" x2="402.1" y2="514.5" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
+      </g>
+      <g id="Line_46">
+        <path d="M 479.2467 498.5 L 480 328 L 560.10116 329.22604" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
+      </g>
+    </g>
+  </g>
+</svg>