Merge pull request #6930 from fidencio/3.2.0-alpha2-branch-bump

# Kata Containers 3.2.0-alpha2

.github/workflows/add-backport-label.yaml

@@ -62,15 +62,15 @@ jobs:
           has_backport_needed_label=${{ contains(github.event.pull_request.labels.*.name, 'needs-backport') }}
           has_no_backport_needed_label=${{ contains(github.event.pull_request.labels.*.name, 'no-backport-needed') }}
 
-          echo "::set-output name=add_backport_label::false"
+          echo "add_backport_label=false" >> $GITHUB_OUTPUT
           if [ $has_backport_needed_label  = true ] || [ $has_bug  = true ]; then
             if [[ $has_no_backport_needed_label = false ]]; then
-              echo "::set-output name=add_backport_label::true"
+              echo "add_backport_label=true" >> $GITHUB_OUTPUT
             fi
           fi
 
           # Do not spam comment, only if auto-backport label is going to be newly added.
-          echo "::set-output name=auto_backport_added::$CONTAINS_AUTO_BACKPORT"
+          echo "auto_backport_added=$CONTAINS_AUTO_BACKPORT" >> $GITHUB_OUTPUT
 
       - name: Add comment
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && steps.add_label.outputs.add_backport_label == 'true' && steps.add_label.outputs.auto_backport_added == 'false' }}
@@ -97,4 +97,4 @@ jobs:
         uses: andymckay/labeler@e6c4322d0397f3240f0e7e30a33b5c5df2d39e90
         with:
           add-labels: "auto-backport"
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/build-kata-static-tarball-amd64.yaml

@@ -0,0 +1,94 @@
+name: CI | Build kata-static tarball for amd64
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      push-to-registry:
+        required: false
+        type: string
+        default: no
+
+jobs:
+  build-asset:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        asset:
+          - cloud-hypervisor
+          - firecracker
+          - kernel
+          - kernel-sev
+          - kernel-dragonball-experimental
+          - kernel-tdx-experimental
+          - kernel-nvidia-gpu
+          - kernel-nvidia-gpu-snp
+          - kernel-nvidia-gpu-tdx-experimental
+          - nydus
+          - ovmf
+          - ovmf-sev
+          - qemu
+          - qemu-snp-experimental
+          - qemu-tdx-experimental
+          - rootfs-image
+          - rootfs-initrd
+          - rootfs-initrd-sev
+          - shim-v2
+          - tdvf
+          - virtiofsd
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-amd64${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: build-asset
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error

.github/workflows/build-kata-static-tarball-arm64.yaml

@@ -0,0 +1,90 @@
+name: CI | Build kata-static tarball for arm64
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      push-to-registry:
+        required: false
+        type: string
+        default: no
+
+jobs:
+  build-asset:
+    runs-on: arm64
+    strategy:
+      matrix:
+        asset:
+          - cloud-hypervisor
+          - firecracker
+          - kernel
+          - kernel-dragonball-experimental
+          - nydus
+          - qemu
+          - rootfs-image
+          - rootfs-initrd
+          - shim-v2
+          - virtiofsd
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-arm64${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: arm64
+    needs: build-asset
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts-arm64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball-arm64${{ inputs.tarball-suffix }}
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error

.github/workflows/build-kata-static-tarball-s390x.yaml

@@ -0,0 +1,87 @@
+name: CI | Build kata-static tarball for s390x
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      push-to-registry:
+        required: false
+        type: string
+        default: no
+
+jobs:
+  build-asset:
+    runs-on: s390x
+    strategy:
+      matrix:
+        asset:
+          - kernel
+          - qemu
+          - rootfs-image
+          - rootfs-initrd
+          - shim-v2
+          - virtiofsd
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+          sudo chown -R $(id -u):$(id -g) "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-s390x${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: s390x
+    needs: build-asset
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts-s390x${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball-s390x${{ inputs.tarball-suffix }}
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error

.github/workflows/cargo-deny-runner.yaml

@@ -1,5 +1,12 @@
 name: Cargo Crates Check Runner
-on: [pull_request]
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+    paths-ignore: [ '**.md', '**.png', '**.jpg', '**.jpeg', '**.svg', '/docs/**' ]
 jobs:
   cargo-deny-runner:
     runs-on: ubuntu-latest

.github/workflows/ci-on-push.yaml

@@ -0,0 +1,54 @@
+name: Kata Containers CI
+on:
+  pull_request_target:
+    branches:
+      - 'main'
+
+jobs:
+  build-kata-static-tarball-amd64:
+    uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
+    with:
+      tarball-suffix: -${{ github.event.pull_request.number}}-${{ github.event.pull_request.head.sha }}
+
+  publish-kata-deploy-payload-amd64:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/publish-kata-deploy-payload-amd64.yaml
+    with:
+      tarball-suffix: -${{ github.event.pull_request.number}}-${{ github.event.pull_request.head.sha }}
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64
+    secrets: inherit
+
+  run-k8s-tests-on-aks:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-k8s-tests-on-aks.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64
+    secrets: inherit
+
+  run-k8s-tests-on-sev:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-k8s-tests-on-sev.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64
+
+  run-k8s-tests-on-snp:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-k8s-tests-on-snp.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64
+
+  run-k8s-tests-on-tdx:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-k8s-tests-on-tdx.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64

.github/workflows/commit-message-check.yaml

@@ -62,6 +62,9 @@ jobs:
         #   to be specified at the start of the regex as the action is passed
         #   the entire commit message.
         #
+        # - This check will pass if the commit message only contains a subject
+        #   line, as other body message properties are enforced elsewhere.
+        #
         # - Body lines *can* be longer than the maximum if they start
         #   with a non-alphabetic character or if there is no whitespace in
         #   the line.
@@ -75,7 +78,7 @@ jobs:
         #
         # - A SoB comment can be any length (as it is unreasonable to penalise
         #   people with long names/email addresses :)
-        pattern: '^.+(\n([a-zA-Z].{0,150}|[^a-zA-Z\n].*|[^\s\n]*|Signed-off-by:.*|))+$'
+        pattern: '(^[^\n]+$|^.+(\n([a-zA-Z].{0,150}|[^a-zA-Z\n].*|[^\s\n]*|Signed-off-by:.*|))+$)'
         error: 'Body line too long (max 150)'
         post_error: ${{ env.error_msg }}
 

.github/workflows/darwin-tests.yaml

@@ -5,20 +5,16 @@ on:
       - edited
       - reopened
       - synchronize
-
+    paths-ignore: [ '**.md', '**.png', '**.jpg', '**.jpeg', '**.svg', '/docs/**' ]
 name: Darwin tests
 jobs:
   test:
-    strategy:
-      matrix:
-        go-version: [1.16.x, 1.17.x]
-        os: [macos-latest]
-    runs-on: ${{ matrix.os }}
+    runs-on: macos-latest
     steps:
     - name: Install Go
       uses: actions/setup-go@v2
       with:
-        go-version: ${{ matrix.go-version }}
+        go-version: 1.19.3
     - name: Checkout code
       uses: actions/checkout@v2
     - name: Build utils

.github/workflows/docs-url-alive-check.yaml

@@ -5,11 +5,7 @@ on:
 name: Docs URL Alive Check
 jobs:
   test:
-    strategy:
-      matrix:
-        go-version: [1.17.x]
-        os: [ubuntu-20.04]
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-20.04
     # don't run this action on forks
     if: github.repository_owner == 'kata-containers'
     env:
@@ -18,7 +14,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v2
       with:
-        go-version: ${{ matrix.go-version }}
+        go-version: 1.19.3
       env:
         GOPATH: ${{ runner.workspace }}/kata-containers
     - name: Set env

.github/workflows/kata-deploy-push.yaml

@@ -1,84 +0,0 @@
-name: kata deploy build
-
-on: 
-  pull_request:
-    types:
-      - opened
-      - edited
-      - reopened
-      - synchronize
-    paths:
-      - tools/**
-      - versions.yaml
-
-jobs:
-  build-asset:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        asset:
-          - kernel
-          - shim-v2
-          - qemu
-          - cloud-hypervisor
-          - firecracker
-          - rootfs-image
-          - rootfs-initrd
-          - virtiofsd
-    steps:
-      - uses: actions/checkout@v2
-      - name: Install docker
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          curl -fsSL https://test.docker.com -o test-docker.sh
-          sh test-docker.sh
-
-      - name: Build ${{ matrix.asset }}
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r --preserve=all "${build_dir}" "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-
-      - name: store-artifact ${{ matrix.asset }}
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: ubuntu-latest
-    needs: build-asset
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifacts
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-artifacts
-          path: build
-      - name: merge-artifacts
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          make merge-builds
-      - name: store-artifacts
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-static-tarball
-          path: kata-static.tar.xz
-
-  make-kata-tarball:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: make kata-tarball
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          make kata-tarball
-          sudo make install-tarball

.github/workflows/kata-deploy-test.yaml

@@ -1,149 +0,0 @@
-on:
-  workflow_dispatch: # this is used to trigger the workflow on non-main branches
-  issue_comment:
-    types: [created, edited]
-
-name: test-kata-deploy
-
-jobs:
-  check-comment-and-membership:
-    runs-on: ubuntu-latest
-    if: |
-      github.event.issue.pull_request
-      && github.event_name == 'issue_comment'
-      && github.event.action == 'created'
-      && startsWith(github.event.comment.body, '/test_kata_deploy')
-    steps:
-      - name: Check membership
-        uses: kata-containers/is-organization-member@1.0.1
-        id: is_organization_member
-        with:
-          organization: kata-containers
-          username: ${{ github.event.comment.user.login }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Fail if not member
-        run: |
-          result=${{ steps.is_organization_member.outputs.result }}
-          if [ $result == false ]; then
-              user=${{ github.event.comment.user.login }}
-              echo Either ${user} is not part of the kata-containers organization
-              echo or ${user} has its Organization Visibility set to Private at
-              echo https://github.com/orgs/kata-containers/people?query=${user}
-              echo 
-              echo Ensure you change your Organization Visibility to Public and
-              echo trigger the test again.
-              exit 1
-          fi
-
-  build-asset:
-    runs-on: ubuntu-latest
-    needs: check-comment-and-membership
-    strategy:
-      matrix:
-        asset:
-          - cloud-hypervisor
-          - firecracker
-          - kernel
-          - qemu
-          - rootfs-image
-          - rootfs-initrd
-          - shim-v2
-          - virtiofsd
-    steps:
-      - name: get-PR-ref
-        id: get-PR-ref
-        run: |
-            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
-            echo "reference for PR: " ${ref}
-            echo "##[set-output name=pr-ref;]${ref}"
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
-
-      - name: Install docker
-        run: |
-          curl -fsSL https://test.docker.com -o test-docker.sh
-          sh test-docker.sh
-
-      - name: Build ${{ matrix.asset }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: ubuntu-latest
-    needs: build-asset
-    steps:
-      - name: get-PR-ref
-        id: get-PR-ref
-        run: |
-            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
-            echo "reference for PR: " ${ref}
-            echo "##[set-output name=pr-ref;]${ref}"
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
-      - name: get-artifacts
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-static-tarball
-          path: kata-static.tar.xz
-
-  kata-deploy:
-    needs: create-kata-tarball
-    runs-on: ubuntu-latest
-    steps:
-      - name: get-PR-ref
-        id: get-PR-ref
-        run: |
-            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
-            echo "reference for PR: " ${ref}
-            echo "##[set-output name=pr-ref;]${ref}"
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-static-tarball
-      - name: build-and-push-kata-deploy-ci
-        id: build-and-push-kata-deploy-ci
-        run: |
-          PR_SHA=$(git log --format=format:%H -n1)
-          mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
-          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t quay.io/kata-containers/kata-deploy-ci:$PR_SHA $GITHUB_WORKSPACE/tools/packaging/kata-deploy
-          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
-          docker push quay.io/kata-containers/kata-deploy-ci:$PR_SHA
-          mkdir -p packaging/kata-deploy
-          ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
-          echo "::set-output name=PKG_SHA::${PR_SHA}"
-      - name: test-kata-deploy-ci-in-aks
-        uses: ./packaging/kata-deploy/action
-        with:
-          packaging-sha: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
-        env:
-          PKG_SHA: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
-          AZ_APPID: ${{ secrets.AZ_APPID }}
-          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
-          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
-          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}

.github/workflows/payload-after-push.yaml

@@ -0,0 +1,74 @@
+name: CI | Publish Kata Containers payload
+on:
+  push:
+    branches:
+      - main
+      - stable-*
+
+jobs:
+  build-assets-amd64:
+    uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
+    with:
+      push-to-registry: yes
+    secrets: inherit
+
+  build-assets-arm64:
+    uses: ./.github/workflows/build-kata-static-tarball-arm64.yaml
+    with:
+      push-to-registry: yes
+    secrets: inherit
+
+  build-assets-s390x:
+    uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml
+    with:
+      push-to-registry: yes
+    secrets: inherit
+
+  publish-kata-deploy-payload-amd64:
+    needs: build-assets-amd64
+    uses: ./.github/workflows/publish-kata-deploy-payload-amd64.yaml
+    with:
+      registry: quay.io
+      repo: kata-containers/kata-deploy-ci
+      tag: kata-containers-amd64
+    secrets: inherit
+
+  publish-kata-deploy-payload-arm64:
+    needs: build-assets-arm64
+    uses: ./.github/workflows/publish-kata-deploy-payload-arm64.yaml
+    with:
+      registry: quay.io
+      repo: kata-containers/kata-deploy-ci
+      tag: kata-containers-arm64
+    secrets: inherit
+
+  publish-kata-deploy-payload-s390x:
+    needs: build-assets-s390x
+    uses: ./.github/workflows/publish-kata-deploy-payload-s390x.yaml
+    with:
+      registry: quay.io
+      repo: kata-containers/kata-deploy-ci
+      tag: kata-containers-s390x
+    secrets: inherit
+
+  publish-manifest:
+    runs-on: ubuntu-latest
+    needs: [publish-kata-deploy-payload-amd64, publish-kata-deploy-payload-arm64, publish-kata-deploy-payload-s390x]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Push multi-arch manifest
+        run: |
+          docker manifest create quay.io/kata-containers/kata-deploy-ci:kata-containers-latest \
+          --amend quay.io/kata-containers/kata-deploy-ci:kata-containers-amd64 \
+          --amend quay.io/kata-containers/kata-deploy-ci:kata-containers-arm64 \
+          --amend quay.io/kata-containers/kata-deploy-ci:kata-containers-s390x
+          docker manifest push quay.io/kata-containers/kata-deploy-ci:kata-containers-latest

.github/workflows/publish-kata-deploy-payload-amd64.yaml

@@ -0,0 +1,52 @@
+name: CI | Publish kata-deploy payload for amd64
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+
+jobs:
+  kata-payload:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.registry == 'quay.io' }}
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Login to Kata Containers ghcr.io
+        if: ${{ inputs.registry == 'ghcr.io' }}
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz \
+          ${{ inputs.registry }}/${{ inputs.repo }} ${{ inputs.tag }}

.github/workflows/publish-kata-deploy-payload-arm64.yaml

@@ -0,0 +1,57 @@
+name: CI | Publish kata-deploy payload for arm64
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+
+jobs:
+  kata-payload:
+    runs-on: arm64
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-arm64${{ inputs.tarball-suffix }}
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.registry == 'quay.io' }}
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Login to Kata Containers ghcr.io
+        if: ${{ inputs.registry == 'ghcr.io' }}
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz \
+          ${{ inputs.registry }}/${{ inputs.repo }} ${{ inputs.tag }}
+

.github/workflows/publish-kata-deploy-payload-s390x.yaml

@@ -0,0 +1,56 @@
+name: CI | Publish kata-deploy payload for s390x
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+
+jobs:
+  kata-payload:
+    runs-on: s390x
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-s390x${{ inputs.tarball-suffix }}
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.registry == 'quay.io' }}
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Login to Kata Containers ghcr.io
+        if: ${{ inputs.registry == 'ghcr.io' }}
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz \
+          ${{ inputs.registry }}/${{ inputs.repo }} ${{ inputs.tag }}

.github/workflows/release-amd64.yaml

@@ -0,0 +1,63 @@
+name: Publish Kata release artifacts for amd64
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-kata-static-tarball-amd64:
+    uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
+
+  kata-deploy:
+    needs: build-kata-static-tarball-amd64
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to Kata Containers docker.io
+        uses: docker/login-action@v3
+        with:
+          registry: docker.io
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64
+
+      - name: build-and-push-kata-deploy-ci-amd64
+        id: build-and-push-kata-deploy-ci-amd64
+        run: |
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          pushd $GITHUB_WORKSPACE
+          git checkout $tag
+          pkg_sha=$(git rev-parse HEAD)
+          popd
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+            $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy-ci" \
+            "${pkg_sha}-${{ inputs.target-arch }}"
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+            $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy-ci" \
+            "${pkg_sha}-${{ inputs.target-arch }}"
+
+      - name: push-tarball
+        run: |
+          # tag the container image we created and push to DockerHub
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tags=($tag)
+          tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
+          for tag in ${tags[@]}; do
+            docker tag docker.io/katadocker/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci-amd64.outputs.PKG_SHA}}-${{ inputs.target-arch }} docker.io/katadocker/kata-deploy:${tag}-${{ inputs.target-arch }}
+            docker tag quay.io/kata-containers/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci-amd64.outputs.PKG_SHA}}-${{ inputs.target-arch }} quay.io/kata-containers/kata-deploy:${tag}-${{ inputs.target-arch }}
+            docker push docker.io/katadocker/kata-deploy:${tag}-${{ inputs.target-arch }}
+            docker push quay.io/kata-containers/kata-deploy:${tag}-${{ inputs.target-arch }}
+          done

.github/workflows/release-arm64.yaml

@@ -0,0 +1,63 @@
+name: Publish Kata release artifacts for arm64
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-kata-static-tarball-arm64:
+    uses: ./.github/workflows/build-kata-static-tarball-arm64.yaml
+
+  kata-deploy:
+    needs: build-kata-static-tarball-arm64
+    runs-on: arm64
+    steps:
+      - name: Login to Kata Containers docker.io
+        uses: docker/login-action@v3
+        with:
+          registry: docker.io
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-arm64
+
+      - name: build-and-push-kata-deploy-ci-arm64
+        id: build-and-push-kata-deploy-ci-arm64
+        run: |
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          pushd $GITHUB_WORKSPACE
+          git checkout $tag
+          pkg_sha=$(git rev-parse HEAD)
+          popd
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+            $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy-ci" \
+            "${pkg_sha}-${{ inputs.target-arch }}"
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+            $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy-ci" \
+            "${pkg_sha}-${{ inputs.target-arch }}"
+
+      - name: push-tarball
+        run: |
+          # tag the container image we created and push to DockerHub
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tags=($tag)
+          tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
+          for tag in ${tags[@]}; do
+            docker tag docker.io/katadocker/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci-arm64.outputs.PKG_SHA}}-${{ inputs.target-arch }} docker.io/katadocker/kata-deploy:${tag}-${{ inputs.target-arch }}
+            docker tag quay.io/kata-containers/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci-arm64.outputs.PKG_SHA}}-${{ inputs.target-arch }} quay.io/kata-containers/kata-deploy:${tag}-${{ inputs.target-arch }}
+            docker push docker.io/katadocker/kata-deploy:${tag}-${{ inputs.target-arch }}
+            docker push quay.io/kata-containers/kata-deploy:${tag}-${{ inputs.target-arch }}
+          done

.github/workflows/release-s390x.yaml

@@ -0,0 +1,63 @@
+name: Publish Kata release artifacts for s390x
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-kata-static-tarball-s390x:
+    uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml
+
+  kata-deploy:
+    needs: build-kata-static-tarball-s390x
+    runs-on: s390x
+    steps:
+      - name: Login to Kata Containers docker.io
+        uses: docker/login-action@v3
+        with:
+          registry: docker.io
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+
+      - name: build-and-push-kata-deploy-ci-s390x
+        id: build-and-push-kata-deploy-ci-s390x
+        run: |
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          pushd $GITHUB_WORKSPACE
+          git checkout $tag
+          pkg_sha=$(git rev-parse HEAD)
+          popd
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+            $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy-ci" \
+            "${pkg_sha}-${{ inputs.target-arch }}"
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+            $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy-ci" \
+            "${pkg_sha}-${{ inputs.target-arch }}"
+
+      - name: push-tarball
+        run: |
+          # tag the container image we created and push to DockerHub
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tags=($tag)
+          tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
+          for tag in ${tags[@]}; do
+            docker tag docker.io/katadocker/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci-s390x.outputs.PKG_SHA}}-${{ inputs.target-arch }} docker.io/katadocker/kata-deploy:${tag}-${{ inputs.target-arch }}
+            docker tag quay.io/kata-containers/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci-s390x.outputs.PKG_SHA}}-${{ inputs.target-arch }} quay.io/kata-containers/kata-deploy:${tag}-${{ inputs.target-arch }}
+            docker push docker.io/katadocker/kata-deploy:${tag}-${{ inputs.target-arch }}
+            docker push quay.io/kata-containers/kata-deploy:${tag}-${{ inputs.target-arch }}
+          done

.github/workflows/release.yaml

@@ -5,127 +5,83 @@ on:
       - '[0-9]+.[0-9]+.[0-9]+*'
 
 jobs:
-  build-asset:
+  build-and-push-assets-amd64:
+    uses: ./.github/workflows/release-amd64.yaml
+    with:
+      target-arch: amd64
+    secrets: inherit
+
+  build-and-push-assets-arm64:
+    uses: ./.github/workflows/release-arm64.yaml
+    with:
+      target-arch: arm64
+    secrets: inherit
+
+  build-and-push-assets-s390x:
+    uses: ./.github/workflows/release-s390x.yaml
+    with:
+      target-arch: s390x
+    secrets: inherit
+
+  publish-multi-arch-images:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        asset:
-          - cloud-hypervisor
-          - firecracker
-          - kernel
-          - qemu
-          - rootfs-image
-          - rootfs-initrd
-          - shim-v2
-          - virtiofsd
+    needs: [build-and-push-assets-amd64, build-and-push-assets-arm64, build-and-push-assets-s390x]
     steps:
-      - uses: actions/checkout@v2
-      - name: Install docker
-        run: |
-          curl -fsSL https://test.docker.com -o test-docker.sh
-          sh test-docker.sh
+      - name: Checkout repository
+        uses: actions/checkout@v3
 
-      - name: Build ${{ matrix.asset }}
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh --build="${KATA_ASSET}"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+      - name: Login to Kata Containers docker.io
+        uses: docker/login-action@v3
+        with:
+          registry: docker.io
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
 
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v2
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v3
         with:
-          name: kata-artifacts
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          if-no-files-found: error
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-  create-kata-tarball:
-    runs-on: ubuntu-latest
-    needs: build-asset
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifacts
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-static-tarball
-          path: kata-static.tar.xz
-
-  kata-deploy:
-    needs: create-kata-tarball
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-static-tarball
-      - name: build-and-push-kata-deploy-ci
-        id: build-and-push-kata-deploy-ci
-        run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          pushd $GITHUB_WORKSPACE
-          git checkout $tag
-          pkg_sha=$(git rev-parse HEAD)
-          popd
-          mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
-          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:$pkg_sha -t quay.io/kata-containers/kata-deploy-ci:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
-          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
-          docker push katadocker/kata-deploy-ci:$pkg_sha
-          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
-          docker push quay.io/kata-containers/kata-deploy-ci:$pkg_sha
-          mkdir -p packaging/kata-deploy
-          ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
-          echo "::set-output name=PKG_SHA::${pkg_sha}"
-      - name: test-kata-deploy-ci-in-aks
-        uses: ./packaging/kata-deploy/action
-        with:
-          packaging-sha: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
-        env:
-          PKG_SHA: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
-          AZ_APPID: ${{ secrets.AZ_APPID }}
-          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
-          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
-          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
-      - name: push-tarball
+      - name: Push multi-arch manifest
         run: |
           # tag the container image we created and push to DockerHub
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
           tags=($tag)
           tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
-          for tag in ${tags[@]}; do \
-            docker tag katadocker/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} katadocker/kata-deploy:${tag} && \
-            docker tag quay.io/kata-containers/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} quay.io/kata-containers/kata-deploy:${tag} && \
-            docker push katadocker/kata-deploy:${tag} && \
-            docker push quay.io/kata-containers/kata-deploy:${tag}; \
+          # push to quay.io and docker.io
+          for tag in ${tags[@]}; do
+            docker manifest create quay.io/kata-containers/kata-deploy:${tag} \
+              --amend quay.io/kata-containers/kata-deploy:${tag}-amd64 \
+              --amend quay.io/kata-containers/kata-deploy:${tag}-arm64 \
+              --amend quay.io/kata-containers/kata-deploy:${tag}-s390x
+
+            docker manifest create docker.io/katadocker/kata-deploy:${tag} \
+              --amend docker.io/katadocker/kata-deploy:${tag}-amd64 \
+              --amend docker.io/katadocker/kata-deploy:${tag}-arm64 \
+              --amend docker.io/katadocker/kata-deploy:${tag}-s390x
+
+            docker manifest push quay.io/kata-containers/kata-deploy:${tag}
+            docker manifest push docker.io/katadocker/kata-deploy:${tag}
           done
 
-  upload-static-tarball:
-    needs: kata-deploy
+  upload-multi-arch-static-tarball:
+    needs: publish-multi-arch-images
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: download-artifacts
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-static-tarball
+      - uses: actions/checkout@v3
       - name: install hub
         run: |
           HUB_VER=$(curl -s "https://api.github.com/repos/github/hub/releases/latest" | jq -r .tag_name | sed 's/^v//')
           wget -q -O- https://github.com/github/hub/releases/download/v$HUB_VER/hub-linux-amd64-$HUB_VER.tgz | \
           tar xz --strip-components=2 --wildcards '*/bin/hub' && sudo mv hub /usr/local/bin/hub
-      - name: push static tarball to github
+
+      - name: download-artifacts-amd64
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64
+      - name: push amd64 static tarball to github
         run: |
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
           tarball="kata-static-$tag-x86_64.tar.xz"
@@ -135,11 +91,39 @@ jobs:
           GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}"
           popd
 
+      - name: download-artifacts-arm64
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-arm64
+      - name: push arm64 static tarball to github
+        run: |
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tarball="kata-static-$tag-aarch64.tar.xz"
+          mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
+          pushd $GITHUB_WORKSPACE
+          echo "uploading asset '${tarball}' for tag: ${tag}"
+          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}"
+          popd
+
+      - name: download-artifacts-s390x
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+      - name: push s390x static tarball to github
+        run: |
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tarball="kata-static-$tag-s390x.tar.xz"
+          mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
+          pushd $GITHUB_WORKSPACE
+          echo "uploading asset '${tarball}' for tag: ${tag}"
+          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}"
+          popd
+
   upload-cargo-vendored-tarball:
-    needs: upload-static-tarball
+    needs: upload-multi-arch-static-tarball
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: generate-and-upload-tarball
         run: |
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
@@ -153,7 +137,7 @@ jobs:
     needs: upload-cargo-vendored-tarball
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: download-and-upload-tarball
         env:
           GITHUB_TOKEN: ${{ secrets.GIT_UPLOAD_TOKEN }}

.github/workflows/run-k8s-tests-on-aks.yaml

@@ -0,0 +1,95 @@
+name: CI | Run kubernetes tests on AKS
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+
+jobs:
+  run-k8s-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - clh
+          - dragonball
+          - qemu
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Download Azure CLI
+        run: |
+          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+
+      - name: Log into the Azure account
+        run: |
+          az login \
+            --service-principal \
+            -u "${{ secrets.AZ_APPID }}" \
+            -p "${{ secrets.AZ_PASSWORD }}" \
+            --tenant "${{ secrets.AZ_TENANT_ID }}"
+
+      - name: Create AKS cluster
+        run: |
+          az aks create \
+            -g "kataCI" \
+            -n "${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-amd64" \
+            -s "Standard_D4s_v5" \
+            --node-count 1 \
+            --generate-ssh-keys
+
+      - name: Install `bats`
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install bats
+
+      - name: Install `kubectl`
+        run: |
+          sudo az aks install-cli
+
+      - name: Download credentials for the Kubernetes CLI to use them
+        run: |
+          az aks get-credentials -g "kataCI" -n ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-amd64
+
+      - name: Run tests
+        timeout-minutes: 35
+        run: |
+          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
+
+          kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
+          kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+          kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
+          kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
+
+          # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
+          # which may cause issues like not having the node properly labeled or the artefacts
+          # properly deployed when the tests actually start running.
+          sleep 150s
+
+          pushd tests/integration/kubernetes
+          sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
+          bash run_kubernetes_tests.sh
+          popd
+        env:
+          KATA_HYPERVISOR: ${{ matrix.vmm }}
+
+      - name: Delete AKS cluster
+        if: always()
+        run: |
+          az aks delete \
+            -g "kataCI" \
+            -n "${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-amd64" \
+            --yes \
+            --no-wait

.github/workflows/run-k8s-tests-on-sev.yaml

@@ -0,0 +1,68 @@
+name: CI | Run kubernetes tests on SEV
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+
+jobs:
+  run-k8s-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-sev
+    runs-on: sev
+    env:
+      KUBECONFIG: /home/kata/.kube/config
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: |
+          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
+
+          kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
+          kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+          kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
+          kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
+
+          # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
+          # which may cause issues like not having the node properly labeled or the artefacts
+          # properly deployed when the tests actually start running.
+          sleep 60s
+
+          pushd tests/integration/kubernetes
+          sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
+          bash run_kubernetes_tests.sh
+          popd
+        env:
+          KATA_HYPERVISOR: ${{ matrix.vmm }}
+
+      - name: Delete kata-deploy
+        if: always()
+        run: |
+          kubectl delete -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+          kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
+
+          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
+          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
+          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
+          kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
+          sleep 180s
+
+          kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
+          kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
+          kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml

.github/workflows/run-k8s-tests-on-snp.yaml

@@ -0,0 +1,68 @@
+name: CI | Run kubernetes tests on SEV-SNP
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+
+jobs:
+  run-k8s-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-snp
+    runs-on: sev-snp
+    env:
+      KUBECONFIG: /home/kata/.kube/config
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: |
+          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
+
+          kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
+          kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+          kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
+          kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
+
+          # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
+          # which may cause issues like not having the node properly labeled or the artefacts
+          # properly deployed when the tests actually start running.
+          sleep 60s
+
+          pushd tests/integration/kubernetes
+          sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
+          bash run_kubernetes_tests.sh
+          popd
+        env:
+          KATA_HYPERVISOR: ${{ matrix.vmm }}
+
+      - name: Delete kata-deploy
+        if: always()
+        run: |
+          kubectl delete -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+          kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
+
+          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
+          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
+          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
+          kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
+          sleep 180s
+
+          kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
+          kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
+          kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml

.github/workflows/run-k8s-tests-on-tdx.yaml

@@ -0,0 +1,68 @@
+name: CI | Run kubernetes tests on TDX
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+
+jobs:
+  run-k8s-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-tdx
+    runs-on: tdx
+    env:
+      KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: |
+          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
+
+          kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
+          kubectl apply -k tools/packaging/kata-deploy/kata-deploy/overlays/k3s
+          kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
+          kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
+
+          # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
+          # which may cause issues like not having the node properly labeled or the artefacts
+          # properly deployed when the tests actually start running.
+          sleep 60s
+
+          pushd tests/integration/kubernetes
+          sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
+          bash run_kubernetes_tests.sh
+          popd
+        env:
+          KATA_HYPERVISOR: ${{ matrix.vmm }}
+
+      - name: Delete kata-deploy
+        if: always()
+        run: |
+          kubectl delete -k tools/packaging/kata-deploy/kata-deploy/overlays/k3s
+          kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
+
+          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
+          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
+          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
+          kubectl apply -k tools/packaging/kata-deploy/kata-cleanup/overlays/k3s
+          sleep 180s
+
+          kubectl delete -k tools/packaging/kata-deploy/kata-cleanup/overlays/k3s
+          kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
+          kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml

.github/workflows/snap-release.yaml

@@ -4,24 +4,35 @@ on:
     tags:
       - '[0-9]+.[0-9]+.[0-9]+*'
 
+env:
+  SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.snapcraft_token }}
+
 jobs:
   release-snap:
     runs-on: ubuntu-20.04
     steps:
       - name: Check out Git repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       - name: Install Snapcraft
-        uses: samuelmeuli/action-snapcraft@v1
-        with:
-          snapcraft_token: ${{ secrets.snapcraft_token }}
+        run: |
+          # Required to avoid snapcraft install failure
+          sudo chown root:root /
+
+          # "--classic" is needed for the GitHub action runner
+          # environment.
+          sudo snap install snapcraft --classic
+
+          # Allow other parts to access snap binaries
+          echo /snap/bin >> "$GITHUB_PATH"
 
       - name: Build snap
         run: |
           # Removing man-db, workflow kept failing, fixes: #4480
           sudo apt -y remove --purge man-db
+          sudo apt-get update
           sudo apt-get install -y git git-extras
           kata_url="https://github.com/kata-containers/kata-containers"
           latest_version=$(git ls-remote --tags ${kata_url}  | egrep -o "refs.*" | egrep -v "\-alpha|\-rc|{}" | egrep -o "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+" | sort -V -r | head -1)

.github/workflows/snap.yaml

@@ -6,6 +6,7 @@ on:
       - synchronize
       - reopened
       - edited
+    paths-ignore: [ '**.md', '**.png', '**.jpg', '**.jpeg', '**.svg', '/docs/**' ]
 
 jobs:
   test:
@@ -13,13 +14,22 @@ jobs:
     steps:
       - name: Check out
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       - name: Install Snapcraft
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: samuelmeuli/action-snapcraft@v1
+        run: |
+          # Required to avoid snapcraft install failure
+          sudo chown root:root /
+
+          # "--classic" is needed for the GitHub action runner
+          # environment.
+          sudo snap install snapcraft --classic
+
+          # Allow other parts to access snap binaries
+          echo /snap/bin >> "$GITHUB_PATH"
 
       - name: Build snap
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}

.github/workflows/static-checks-dragonball.yaml

@@ -0,0 +1,33 @@
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+    paths-ignore: [ '**.md', '**.png', '**.jpg', '**.jpeg', '**.svg', '/docs/**' ]
+
+name: Static checks dragonball
+jobs:
+  test-dragonball:
+    runs-on: self-hosted
+    env:
+      RUST_BACKTRACE: "1"
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set env
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        run: |
+          echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
+      - name: Install Rust
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        run: |
+          ./ci/install_rust.sh
+          PATH=$PATH:"$HOME/.cargo/bin"
+      - name: Run Unit Test
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        run: |
+          cd src/dragonball
+          cargo version
+          rustc --version
+          sudo -E env PATH=$PATH LIBC=gnu SUPPORT_VIRTUALIZATION=true make test

.github/workflows/static-checks.yaml

@@ -8,56 +8,57 @@ on:
 
 name: Static checks
 jobs:
-  test:
+  static-checks:
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
-        go-version: [1.16.x, 1.17.x]
-        os: [ubuntu-20.04]
-    runs-on: ${{ matrix.os }}
+        cmd:
+          - "make vendor"
+          - "make static-checks"
+          - "make check"
+          - "make test"
+          - "sudo -E PATH=\"$PATH\" make test"
     env:
-      TRAVIS: "true"
-      TRAVIS_BRANCH: ${{ github.base_ref }}
-      TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }}
-      TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }}
       RUST_BACKTRACE: "1"
       target_branch: ${{ github.base_ref }}
+      GOPATH: ${{ github.workspace }}
     steps:
-    - name: Install Go
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/setup-go@v2
-      with:
-        go-version: ${{ matrix.go-version }}
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Setup GOPATH
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+    - name: Free disk space
       run: |
-        echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}"
-        echo "TRAVIS: ${TRAVIS}"
-    - name: Set env
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+        sudo rm -rf /usr/share/dotnet
+        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
     - name: Checkout code
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
       with:
         fetch-depth: 0
         path: ./src/github.com/${{ github.repository }}
-    - name: Setup travis references
+    - name: Install Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.19.3
+    - name: Check kernel config version
+      run: |
+        cd "${{ github.workspace }}/src/github.com/${{ github.repository }}"
+        kernel_dir="tools/packaging/kernel/"
+        kernel_version_file="${kernel_dir}kata_config_version"
+        modified_files=$(git diff --name-only origin/main..HEAD)
+        if git diff --name-only origin/main..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then
+          echo "Kernel directory has changed, checking if $kernel_version_file has been updated"
+          if echo "$modified_files" | grep -v "README.md" | grep "${kernel_dir}" >>"/dev/null"; then
+            echo "$modified_files" | grep "$kernel_version_file" >>/dev/null || ( echo "Please bump version in $kernel_version_file" && exit 1)
+          else
+            echo "Readme file changed, no need for kernel config version update."
+          fi
+          echo "Check passed"
+        fi
+    - name: Set PATH
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
-        echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}"
-        target_branch=${TRAVIS_BRANCH}
+        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
     - name: Setup
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
         cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
     - name: Installing rust
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
@@ -66,6 +67,7 @@ jobs:
         rustup target add x86_64-unknown-linux-musl
         rustup component add rustfmt clippy
     - name: Setup seccomp
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
         libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
         gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
@@ -73,24 +75,7 @@ jobs:
         echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
         echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
         echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
-    # Check whether the vendored code is up-to-date & working as the first thing
-    - name: Check vendored code
+    - name: Run check
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make vendor
-    - name: Static Checks
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make static-checks
-    - name: Run Compiler Checks
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make check
-    - name: Run Unit Tests
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make test
-    - name: Run Unit Tests As Root User
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && sudo -E PATH="$PATH" make test
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && ${{ matrix.cmd }}

.gitignore

@@ -4,6 +4,10 @@
 **/*.rej
 **/target
 **/.vscode
+**/.idea
+**/.fleet
+**/*.swp
+**/*.swo
 pkg/logging/Cargo.lock
 src/agent/src/version.rs
 src/agent/kata-agent.service
@@ -11,4 +15,3 @@ src/agent/protocols/src/*.rs
 !src/agent/protocols/src/lib.rs
 build
 src/tools/log-parser/kata-log-parser
-

Makefile

@@ -8,6 +8,7 @@ COMPONENTS =
 
 COMPONENTS += libs
 COMPONENTS += agent
+COMPONENTS += dragonball
 COMPONENTS += runtime
 COMPONENTS += runtime-rs
 
@@ -15,11 +16,13 @@ COMPONENTS += runtime-rs
 TOOLS =
 
 TOOLS += agent-ctl
-TOOLS += trace-forwarder
-TOOLS += runk
+TOOLS += kata-ctl
 TOOLS += log-parser
+TOOLS += log-parser-rs
+TOOLS += runk
+TOOLS += trace-forwarder
 
-STANDARD_TARGETS = build check clean install test vendor
+STANDARD_TARGETS = build check clean install static-checks-build test vendor
 
 default: all
 
@@ -35,7 +38,7 @@ generate-protocols:
 	make -C src/agent generate-protocols
 
 # Some static checks rely on generated source files of components.
-static-checks: build
+static-checks: static-checks-build
 	bash ci/static-checks.sh
 
 docs-url-alive-check:
@@ -43,10 +46,8 @@ docs-url-alive-check:
 
 .PHONY: \
 	all \
-	binary-tarball \
+	kata-tarball \
+	install-tarball \
 	default \
-	install-binary-tarball \
 	static-checks \
 	docs-url-alive-check
-
-

README.md

@@ -119,10 +119,8 @@ The table below lists the core parts of the project:
 | [runtime](src/runtime) | core | Main component run by a container manager and providing a containerd shimv2 runtime implementation. |
 | [runtime-rs](src/runtime-rs) | core | The Rust version runtime. |
 | [agent](src/agent) | core | Management process running inside the virtual machine / POD that sets up the container environment. |
-| [libraries](src/libs) | core | Library crates shared by multiple Kata Container components or published to [`crates.io`](https://crates.io/index.html) |
 | [`dragonball`](src/dragonball) | core | An optional built-in VMM brings out-of-the-box Kata Containers experience with optimizations on container workloads |
 | [documentation](docs) | documentation | Documentation common to all components (such as design and install documentation). |
-| [libraries](src/libs) | core | Library crates shared by multiple Kata Container components or published to [`crates.io`](https://crates.io/index.html) |
 | [tests](https://github.com/kata-containers/tests) | tests | Excludes unit tests which live with the main code. |
 
 ### Additional components
@@ -135,6 +133,8 @@ The table below lists the remaining parts of the project:
 | [kernel](https://www.kernel.org) | kernel | Linux kernel used by the hypervisor to boot the guest image. Patches are stored [here](tools/packaging/kernel). |
 | [osbuilder](tools/osbuilder) | infrastructure | Tool to create "mini O/S" rootfs and initrd images and kernel for the hypervisor. |
 | [`agent-ctl`](src/tools/agent-ctl) | utility | Tool that provides low-level access for testing the agent. |
+| [`kata-ctl`](src/tools/kata-ctl) | utility | Tool that provides advanced commands and debug facilities. |
+| [`log-parser-rs`](src/tools/log-parser-rs) | utility | Tool that aid in analyzing logs from the kata runtime. |
 | [`trace-forwarder`](src/tools/trace-forwarder) | utility | Agent tracing helper. |
 | [`runk`](src/tools/runk) | utility | Standard OCI container runtime based on the agent. |
 | [`ci`](https://github.com/kata-containers/ci) | CI | Continuous Integration configuration files and scripts. |

VERSION

@@ -1 +1 @@
-3.0.0-rc0
+3.2.0-alpha2

ci/install_yq.sh

@@ -43,6 +43,16 @@ function install_yq() {
 	"aarch64")
 		goarch=arm64
 		;;
+	"arm64")
+		# If we're on an apple silicon machine, just assign amd64. 
+		# The version of yq we use doesn't have a darwin arm build, 
+		# but Rosetta can come to the rescue here.
+		if [ $goos == "Darwin" ]; then 
+			goarch=amd64
+		else 
+			goarch=arm64
+		fi
+		;;
 	"ppc64le")
 		goarch=ppc64le
 		;;
@@ -64,7 +74,7 @@ function install_yq() {
 	fi
 
 	## NOTE: ${var,,} => gives lowercase value of var
-	local yq_url="https://${yq_pkg}/releases/download/${yq_version}/yq_${goos,,}_${goarch}"
+	local yq_url="https://${yq_pkg}/releases/download/${yq_version}/yq_${goos}_${goarch}"
 	curl -o "${yq_path}" -LSsf "${yq_url}"
 	[ $? -ne 0 ] && die "Download ${yq_url} failed"
 	chmod +x "${yq_path}"

docs/Developer-Guide.md

@@ -2,6 +2,8 @@
 
 This document is written **specifically for developers**: it is not intended for end users.
 
+If you want to contribute changes that you have made, please read the [community guidelines](https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md) for information about our processes.
+
 # Assumptions
 
 - You are working on a non-critical test or development system.
@@ -33,51 +35,41 @@ You need to install the following to build Kata Containers components:
 - `make`.
 - `gcc` (required for building the shim and runtime).
 
-# Build and install the Kata Containers runtime
+# Build and install Kata Containers
+## Build and install the Kata Containers runtime
 
-```
-$ go get -d -u github.com/kata-containers/kata-containers
-$ cd $GOPATH/src/github.com/kata-containers/kata-containers/src/runtime
-$ make && sudo -E PATH=$PATH make install
+```bash
+$ git clone https://github.com/kata-containers/kata-containers.git
+$ pushd kata-containers/src/runtime
+$ make && sudo -E "PATH=$PATH" make install
+$ sudo mkdir -p /etc/kata-containers/
+$ sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers
+$ popd
 ```
 
 The build will create the following:
 
 - runtime binary: `/usr/local/bin/kata-runtime` and `/usr/local/bin/containerd-shim-kata-v2`
-- configuration file: `/usr/share/defaults/kata-containers/configuration.toml`
-
-# Check hardware requirements
-
-You can check if your system is capable of creating a Kata Container by running the following:
-
-```
-$ sudo kata-runtime check
-```
-
-If your system is *not* able to run Kata Containers, the previous command will error out and explain why.
+- configuration file: `/usr/share/defaults/kata-containers/configuration.toml` and `/etc/kata-containers/configuration.toml`
 
 ## Configure to use initrd or rootfs image
 
 Kata containers can run with either an initrd image or a rootfs image.
 
-If you want to test with `initrd`, make sure you have `initrd = /usr/share/kata-containers/kata-containers-initrd.img`
-in your configuration file, commenting out the `image` line:
+If you want to test with `initrd`, make sure you have uncommented `initrd = /usr/share/kata-containers/kata-containers-initrd.img`
+in your configuration file, commenting out the `image` line in
+`/etc/kata-containers/configuration.toml`. For example:
 
-`/usr/share/defaults/kata-containers/configuration.toml` and comment out the `image` line with the following. For example:
-
-```
-$ sudo mkdir -p /etc/kata-containers/
-$ sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers
+```bash
 $ sudo sed -i 's/^\(image =.*\)/# \1/g' /etc/kata-containers/configuration.toml
+$ sudo sed -i 's/^# \(initrd =.*\)/\1/g' /etc/kata-containers/configuration.toml
 ```
 You can create the initrd image as shown in the [create an initrd image](#create-an-initrd-image---optional) section.
 
-If you want to test with a rootfs `image`, make sure you have `image = /usr/share/kata-containers/kata-containers.img`
+If you want to test with a rootfs `image`, make sure you have uncommented `image = /usr/share/kata-containers/kata-containers.img`
 in your configuration file, commenting out the `initrd` line. For example:
 
-```
-$ sudo mkdir -p /etc/kata-containers/
-$ sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers
+```bash
 $ sudo sed -i 's/^\(initrd =.*\)/# \1/g' /etc/kata-containers/configuration.toml
 ```
 The rootfs image is created as shown in the [create a rootfs image](#create-a-rootfs-image) section.
@@ -90,19 +82,38 @@ rootfs `image`(100MB+).
 
 Enable seccomp as follows:
 
-```
+```bash
 $ sudo sed -i '/^disable_guest_seccomp/ s/true/false/' /etc/kata-containers/configuration.toml
 ```
 
 This will pass container seccomp profiles to the kata agent.
 
+## Enable SELinux on the guest
+
+> **Note:**
+>
+> - To enable SELinux on the guest, SELinux MUST be also enabled on the host.
+> - You MUST create and build a rootfs image for SELinux in advance.
+>   See [Create a rootfs image](#create-a-rootfs-image) and [Build a rootfs image](#build-a-rootfs-image).
+> - SELinux on the guest is supported in only a rootfs image currently, so
+>   you cannot enable SELinux with the agent init (`AGENT_INIT=yes`) yet.
+
+Enable guest SELinux in Enforcing mode as follows:
+
+```
+$ sudo sed -i '/^disable_guest_selinux/ s/true/false/g' /etc/kata-containers/configuration.toml
+```
+
+The runtime automatically will set `selinux=1` to the kernel parameters and `xattr` option to
+`virtiofsd` when `disable_guest_selinux` is set to `false`.
+
+If you want to enable SELinux in Permissive mode, add `enforcing=0` to the kernel parameters.
+
 ## Enable full debug
 
 Enable full debug as follows:
 
-```
-$ sudo mkdir -p /etc/kata-containers/
-$ sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers
+```bash
 $ sudo sed -i -e 's/^# *\(enable_debug\).*=.*$/\1 = true/g' /etc/kata-containers/configuration.toml
 $ sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.log=debug initcall_debug"/g' /etc/kata-containers/configuration.toml
 ```
@@ -175,7 +186,7 @@ and offers possible workarounds and fixes.
 it stores. When messages are suppressed, it is noted in the logs. This can be checked
 for by looking for those notifications, such as:
 
-```sh
+```bash
 $ sudo journalctl --since today | fgrep Suppressed
 Jun 29 14:51:17 mymachine systemd-journald[346]: Suppressed 4150 messages from /system.slice/docker.service
 ```
@@ -200,7 +211,7 @@ RateLimitBurst=0
 
 Restart `systemd-journald` for the changes to take effect:
 
-```sh
+```bash
 $ sudo systemctl restart systemd-journald
 ```
 
@@ -214,39 +225,52 @@ $ sudo systemctl restart systemd-journald
 
 The agent is built with a statically linked `musl.` The default `libc` used is `musl`, but on `ppc64le` and `s390x`, `gnu` should be used. To configure this:
 
-```
-$ export ARCH=$(uname -m)
+```bash
+$ export ARCH="$(uname -m)"
 $ if [ "$ARCH" = "ppc64le" -o "$ARCH" = "s390x" ]; then export LIBC=gnu; else export LIBC=musl; fi
-$ [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
-$ rustup target add ${ARCH}-unknown-linux-${LIBC}
+$ [ "${ARCH}" == "ppc64le" ] && export ARCH=powerpc64le
+$ rustup target add "${ARCH}-unknown-linux-${LIBC}"
 ```
 
 To build the agent:
 
-```
-$ go get -d -u github.com/kata-containers/kata-containers
-$ cd $GOPATH/src/github.com/kata-containers/kata-containers/src/agent && make
-```
-
 The agent is built with seccomp capability by default.
 If you want to build the agent without the seccomp capability, you need to run `make` with `SECCOMP=no` as follows.
 
+```bash
+$ make -C kata-containers/src/agent SECCOMP=no
 ```
-$ make -C $GOPATH/src/github.com/kata-containers/kata-containers/src/agent SECCOMP=no
+
+For building the agent with seccomp support using `musl`, set the environment
+variables for the [`libseccomp` crate](https://github.com/libseccomp-rs/libseccomp-rs).
+
+```bash
+$ export LIBSECCOMP_LINK_TYPE=static
+$ export LIBSECCOMP_LIB_PATH="the path of the directory containing libseccomp.a"
+$ make -C kata-containers/src/agent
 ```
 
+If the compilation fails when the agent tries to link the `libseccomp` library statically
+against `musl`, you will need to build `libseccomp` manually with `-U_FORTIFY_SOURCE`.
+You can use [our script](https://github.com/kata-containers/kata-containers/blob/main/ci/install_libseccomp.sh)
+to install `libseccomp` for the agent.
+
+```bash
+$ mkdir -p ${seccomp_install_path} ${gperf_install_path}
+$ kata-containers/ci/install_libseccomp.sh ${seccomp_install_path} ${gperf_install_path}
+$ export LIBSECCOMP_LIB_PATH="${seccomp_install_path}/lib"
+```
+
+On `ppc64le` and `s390x`, `glibc` is used. You will need to install the `libseccomp` library
+provided by your distribution.
+
+> e.g. `libseccomp-dev` for Ubuntu, or `libseccomp-devel` for CentOS
+
 > **Note:**
 >
 > - If you enable seccomp in the main configuration file but build the agent without seccomp capability,
 >   the runtime exits conservatively with an error message.
 
-## Get the osbuilder
-
-```
-$ go get -d -u github.com/kata-containers/kata-containers
-$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder
-```
-
 ## Create a rootfs image
 ### Create a local rootfs
 
@@ -254,24 +278,32 @@ As a prerequisite, you need to install Docker. Otherwise, you will not be
 able to run the `rootfs.sh` script with `USE_DOCKER=true` as expected in
 the following example.
 
-```
-$ export ROOTFS_DIR=${GOPATH}/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder/rootfs
-$ sudo rm -rf ${ROOTFS_DIR}
-$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder
-$ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true ./rootfs.sh ${distro}'
+```bash
+$ export distro="ubuntu" # example
+$ export ROOTFS_DIR="$(realpath kata-containers/tools/osbuilder/rootfs-builder/rootfs)"
+$ sudo rm -rf "${ROOTFS_DIR}"
+$ pushd kata-containers/tools/osbuilder/rootfs-builder
+$ script -fec 'sudo -E USE_DOCKER=true ./rootfs.sh "${distro}"'
+$ popd
 ```
 
 You MUST choose a distribution (e.g., `ubuntu`) for `${distro}`.
 You can get a supported distributions list in the Kata Containers by running the following.
 
-```
-$ ./rootfs.sh -l
+```bash
+$ ./kata-containers/tools/osbuilder/rootfs-builder/rootfs.sh -l
 ```
 
 If you want to build the agent without seccomp capability, you need to run the `rootfs.sh` script with `SECCOMP=no` as follows.
 
+```bash
+$ script -fec 'sudo -E AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh "${distro}"'
 ```
-$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
+
+If you want to enable SELinux on the guest, you MUST choose `centos` and run the `rootfs.sh` script with `SELINUX=yes` as follows.
+
+```
+$ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true SELINUX=yes ./rootfs.sh centos'
 ```
 
 > **Note:**
@@ -287,18 +319,32 @@ $ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no
 >
 > - You should only do this step if you are testing with the latest version of the agent.
 
-```
-$ sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/usr/bin ../../../src/agent/target/x86_64-unknown-linux-musl/release/kata-agent
-$ sudo install -o root -g root -m 0440 ../../../src/agent/kata-agent.service ${ROOTFS_DIR}/usr/lib/systemd/system/
-$ sudo install -o root -g root -m 0440 ../../../src/agent/kata-containers.target ${ROOTFS_DIR}/usr/lib/systemd/system/
+```bash
+$ sudo install -o root -g root -m 0550 -t "${ROOTFS_DIR}/usr/bin" "${ROOTFS_DIR}/../../../../src/agent/target/x86_64-unknown-linux-musl/release/kata-agent"
+$ sudo install -o root -g root -m 0440 "${ROOTFS_DIR}/../../../../src/agent/kata-agent.service" "${ROOTFS_DIR}/usr/lib/systemd/system/"
+$ sudo install -o root -g root -m 0440 "${ROOTFS_DIR}/../../../../src/agent/kata-containers.target" "${ROOTFS_DIR}/usr/lib/systemd/system/"
 ```
 
 ### Build a rootfs image
 
+```bash
+$ pushd  kata-containers/tools/osbuilder/image-builder
+$ script -fec 'sudo -E USE_DOCKER=true ./image_builder.sh "${ROOTFS_DIR}"'
+$ popd
 ```
-$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/image-builder
-$ script -fec 'sudo -E USE_DOCKER=true ./image_builder.sh ${ROOTFS_DIR}'
+
+If you want to enable SELinux on the guest, you MUST run the `image_builder.sh` script with `SELINUX=yes`
+to label the guest image as follows.
+To label the image on the host, you need to make sure that SELinux is enabled (`selinuxfs` is mounted) on the host
+and the rootfs MUST be created by running the `rootfs.sh` with `SELINUX=yes`.
+
 ```
+$ script -fec 'sudo -E USE_DOCKER=true SELINUX=yes ./image_builder.sh ${ROOTFS_DIR}'
+```
+
+Currently, the `image_builder.sh` uses `chcon` as an interim solution in order to apply `container_runtime_exec_t`
+to the `kata-agent`. Hence, if you run `restorecon` to the guest image after running the `image_builder.sh`,
+the `kata-agent` needs to be labeled `container_runtime_exec_t` again by yourself.
 
 > **Notes:**
 >
@@ -309,25 +355,31 @@ $ script -fec 'sudo -E USE_DOCKER=true ./image_builder.sh ${ROOTFS_DIR}'
 >   variable in the previous command and ensure the `qemu-img` command is
 >   available on your system.
 >   - If `qemu-img` is not installed, you will likely see errors such as `ERROR: File /dev/loop19p1 is not a block device` and `losetup: /tmp/tmp.bHz11oY851: Warning: file is smaller than 512 bytes; the loop device may be useless or invisible for system tools`. These can be mitigated by installing the `qemu-img` command (available in the `qemu-img` package on Fedora or the `qemu-utils` package on Debian).
+> - If `loop` module is not probed, you will likely see errors such as `losetup: cannot find an unused loop device`. Execute `modprobe loop` could resolve it.
 
 
 ### Install the rootfs image
 
-```
-$ commit=$(git log --format=%h -1 HEAD)
-$ date=$(date +%Y-%m-%d-%T.%N%z)
+```bash
+$ pushd kata-containers/tools/osbuilder/image-builder
+$ commit="$(git log --format=%h -1 HEAD)"
+$ date="$(date +%Y-%m-%d-%T.%N%z)"
 $ image="kata-containers-${date}-${commit}"
 $ sudo install -o root -g root -m 0640 -D kata-containers.img "/usr/share/kata-containers/${image}"
 $ (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
+$ popd
 ```
 
 ## Create an initrd image - OPTIONAL
 ### Create a local rootfs for initrd image
-```
-$ export ROOTFS_DIR="${GOPATH}/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder/rootfs"
-$ sudo rm -rf ${ROOTFS_DIR}
-$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder
-$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true ./rootfs.sh ${distro}'
+
+```bash
+$ export distro="ubuntu" # example
+$ export ROOTFS_DIR="$(realpath kata-containers/tools/osbuilder/rootfs-builder/rootfs)"
+$ sudo rm -rf "${ROOTFS_DIR}"
+$ pushd kata-containers/tools/osbuilder/rootfs-builder/
+$ script -fec 'sudo -E AGENT_INIT=yes USE_DOCKER=true ./rootfs.sh "${distro}"'
+$ popd
 ```
 `AGENT_INIT` controls if the guest image uses the Kata agent as the guest `init` process. When you create an initrd image,
 always set `AGENT_INIT` to `yes`.
@@ -335,14 +387,14 @@ always set `AGENT_INIT` to `yes`.
 You MUST choose a distribution (e.g., `ubuntu`) for `${distro}`.
 You can get a supported distributions list in the Kata Containers by running the following.
 
-```
-$ ./rootfs.sh -l
+```bash
+$ ./kata-containers/tools/osbuilder/rootfs-builder/rootfs.sh -l
 ```
 
 If you want to build the agent without seccomp capability, you need to run the `rootfs.sh` script with `SECCOMP=no` as follows.
 
-```
-$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
+```bash
+$ script -fec 'sudo -E AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh "${distro}"'
 ```
 
 > **Note:**
@@ -351,28 +403,31 @@ $ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no
 
 Optionally, add your custom agent binary to the rootfs with the following commands. The default `$LIBC` used
 is `musl`, but on ppc64le and s390x, `gnu` should be used. Also, Rust refers to ppc64le as `powerpc64le`:
-```
-$ export ARCH=$(uname -m)
-$ [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
-$ [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
-$ sudo install -o root -g root -m 0550 -T ../../../src/agent/target/${ARCH}-unknown-linux-${LIBC}/release/kata-agent ${ROOTFS_DIR}/sbin/init
+```bash
+$ export ARCH="$(uname -m)"
+$ [ "${ARCH}" == "ppc64le" ] || [ "${ARCH}" == "s390x" ] && export LIBC=gnu || export LIBC=musl
+$ [ "${ARCH}" == "ppc64le" ] && export ARCH=powerpc64le
+$ sudo install -o root -g root -m 0550 -T "${ROOTFS_DIR}/../../../../src/agent/target/${ARCH}-unknown-linux-${LIBC}/release/kata-agent" "${ROOTFS_DIR}/sbin/init"
 ```
 
 ### Build an initrd image
 
-```
-$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/initrd-builder
-$ script -fec 'sudo -E AGENT_INIT=yes USE_DOCKER=true ./initrd_builder.sh ${ROOTFS_DIR}'
+```bash
+$ pushd kata-containers/tools/osbuilder/initrd-builder
+$ script -fec 'sudo -E AGENT_INIT=yes USE_DOCKER=true ./initrd_builder.sh "${ROOTFS_DIR}"'
+$ popd
 ```
 
 ### Install the initrd image
 
-```
-$ commit=$(git log --format=%h -1 HEAD)
-$ date=$(date +%Y-%m-%d-%T.%N%z)
+```bash
+$ pushd kata-containers/tools/osbuilder/initrd-builder
+$ commit="$(git log --format=%h -1 HEAD)"
+$ date="$(date +%Y-%m-%d-%T.%N%z)"
 $ image="kata-containers-initrd-${date}-${commit}"
 $ sudo install -o root -g root -m 0640 -D kata-containers-initrd.img "/usr/share/kata-containers/${image}"
 $ (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers-initrd.img)
+$ popd
 ```
 
 # Install guest kernel images
@@ -391,44 +446,44 @@ Kata Containers makes use of upstream QEMU branch. The exact version
 and repository utilized can be found by looking at the [versions file](../versions.yaml).
 
 Find the correct version of QEMU from the versions file:
-```
-$ source ${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging/scripts/lib.sh
-$ qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version")
-$ echo ${qemu_version}
+```bash
+$ source kata-containers/tools/packaging/scripts/lib.sh
+$ qemu_version="$(get_from_kata_deps "assets.hypervisor.qemu.version")"
+$ echo "${qemu_version}"
 ```
 Get source from the matching branch of QEMU:
-```
-$ go get -d github.com/qemu/qemu
-$ cd ${GOPATH}/src/github.com/qemu/qemu
-$ git checkout ${qemu_version}
-$ your_qemu_directory=${GOPATH}/src/github.com/qemu/qemu
+```bash
+$ git clone -b "${qemu_version}" https://github.com/qemu/qemu.git
+$ your_qemu_directory="$(realpath qemu)"
 ```
 
 There are scripts to manage the build and packaging of QEMU. For the examples below, set your
 environment as:
-```
-$ go get -d github.com/kata-containers/kata-containers
-$ packaging_dir="${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging"
+```bash
+$ packaging_dir="$(realpath kata-containers/tools/packaging)"
 ```
 
 Kata often utilizes patches for not-yet-upstream and/or backported fixes for components,
 including QEMU. These can be found in the [packaging/QEMU directory](../tools/packaging/qemu/patches),
 and it's *recommended* that you apply them. For example, suppose that you are going to build QEMU
 version 5.2.0, do:
-```
-$ cd $your_qemu_directory
-$ $packaging_dir/scripts/apply_patches.sh $packaging_dir/qemu/patches/5.2.x/
+```bash
+$ "$packaging_dir/scripts/apply_patches.sh" "$packaging_dir/qemu/patches/5.2.x/"
 ```
 
 To build utilizing the same options as Kata, you should make use of the `configure-hypervisor.sh` script. For example:
-```
-$ cd $your_qemu_directory
-$ $packaging_dir/scripts/configure-hypervisor.sh kata-qemu > kata.cfg
+```bash
+$ pushd "$your_qemu_directory"
+$ "$packaging_dir/scripts/configure-hypervisor.sh" kata-qemu > kata.cfg
 $ eval ./configure "$(cat kata.cfg)"
 $ make -j $(nproc --ignore=1)
+# Optional
 $ sudo -E make install
+$ popd
 ```
 
+If you do not want to install the respective QEMU version, the configuration file can be modified to point to the correct binary. In `/etc/kata-containers/configuration.toml`, change `path = "/path/to/qemu/build/qemu-system-x86_64"` to point to the correct QEMU binary.
+
 See the [static-build script for QEMU](../tools/packaging/static-build/qemu/build-static-qemu.sh) for a reference on how to get, setup, configure and build QEMU for Kata.
 
 ### Build a custom QEMU for aarch64/arm64 - REQUIRED
@@ -439,11 +494,33 @@ See the [static-build script for QEMU](../tools/packaging/static-build/qemu/buil
 >   under upstream review for supporting NVDIMM on aarch64.
 >
 You could build the custom `qemu-system-aarch64` as required with the following command:
+```bash
+$ git clone https://github.com/kata-containers/tests.git
+$ script -fec 'sudo -E tests/.ci/install_qemu.sh'
 ```
-$ go get -d github.com/kata-containers/tests
-$ script -fec 'sudo -E ${GOPATH}/src/github.com/kata-containers/tests/.ci/install_qemu.sh'
+
+## Build `virtiofsd`
+
+When using the file system type virtio-fs (default), `virtiofsd` is required
+
+```bash
+$ pushd kata-containers/tools/packaging/static-build/virtiofsd
+$ ./build.sh
+$ popd
 ```
 
+Modify `/etc/kata-containers/configuration.toml` and update value `virtio_fs_daemon = "/path/to/kata-containers/tools/packaging/static-build/virtiofsd/virtiofsd/virtiofsd"` to point to the binary.
+
+# Check hardware requirements
+
+You can check if your system is capable of creating a Kata Container by running the following:
+
+```bash
+$ sudo kata-runtime check
+```
+
+If your system is *not* able to run Kata Containers, the previous command will error out and explain why.
+
 # Run Kata Containers with Containerd
 Refer to the [How to use Kata Containers and Containerd](how-to/containerd-kata.md) how-to guide.
 
@@ -474,7 +551,7 @@ See [Set up a debug console](#set-up-a-debug-console).
 
 ## Checking Docker default runtime
 
-```
+```bash
 $ sudo docker info 2>/dev/null | grep -i "default runtime" | cut -d: -f2- | grep -q runc  && echo "SUCCESS" || echo "ERROR: Incorrect default Docker runtime"
 ```
 ## Set up a debug console
@@ -491,7 +568,7 @@ contain either `/bin/sh` or `/bin/bash`.
 
 Enable debug_console_enabled in the `configuration.toml` configuration file:
 
-```
+```toml
 [agent.kata]
 debug_console_enabled = true
 ```
@@ -502,7 +579,7 @@ This will pass `agent.debug_console agent.debug_console_vport=1026` to agent as
 
 For Kata Containers `2.0.x` releases, the `kata-runtime exec` command depends on the`kata-monitor` running, in order to get the sandbox's `vsock` address to connect to. Thus, first start the `kata-monitor` process.
 
-```
+```bash
 $ sudo kata-monitor
 ```
 
@@ -564,10 +641,10 @@ an additional `coreutils` package.
 
 For example using CentOS:
 
-```
-$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder
-$ export ROOTFS_DIR=${GOPATH}/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder/rootfs
-$ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true EXTRA_PKGS="bash coreutils" ./rootfs.sh centos'
+```bash
+$ pushd kata-containers/tools/osbuilder/rootfs-builder
+$ export ROOTFS_DIR="$(realpath ./rootfs)"
+$ script -fec 'sudo -E USE_DOCKER=true EXTRA_PKGS="bash coreutils" ./rootfs.sh centos'
 ```
 
 #### Build the debug image
@@ -579,12 +656,13 @@ section when using rootfs, or when using initrd, complete the steps in the [Buil
 
 Install the image:
 
->**Note**: When using an initrd image, replace the below rootfs image name `kata-containers.img` 
+>**Note**: When using an initrd image, replace the below rootfs image name `kata-containers.img`
 >with the initrd image name `kata-containers-initrd.img`.
 
-```
+```bash
 $ name="kata-containers-centos-with-debug-console.img"
 $ sudo install -o root -g root -m 0640 kata-containers.img "/usr/share/kata-containers/${name}"
+$ popd
 ```
 
 Next, modify the `image=` values in the `[hypervisor.qemu]` section of the
@@ -593,7 +671,7 @@ to specify the full path to the image name specified in the previous code
 section. Alternatively, recreate the symbolic link so it points to
 the new debug image:
 
-```
+```bash
 $ (cd /usr/share/kata-containers && sudo ln -sf "$name" kata-containers.img)
 ```
 
@@ -604,7 +682,7 @@ to avoid all subsequently created containers from using the debug image.
 
 Create a container as normal. For example using `crictl`:
 
-```
+```bash
 $ sudo crictl run -r kata container.yaml pod.yaml
 ```
 
@@ -612,25 +690,25 @@ $ sudo crictl run -r kata container.yaml pod.yaml
 
 The steps required to enable debug console for QEMU slightly differ with
 those for firecracker / cloud-hypervisor.
- 
+
 ##### Enabling debug console for QEMU
 
-Add `agent.debug_console` to the guest kernel command line to allow the agent process to start a debug console. 
+Add `agent.debug_console` to the guest kernel command line to allow the agent process to start a debug console.
 
-```
+```bash
 $ sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.debug_console"/g' "${kata_configuration_file}"
 ```
 
-Here `kata_configuration_file` could point to `/etc/kata-containers/configuration.toml` 
+Here `kata_configuration_file` could point to `/etc/kata-containers/configuration.toml`
 or `/usr/share/defaults/kata-containers/configuration.toml`
 or `/opt/kata/share/defaults/kata-containers/configuration-{hypervisor}.toml`, if
 you installed Kata Containers using `kata-deploy`.
 
 ##### Enabling debug console for cloud-hypervisor / firecracker
 
-Slightly different configuration is required in case of firecracker and cloud hypervisor. 
-Firecracker and cloud-hypervisor don't have a UNIX socket connected to `/dev/console`. 
-Hence, the kernel command line option `agent.debug_console` will not work for them. 
+Slightly different configuration is required in case of firecracker and cloud hypervisor.
+Firecracker and cloud-hypervisor don't have a UNIX socket connected to `/dev/console`.
+Hence, the kernel command line option `agent.debug_console` will not work for them.
 These hypervisors support `hybrid vsocks`,  which can be used for communication
 between the host and the guest. The kernel command line option `agent.debug_console_vport`
  was added to allow developers specify on which `vsock` port the debugging console should be connected.
@@ -638,12 +716,12 @@ between the host and the guest. The kernel command line option `agent.debug_cons
 
 Add the parameter `agent.debug_console_vport=1026` to the kernel command line
 as shown below:
-```
+```bash
 sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.debug_console_vport=1026"/g' "${kata_configuration_file}"
 ```
 
 > **Note** Ports 1024 and 1025 are reserved for communication with the agent
-> and gathering of agent logs respectively. 
+> and gathering of agent logs respectively.
 
 ##### Connecting to the debug console
 
@@ -651,7 +729,7 @@ Next, connect to the debug console. The VSOCKS paths vary slightly between each
 VMM solution.
 
 In case of cloud-hypervisor, connect to the `vsock` as shown:
-```
+```bash
 $ sudo su -c 'cd /var/run/vc/vm/${sandbox_id}/root/ && socat stdin unix-connect:clh.sock'
 CONNECT 1026
 ```
@@ -659,7 +737,7 @@ CONNECT 1026
 **Note**: You need to type `CONNECT 1026` and press `RETURN` key after entering the `socat` command.
 
 For firecracker, connect to the `hvsock` as shown:
-```
+```bash
 $ sudo su -c 'cd /var/run/vc/firecracker/${sandbox_id}/root/ && socat stdin unix-connect:kata.hvsock'
 CONNECT 1026
 ```
@@ -668,7 +746,7 @@ CONNECT 1026
 
 
 For QEMU, connect to the `vsock` as shown:
-```
+```bash
 $ sudo su -c 'cd /var/run/vc/vm/${sandbox_id} && socat "stdin,raw,echo=0,escape=0x11" "unix-connect:console.sock"'
 ```
 
@@ -681,7 +759,7 @@ If the image is created using
 [osbuilder](../tools/osbuilder), the following YAML
 file exists and contains details of the image and how it was created:
 
-```
+```bash
 $ cat /var/lib/osbuilder/osbuilder.yaml
 ```
 

docs/Release-Process.md

@@ -28,23 +28,6 @@
   $ ./update-repository-version.sh -p "$NEW_VERSION" "$BRANCH"
   ```
 
-### Point tests repository to stable branch
-
-  If you create a new stable branch, i.e. if your release changes a major or minor version number (not a patch release), then
-  you should modify the `tests` repository to point to that newly created stable branch and not the `main` branch.
-  The objective is that changes in the CI on the main branch will not impact the stable branch.
-
-  In the test directory, change references the main branch in:
-  * `README.md`
-  * `versions.yaml`
-  * `cmd/github-labels/labels.yaml.in`
-  * `cmd/pmemctl/pmemctl.sh`
-  * `.ci/lib.sh`
-  * `.ci/static-checks.sh`
-
-  See the commits in [the corresponding PR for stable-2.1](https://github.com/kata-containers/tests/pull/3504) for an example of the changes.
-
-
 ### Merge all bump version Pull requests
 
   - The above step will create a GitHub pull request in the Kata projects. Trigger the CI using `/test` command on each bump Pull request.
@@ -63,6 +46,24 @@
   $ ./tag_repos.sh -p -b "$BRANCH" tag
   ```
 
+### Point tests repository to stable branch
+
+  If your release changes a major or minor version number(not a patch release), then the above 
+  `./tag_repos.sh` script will create a new stable branch in all the repositories in addition to tagging them.
+  This happens when you are making the first `rc` release for a new major or minor version in Kata.
+  In this case, you should modify the `tests` repository to point to the newly created stable branch and not the `main` branch.
+  The objective is that changes in the CI on the main branch will not impact the stable branch.
+
+  In the test directory, change references of the `main` branch to the new stable branch in:
+  * `README.md`
+  * `versions.yaml`
+  * `cmd/github-labels/labels.yaml.in`
+  * `cmd/pmemctl/pmemctl.sh`
+  * `.ci/lib.sh`
+  * `.ci/static-checks.sh`
+
+  See the commits in [the corresponding PR for stable-2.1](https://github.com/kata-containers/tests/pull/3504) for an example of the changes.
+
 ### Check Git-hub Actions
 
   We make use of [GitHub actions](https://github.com/features/actions) in this [file](../.github/workflows/release.yaml) in the `kata-containers/kata-containers` repository to build and upload release artifacts. This action is auto triggered with the above step when a new tag is pushed to the `kata-containers/kata-containers` repository.

docs/design/README.md

@@ -7,8 +7,11 @@ Kata Containers design documents:
 - [Design requirements for Kata Containers](kata-design-requirements.md)
 - [VSocks](VSocks.md)
 - [VCPU handling](vcpu-handling.md)
+- [VCPU threads pinning](vcpu-threads-pinning.md)
 - [Host cgroups](host-cgroups.md)
+- [Agent systemd cgroup](agent-systemd-cgroup.md)
 - [`Inotify` support](inotify.md)
+- [`Hooks` support](hooks-handling.md)
 - [Metrics(Kata 2.0)](kata-2-0-metrics.md)
 - [Design for Kata Containers `Lazyload` ability with `nydus`](kata-nydus-design.md)
 - [Design for direct-assigned volume](direct-blk-device-assignment.md)

docs/design/agent-systemd-cgroup.md

@@ -0,0 +1,84 @@
+# Systemd Cgroup for Agent
+
+As we know, we can interact with cgroups in two ways, **`cgroupfs`** and **`systemd`**. The former is achieved by reading and writing cgroup `tmpfs` files under `/sys/fs/cgroup` while the latter is done by configuring a transient unit by requesting systemd. Kata agent uses **`cgroupfs`** by default, unless you pass the parameter `--systemd-cgroup`.
+
+## usage
+
+For systemd, kata agent configures cgroups according to the following `linux.cgroupsPath` format standard provided by `runc` (`[slice]:[prefix]:[name]`). If you don't provide a valid `linux.cgroupsPath`, kata agent will treat it as `"system.slice:kata_agent:<container-id>"`. 
+
+> Here slice is a systemd slice under which the container is placed. If empty, it defaults to system.slice, except when cgroup v2 is used and rootless container is created, in which case it defaults to user.slice.
+>
+> Note that slice can contain dashes to denote a sub-slice (e.g. user-1000.slice is a correct notation, meaning a `subslice` of user.slice), but it must not contain slashes (e.g. user.slice/user-1000.slice is invalid).
+>
+> A slice of `-` represents a root slice.
+>
+> Next, prefix and name are used to compose the unit name, which is `<prefix>-<name>.scope`, unless name has `.slice` suffix, in which case prefix is ignored and the name is used as is.
+
+## supported properties
+
+The kata agent will translate the parameters in the `linux.resources` of `config.json` into systemd unit properties, and send it to systemd for configuration. Since systemd supports limited properties, only the following parameters in `linux.resources` will be applied. We will simply treat hybrid mode as legacy mode by the way.
+
+- CPU
+
+  - v1
+
+  | runtime spec resource | systemd property name |
+  | --------------------- | --------------------- |
+  | `cpu.shares`          | `CPUShares`           |
+
+  - v2
+
+  | runtime spec resource      | systemd property name      |
+  | -------------------------- | -------------------------- |
+  | `cpu.shares`               | `CPUShares`                |
+  | `cpu.period`               | `CPUQuotaPeriodUSec`(v242) |
+  | `cpu.period` & `cpu.quota` | `CPUQuotaPerSecUSec`       |
+
+- MEMORY
+
+  - v1
+
+  | runtime spec resource | systemd property name |
+  | --------------------- | --------------------- |
+  | `memory.limit`        | `MemoryLimit`         |
+
+  - v2
+
+  | runtime spec resource          | systemd property name |
+  | ------------------------------ | --------------------- |
+  | `memory.low`                   | `MemoryLow`           |
+  | `memory.max`                   | `MemoryMax`           |
+  | `memory.swap` & `memory.limit` | `MemorySwapMax`       |
+
+- PIDS
+
+  | runtime spec resource | systemd property name |
+  | --------------------- | --------------------- |
+  | `pids.limit `         | `TasksMax`            |
+
+- CPUSET
+
+  | runtime spec resource | systemd property name      |
+  | --------------------- | -------------------------- |
+  | `cpuset.cpus`         | `AllowedCPUs`(v244)        |
+  | `cpuset.mems`         | `AllowedMemoryNodes`(v244) |
+
+## Systemd Interface
+
+`session.rs` and `system.rs` in `src/agent/rustjail/src/cgroups/systemd/interface` are automatically generated by `zbus-xmlgen`, which is is an accompanying tool provided by `zbus` to generate Rust code from `D-Bus XML interface descriptions`. The specific commands to generate these two files are as follows: 
+
+```shell
+// system.rs
+zbus-xmlgen --system org.freedesktop.systemd1 /org/freedesktop/systemd1
+// session.rs
+zbus-xmlgen --session org.freedesktop.systemd1 /org/freedesktop/systemd1
+```
+
+The current implementation of `cgroups/systemd` uses `system.rs` while `session.rs` could be used to build rootless containers in the future.
+
+## references
+
+- [runc - systemd cgroup driver](https://github.com/opencontainers/runc/blob/main/docs/systemd.md)
+
+- [systemd.resource-control  — Resource control unit settings](https://www.freedesktop.org/software/systemd/man/systemd.resource-control.html)
+

docs/design/architecture/networking.md

@@ -36,7 +36,7 @@ compatibility, and performance on par with MACVTAP.
 Kata Containers has deprecated support for bridge due to lacking performance relative to TC-filter and MACVTAP.
 
 Kata Containers supports both
-[CNM](https://github.com/docker/libnetwork/blob/master/docs/design.md#the-container-network-model)
+[CNM](https://github.com/moby/libnetwork/blob/master/docs/design.md#the-container-network-model)
 and [CNI](https://github.com/containernetworking/cni) for networking management.
 
 ## Network Hotplug

docs/design/architecture_3.0/README.md

@@ -64,8 +64,8 @@ The kata-runtime is controlled by TOKIO_RUNTIME_WORKER_THREADS to run the OS thr
   ├─ TTRPC listener thread(M * tokio task)
   ├─ TTRPC client handler thread(7 * M * tokio task)
   ├─ container stdin io thread(M * tokio task)
-  ├─ container stdin io thread(M * tokio task)
-  └─ container stdin io thread(M * tokio task)
+  ├─ container stdout io thread(M * tokio task)
+  └─ container stderr io thread(M * tokio task)
 ```
 ### Extensible Framework
 The Kata 3.x runtime is designed with the extension of service, runtime, and hypervisor, combined with configuration to meet the needs of different scenarios. At present, the service provides a register mechanism to support multiple services. Services could interact with runtime through messages. In addition, the runtime handler handles messages from services. To meet the needs of a binary that supports multiple runtimes and hypervisors, the startup must obtain the runtime handler type and hypervisor type through configuration.

docs/design/direct-blk-device-assignment.md

@@ -81,7 +81,7 @@ Notes: given that the `mountInfo` is persisted to the disk by the Kata runtime,
 Instead of the CSI node driver writing the mount info into a `csiPlugin.json` file under the volume root, 
 as described in the original proposal, here we propose that the CSI node driver passes the mount information to 
 the Kata Containers runtime through a new `kata-runtime` commandline command. The `kata-runtime` then writes the mount 
-information to a `mount-info.json` file in a predefined location (`/run/kata-containers/shared/direct-volumes/[volume_path]/`).
+information to a `mountInfo.json` file in a predefined location (`/run/kata-containers/shared/direct-volumes/[volume_path]/`).
 
 When the Kata Containers runtime starts a container, it verifies whether a volume mount is a direct-assigned volume by checking 
 whether there is a `mountInfo` file under the computed Kata `direct-volumes` directory. If it is, the runtime parses the `mountInfo` file, 

docs/design/hooks-handling.md

@@ -0,0 +1,63 @@
+# Kata Containers support for `Hooks`
+
+## Introduction
+
+During container's lifecycle, different Hooks can be executed to do custom actions. In Kata Containers, we support two types of Hooks, `OCI Hooks` and `Kata Hooks`.
+
+### OCI Hooks
+
+The OCI Spec stipulates six hooks that can be executed at different time points and namespaces, including `Prestart Hooks`, `CreateRuntime Hooks`, `CreateContainer Hooks`, `StartContainer Hooks`, `Poststart Hooks` and `Poststop Hooks`. We support these types of Hooks as compatible as possible in Kata Containers.
+
+The path and arguments of these hooks will be passed to Kata for execution via `bundle/config.json`. For example:
+```
+...
+"hooks": {
+  "prestart": [
+    {
+      "path": "/usr/bin/prestart-hook",
+      "args": ["prestart-hook", "arg1", "arg2"],
+      "env":  [ "key1=value1"]
+    }
+  ],
+  "createRuntime": [
+    {
+      "path": "/usr/bin/createRuntime-hook",
+      "args": ["createRuntime-hook", "arg1", "arg2"],
+      "env":  [ "key1=value1"]
+    }
+  ]
+}
+...
+```
+
+### Kata Hooks
+
+In Kata, we support another three kinds of hooks executed in guest VM, including `Guest Prestart Hook`, `Guest Poststart Hook`, `Guest Poststop Hook`.
+
+The executable files for Kata Hooks must be packaged in the *guest rootfs*. The file path to those guest hooks should be specified in the configuration file, and guest hooks must be stored in a subdirectory of `guest_hook_path` according to their hook type. For example:
+
++ In configuration file:
+```
+guest_hook_path="/usr/share/hooks"
+```
++ In guest rootfs, prestart-hook is stored in `/usr/share/hooks/prestart/prestart-hook`.
+
+## Execution
+The table below summarized when and where those different hooks will be executed in Kata Containers:
+
+| Hook Name | Hook Type | Hook Path | Exec Place | Exec Time |
+|---|---|---|---|---|
+| `Prestart(deprecated)` | OCI hook | host runtime namespace | host runtime namespace | After VM is started, before container is created. |
+| `CreateRuntime` | OCI hook | host runtime namespace | host runtime namespace | After VM is started, before container is created, after `Prestart` hooks. |
+| `CreateContainer` | OCI hook | host runtime namespace | host vmm namespace* | After VM is started, before container is created, after `CreateRuntime` hooks. |
+| `StartContainer` | OCI hook | guest container namespace | guest container namespace | After container is created, before container is started. |
+| `Poststart` | OCI hook | host runtime namespace | host runtime namespace | After container is started, before start operation returns. |
+| `Poststop` | OCI hook | host runtime namespace | host runtime namespace | After container is deleted, before delete operation returns. |
+| `Guest Prestart` | Kata hook | guest agent namespace | guest agent namespace | During start operation, before container command is executed. |
+| `Guest Poststart` | Kata hook | guest agent namespace | guest agent namespace | During start operation, after container command is executed, before start operation returns. |
+| `Guest Poststop` | Kata hook | guest agent namespace | guest agent namespace | During delete operation, after container is deleted, before delete operation returns. |
+
++ `Hook Path` specifies where hook's path be resolved.
++ `Exec Place` specifies in which namespace those hooks can be executed.
+  + For `CreateContainer` Hooks, OCI requires to run them inside the container namespace while the hook executable path is in the host runtime, which is a non-starter for VM-based containers. So we design to keep them running in the *host vmm namespace.* 
++ `Exec Time` specifies at which time point those hooks can be executed.

docs/design/vcpu-threads-pinning.md

@@ -0,0 +1,37 @@
+# Design Doc for Kata Containers' VCPUs Pinning Feature
+
+## Background
+By now, vCPU threads of Kata Containers are scheduled randomly to CPUs. And each pod would request a specific set of CPUs which we call it CPU set (just the CPU set meaning in Linux cgroups).    
+
+If the number of vCPU threads are equal to that of CPUs claimed in CPU set, we can then pin each vCPU thread to one specified CPU, to reduce the cost of random scheduling. 
+
+## Detailed Design
+
+### Passing Config Parameters
+Two ways are provided to use this vCPU thread pinning feature: through `QEMU` configuration file and through annotations. Finally the pinning parameter is passed to `HypervisorConfig`.
+
+### Related Linux Thread Scheduling API
+
+| API Info          | Value                                                     |
+|-------------------|-----------------------------------------------------------|
+| Package           | `golang.org/x/sys/unix`                                     |
+| Method            | `unix.SchedSetaffinity(thread_id, &unixCPUSet)`             |
+| Official Doc Page | https://pkg.go.dev/golang.org/x/sys/unix#SchedSetaffinity |
+
+### When is VCPUs Pinning Checked?
+
+As shown in Section 1, when `num(vCPU threads) == num(CPUs in CPU set)`, we shall pin each vCPU thread to a specified CPU. And when this condition is broken, we should restore to the original random scheduling pattern.  
+So when may `num(CPUs in CPU set)` change? There are 5 possible scenes:
+
+| Possible scenes                   | Related Code                               |
+|-----------------------------------|--------------------------------------------|
+| when creating a container         | File Sandbox.go, in method `CreateContainer`  |
+| when starting a container         | File Sandbox.go, in method `StartContainer`   |
+| when deleting a container         | File Sandbox.go, in method `DeleteContainer`  |
+| when updating a container         | File Sandbox.go, in method `UpdateContainer`  |
+| when creating multiple containers | File Sandbox.go, in method `createContainers` |
+
+### Core Pinning Logics
+
+We can split the whole process into the following steps. Related methods are `checkVCPUsPinning` and `resetVCPUsPinning`, in file Sandbox.go.
+![](arch-images/vcpus-pinning-process.png) 

docs/design/virtualization.md

@@ -110,7 +110,7 @@ Devices and features used:
 - VFIO
 - hotplug
 - seccomp filters
-- [HTTP OpenAPI](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/vmm/src/api/openapi/cloud-hypervisor.yaml)
+- [HTTP OpenAPI](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/main/vmm/src/api/openapi/cloud-hypervisor.yaml)
 
 ### Summary
 

docs/how-to/README.md

@@ -42,4 +42,6 @@
 - [How to setup swap devices in guest kernel](how-to-setup-swap-devices-in-guest-kernel.md)
 - [How to run rootless vmm](how-to-run-rootless-vmm.md)
 - [How to run Docker with Kata Containers](how-to-run-docker-with-kata.md)
-- [How to run Kata Containers with `nydus`](how-to-use-virtio-fs-nydus-with-kata.md)
+- [How to run Kata Containers with `nydus`](how-to-use-virtio-fs-nydus-with-kata.md)
+- [How to run Kata Containers with AMD SEV-SNP](how-to-run-kata-containers-with-SNP-VMs.md)
+- [How to use EROFS to build rootfs in Kata Containers](how-to-use-erofs-build-rootfs.md)

docs/how-to/containerd-kata.md

@@ -77,8 +77,8 @@ $ command -v containerd
 You can manually install CNI plugins as follows:
 
 ```bash
-$ go get github.com/containernetworking/plugins
-$ pushd $GOPATH/src/github.com/containernetworking/plugins
+$ git clone https://github.com/containernetworking/plugins.git
+$ pushd plugins
 $ ./build_linux.sh
 $ sudo mkdir /opt/cni
 $ sudo cp -r bin /opt/cni/
@@ -93,8 +93,8 @@ $ popd
 You can install the `cri-tools` from source code:
 
 ```bash
-$ go get github.com/kubernetes-sigs/cri-tools
-$ pushd $GOPATH/src/github.com/kubernetes-sigs/cri-tools
+$ git clone https://github.com/kubernetes-sigs/cri-tools.git
+$ pushd cri-tools
 $ make
 $ sudo -E make install
 $ popd
@@ -257,6 +257,48 @@ This launches a BusyBox container named `hello`, and it will be removed by `--rm
 The `--cni` flag enables CNI networking for the container. Without this flag, a container with just a
 loopback interface is created.
 
+### Launch containers using `ctr` command line with rootfs bundle
+
+#### Get rootfs
+Use the script to create rootfs
+```bash
+ctr i pull quay.io/prometheus/busybox:latest
+ctr i export rootfs.tar quay.io/prometheus/busybox:latest
+
+rootfs_tar=rootfs.tar
+bundle_dir="./bundle"
+mkdir -p "${bundle_dir}"
+
+# extract busybox rootfs
+rootfs_dir="${bundle_dir}/rootfs"
+mkdir -p "${rootfs_dir}"
+layers_dir="$(mktemp -d)"
+tar -C "${layers_dir}" -pxf "${rootfs_tar}"
+for ((i=0;i<$(cat ${layers_dir}/manifest.json | jq -r ".[].Layers | length");i++)); do
+  tar -C ${rootfs_dir} -xf ${layers_dir}/$(cat ${layers_dir}/manifest.json | jq -r ".[].Layers[${i}]")
+done
+```
+#### Get `config.json`
+Use runc spec to generate `config.json`
+```bash
+cd ./bundle/rootfs
+runc spec
+mv config.json ../
+```
+Change the root `path` in `config.json` to the absolute path of rootfs
+
+```JSON
+"root":{
+    "path":"/root/test/bundle/rootfs",
+    "readonly": false
+},
+```
+
+#### Run container
+```bash
+sudo ctr run -d --runtime io.containerd.run.kata.v2 --config bundle/config.json hello
+sudo ctr t exec --exec-id ${ID} -t hello sh
+```
 ### Launch Pods with `crictl` command line
 
 With the `crictl` command line of `cri-tools`, you can specify runtime class with `-r` or `--runtime` flag.

docs/how-to/how-to-hotplug-memory-arm64.md

@@ -15,6 +15,18 @@ $ sudo .ci/aarch64/install_rom_aarch64.sh
 $ popd
 ```
 
+## Config KATA QEMU
+
+After executing the above script, two files will be generated under the directory `/usr/share/kata-containers/` by default, namely `kata-flash0.img` and `kata-flash1.img`. Next we need to change the configuration file of `kata qemu`, which is in `/opt/kata/share/defaults/kata-containers/configuration-qemu.toml` by default, specify in the configuration file to use the UEFI ROM installed above. The above is an example of `kata deploy` installation. For package management installation, please use `kata-runtime env` to find the location of the configuration file. Please refer to the following configuration.
+
+```
+[hypervisor.qemu]
+
+# -pflash can add image file to VM. The arguments of it should be in format
+# of ["/path/to/flash0.img", "/path/to/flash1.img"]
+pflashes = ["/usr/share/kata-containers/kata-flash0.img", "/usr/share/kata-containers/kata-flash1.img"]
+```
+
 ## Run for test
 
 Let's test if the memory hotplug is ready for Kata after install the UEFI ROM. Make sure containerd is ready to run Kata before test.

docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md

@@ -0,0 +1,158 @@
+# Kata Containers with AMD SEV-SNP VMs
+
+## Disclaimer
+
+This guide is designed for developers and is - same as the Developer Guide - not intended for production systems or end users. It is advisable to only follow this guide on non-critical development systems.
+
+## Prerequisites
+
+To run  Kata Containers in SNP-VMs, the following software stack is used.
+
+![Kubernetes integration with shimv2](./images/SNP-stack.svg)
+
+The host BIOS and kernel must be capable of supporting AMD SEV-SNP and configured accordingly. For Kata Containers, the host kernel with branch [`sev-snp-iommu-avic_5.19-rc6_v3`](https://github.com/AMDESE/linux/tree/sev-snp-iommu-avic_5.19-rc6_v3) and commit [`3a88547`](https://github.com/AMDESE/linux/commit/3a885471cf89156ea555341f3b737ad2a8d9d3d0) is known to work in conjunction with SEV Firmware version 1.51.3 (0xh\_1.33.03) available on AMD's [SEV developer website](https://developer.amd.com/sev/). See [AMD's guide](https://github.com/AMDESE/AMDSEV/tree/sev-snp-devel) to configure the host accordingly. Verify that you are able to run SEV-SNP encrypted VMs first. The guest components required for Kata Containers are built as described below.
+
+**Tip**: It is easiest to first have Kata Containers running on your system and then modify it to run containers in SNP-VMs. Follow the [Developer guide](../Developer-Guide.md#warning) and then follow the below steps. Nonetheless, you can just follow this guide from the start.
+
+## How to build
+
+Follow all of the below steps to install Kata Containers with SNP-support from scratch. These steps mostly follow the developer guide with modifications to support SNP
+
+__Steps from the Developer Guide:__
+- Get all the [required components](../Developer-Guide.md#requirements-to-build-individual-components) for building the kata-runtime
+- [Build the and install kata-runtime](../Developer-Guide.md#build-and-install-the-kata-containers-runtime)
+- [Build a custom agent](../Developer-Guide.md#build-a-custom-kata-agent---optional)
+- [Create an initrd image](../Developer-Guide.md#create-an-initrd-image---optional) by first building a rootfs, then building the initrd based on the rootfs, use a custom agent and install. `ubuntu` works as the distribution of choice.
+- Get the [required components](../../tools/packaging/kernel/README.md#requirements) to build a custom kernel
+
+__SNP-specific steps:__
+- Build the SNP-specific kernel as shown below (see this [guide](../../tools/packaging/kernel/README.md#build-kata-containers-kernel) for more information)
+```bash
+$ pushd kata-containers/tools/packaging/kernel/
+$ ./build-kernel.sh -a x86_64 -x snp setup
+$ ./build-kernel.sh -a x86_64 -x snp build
+$ sudo -E PATH="${PATH}" ./build-kernel.sh -x snp install
+$ popd
+```
+- Build a current OVMF capable of SEV-SNP:
+```bash
+$ pushd kata-containers/tools/packaging/static-build/ovmf
+$ ./build.sh
+$ tar -xvf edk2-x86_64.tar.gz
+$ popd
+```
+- Build a custom QEMU
+```bash
+$ source kata-containers/tools/packaging/scripts/lib.sh
+$ qemu_url="$(get_from_kata_deps "assets.hypervisor.qemu-snp-experimental.url")"
+$ qemu_tag="$(get_from_kata_deps "assets.hypervisor.qemu-snp-experimental.tag")"
+$ git clone "${qemu_url}"
+$ pushd qemu
+$ git checkout "${qemu_tag}"
+$ ./configure --enable-virtfs --target-list=x86_64-softmmu --enable-debug
+$ make -j "$(nproc)"
+$ popd
+```
+
+### Kata Containers Configuration for SNP
+
+The configuration file located at `/etc/kata-containers/configuration.toml` must be adapted as follows to support SNP-VMs:
+- Use the SNP-specific kernel for the guest VM (change path)
+```toml
+kernel = "/usr/share/kata-containers/vmlinuz-snp.container"
+```
+- Enable the use of an initrd (uncomment)
+```toml
+initrd = "/usr/share/kata-containers/kata-containers-initrd.img"
+```
+- Disable the use of a rootfs (comment out)
+```toml
+# image = "/usr/share/kata-containers/kata-containers.img"
+```
+- Use the custom QEMU capable of SNP (change path)
+```toml
+path = "/path/to/qemu/build/qemu-system-x86_64"
+```
+- Use `virtio-9p` device since `virtio-fs` is unsupported due to bugs / shortcomings in QEMU version [`snp-v3`](https://github.com/AMDESE/qemu/tree/snp-v3) for SEV and SEV-SNP (change value)
+```toml
+shared_fs = "virtio-9p"
+```
+- Disable `virtiofsd` since it is no longer required (comment out)
+```toml
+# virtio_fs_daemon = "/usr/libexec/virtiofsd"
+```
+- Disable NVDIMM (uncomment)
+```toml
+disable_image_nvdimm = true
+```
+- Disable shared memory (uncomment)
+```toml
+file_mem_backend = ""
+```
+- Enable confidential guests (uncomment)
+```toml
+confidential_guest = true
+```
+- Enable SNP-VMs (uncomment)
+```toml
+sev_snp_guest = true
+```
+  - Configure an OVMF (add path)
+```toml
+firmware = "/path/to/kata-containers/tools/packaging/static-build/ovmf/opt/kata/share/ovmf/OVMF.fd"
+```
+
+## Test Kata Containers with Containerd
+
+With Kata Containers configured to support SNP-VMs, we use containerd to test and deploy containers in these VMs.
+
+### Install Containerd
+If not already present, follow [this guide](./containerd-kata.md#install) to install containerd and its related components including `CNI` and the `cri-tools` (skip Kata Containers since we already installed it)
+
+### Containerd Configuration
+
+Follow [this guide](./containerd-kata.md#configuration) to configure containerd to use Kata Containers
+
+## Run Kata Containers in SNP-VMs
+
+Run the below commands to start a container. See [this guide](./containerd-kata.md#run) for more information
+```bash
+$ sudo ctr image pull docker.io/library/busybox:latest
+$ sudo ctr run --cni --runtime io.containerd.run.kata.v2 -t --rm docker.io/library/busybox:latest hello sh
+```
+
+### Check for active SNP:
+
+Inside the running container, run the following commands to check if SNP is active. It should look something like this:
+```
+/ # dmesg | grep -i sev
+[    0.299242] Memory Encryption Features active: AMD SEV SEV-ES SEV-SNP
+[    0.472286] SEV: Using SNP CPUID table, 31 entries present.
+[    0.514574] SEV: SNP guest platform device initialized.
+[    0.885425] sev-guest sev-guest: Initialized SEV guest driver (using vmpck_id 0)
+```
+
+### Obtain an SNP Attestation Report
+
+To obtain an attestation report inside the container, the `/dev/sev-guest` must first be configured. As of now, the VM does not perform this step, however it can be performed inside the container, either in the terminal or in code.
+
+Example for shell:
+```
+/ # SNP_MAJOR=$(cat /sys/devices/virtual/misc/sev-guest/dev | awk -F: '{print $1}')
+/ # SNP_MINOR=$(cat /sys/devices/virtual/misc/sev-guest/dev | awk -F: '{print $2}')
+/ # mknod -m 600 /dev/sev-guest c "${SNP_MAJOR}" "${SNP_MINOR}"
+```
+
+## Known Issues
+
+- Support for cgroups v2 is still [work in progress](https://github.com/kata-containers/kata-containers/issues/927). If issues occur due to cgroups v2 becoming the default in newer systems, one possible solution is to downgrade cgroups to v1:
+```bash
+sudo sed -i 's/^\(GRUB_CMDLINE_LINUX=".*\)"/\1 systemd.unified_cgroup_hierarchy=0"/' /etc/default/grub
+sudo update-grub
+sudo reboot
+```
+- If both SEV and SEV-SNP are supported by the host, Kata Containers uses SEV-SNP by default. You can verify what features are enabled by checking `/sys/module/kvm_amd/parameters/sev` and `sev_snp`. This means that Kata Containers can not run both SEV-SNP-VMs and SEV-VMs at the same time. If SEV is to be used by Kata Containers instead, reload the `kvm_amd` kernel module without SNP-support, this will disable SNP-support for the entire platform.
+```bash
+sudo rmmod kvm_amd && sudo modprobe kvm_amd sev_snp=0
+```
+

docs/how-to/how-to-run-rootless-vmm.md

@@ -1,5 +1,5 @@
 ## Introduction
-To improve security, Kata Container supports running the VMM process (currently only QEMU) as a non-`root` user.
+To improve security, Kata Container supports running the VMM process (QEMU and cloud-hypervisor) as a non-`root` user. 
 This document describes how to enable the rootless VMM mode and its limitations.
 
 ## Pre-requisites
@@ -27,7 +27,7 @@ Another necessary change is to move the hypervisor runtime files (e.g. `vhost-fs
 ## Limitations
 
 1. Only the VMM process is running as a non-root user. Other processes such as Kata Container shimv2 and `virtiofsd` still run as the root user.
-2. Currently, this feature is only supported in QEMU. Still need to bring it to Firecracker and Cloud Hypervisor (see https://github.com/kata-containers/kata-containers/issues/2567).
+2. Currently, this feature is only supported in QEMU and cloud-hypervisor. For firecracker, you can use jailer to run the VMM process with a non-root user.
 3. Certain features will not work when rootless VMM is enabled, including:
    1. Passing devices to the guest (`virtio-blk`, `virtio-scsi`) will not work if the non-privileged user does not have permission to access it (leading to a permission denied error). A more permissive permission (e.g. 666) may overcome this issue. However, you need to be aware of the potential security implications of reducing the security on such devices.
    2. `vfio` device will also not work because of permission denied error.

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -57,6 +57,7 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.enable_iothreads` | `boolean`| enable IO to be processed in a separate thread. Supported currently for virtio-`scsi` driver |
 | `io.katacontainers.config.hypervisor.enable_mem_prealloc` | `boolean` | the memory space used for `nvdimm` device by the hypervisor |
 | `io.katacontainers.config.hypervisor.enable_vhost_user_store` | `boolean` | enable vhost-user storage device (QEMU) |
+| `io.katacontainers.config.hypervisor.vhost_user_reconnect_timeout_sec` | `string`| the timeout for reconnecting vhost user socket (QEMU) 
 | `io.katacontainers.config.hypervisor.enable_virtio_mem` | `boolean` | enable virtio-mem (QEMU) |
 | `io.katacontainers.config.hypervisor.entropy_source` (R) | string| the path to a host source of entropy (`/dev/random`, `/dev/urandom` or real hardware RNG device) |
 | `io.katacontainers.config.hypervisor.file_mem_backend` (R) | string | file based memory backend root directory |
@@ -87,7 +88,7 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.use_vsock` | `boolean` | specify use of `vsock` for agent communication |
 | `io.katacontainers.config.hypervisor.vhost_user_store_path` (R) | `string` | specify the directory path where vhost-user devices related folders, sockets and device nodes should be (QEMU) |
 | `io.katacontainers.config.hypervisor.virtio_fs_cache_size` | uint32 | virtio-fs DAX cache size in `MiB` |
-| `io.katacontainers.config.hypervisor.virtio_fs_cache` | string | the cache mode for virtio-fs, valid values are `always`, `auto` and `none` |
+| `io.katacontainers.config.hypervisor.virtio_fs_cache` | string | the cache mode for virtio-fs, valid values are `always`, `auto` and `never` |
 | `io.katacontainers.config.hypervisor.virtio_fs_daemon` | string | virtio-fs `vhost-user` daemon path |
 | `io.katacontainers.config.hypervisor.virtio_fs_extra_args` | string | extra options passed to `virtiofs` daemon |
 | `io.katacontainers.config.hypervisor.enable_guest_swap` | `boolean` | enable swap in the guest |

docs/how-to/how-to-setup-swap-devices-in-guest-kernel.md

@@ -17,9 +17,9 @@ Enable setup swap device in guest kernel as follows:
 $ sudo sed -i -e 's/^#enable_guest_swap.*$/enable_guest_swap = true/g' /etc/kata-containers/configuration.toml
 ```
 
-## Run a Kata Container utilizing swap device
+## Run a Kata Containers utilizing swap device
 
-Use following command to start a Kata Container with swappiness 60 and 1GB swap device (swap_in_bytes - memory_limit_in_bytes).
+Use following command to start a Kata Containers with swappiness 60 and 1GB swap device (swap_in_bytes - memory_limit_in_bytes).
 ```
 $ pod_yaml=pod.yaml
 $ container_yaml=container.yaml
@@ -43,12 +43,12 @@ command:
 - top
 EOF
 $ sudo crictl pull $image
-$ podid=$(sudo crictl runp $pod_yaml)
+$ podid=$(sudo crictl runp --runtime kata $pod_yaml)
 $ cid=$(sudo crictl create $podid $container_yaml $pod_yaml)
 $ sudo crictl start $cid
 ```
 
-Kata Container setups swap device for this container only when `io.katacontainers.container.resource.swappiness` is set.
+Kata Containers setups swap device for this container only when `io.katacontainers.container.resource.swappiness` is set.
 
 The following table shows the swap size how to decide if `io.katacontainers.container.resource.swappiness` is set.
 |`io.katacontainers.container.resource.swap_in_bytes`|`memory_limit_in_bytes`|swap size|

docs/how-to/how-to-use-erofs-build-rootfs.md

@@ -0,0 +1,90 @@
+# Configure Kata Containers to use EROFS build rootfs
+
+## Introduction
+For kata containers, rootfs is used in the read-only way. EROFS can noticeably decrease metadata overhead.
+
+`mkfs.erofs` can generate compressed and uncompressed EROFS images.
+
+For uncompressed images, no files are compressed. However, it is optional to inline the data blocks at the end of the file with the metadata.
+
+For compressed images, each file will be compressed using the lz4 or lz4hc algorithm, and it will be confirmed whether it can save space. Use No compression of the file if compression does not save space.
+
+## Performance comparison
+|                 | EROFS | EXT4 | XFS |
+|-----------------|-------| --- | --- |
+| Image Size [MB] | 106(uncompressed) | 256 | 126 |
+
+
+## Guidance
+### Install the `erofs-utils`
+#### `apt/dnf` install
+On newer `Ubuntu/Debian` systems, it can be installed directly using the `apt` command, and on `Fedora` it can be installed directly using the `dnf` command.
+
+```shell
+# Debian/Ubuntu
+$ apt install erofs-utils
+# Fedora
+$ dnf install erofs-utils
+```
+
+#### Source install
+[https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git](https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git)
+
+##### Compile dependencies
+If you need to enable the `Lz4` compression feature, `Lz4 1.8.0+` is required, and `Lz4 1.9.3+` is strongly recommended.
+
+##### Compilation process
+For some old lz4 versions (lz4-1.8.0~1.8.3), if lz4-static is not installed, the lz4hc algorithm will not be supported. lz4-static can be installed with apt install lz4-static.x86_64. However, these versions have some bugs in compression, and it is not recommended to use these versions directly.
+If you use `lz4 1.9.0+`, you can directly use the following command to compile.
+
+```shell
+$ ./autogen.sh
+$ ./configure
+$ make
+```
+
+The compiled `mkfs.erofs` program will be saved in the `mkfs` directory. Afterwards, the generated tools can be installed to a system directory using make install (requires root privileges).
+
+### Create a local rootfs
+```shell
+$ export distro="ubuntu"
+$ export FS_TYPE="erofs"
+$ export ROOTFS_DIR="realpath kata-containers/tools/osbuilder/rootfs-builder/rootfs"
+$ sudo rm -rf "${ROOTFS_DIR}"
+$ pushd kata-containers/tools/osbuilder/rootfs-builder
+$ script -fec 'sudo -E SECCOMP=no ./rootfs.sh "${distro}"'
+$ popd
+```
+
+### Add a custom agent to the image - OPTIONAL
+> Note:
+> - You should only do this step if you are testing with the latest version of the agent.
+```shell
+$ sudo install -o root -g root -m 0550 -t "${ROOTFS_DIR}/usr/bin" "${ROOTFS_DIR}/../../../../src/agent/target/x86_64-unknown-linux-musl/release/kata-agent"
+$ sudo install -o root -g root -m 0440 "${ROOTFS_DIR}/../../../../src/agent/kata-agent.service" "${ROOTFS_DIR}/usr/lib/systemd/system/"
+$ sudo install -o root -g root -m 0440 "${ROOTFS_DIR}/../../../../src/agent/kata-containers.target" "${ROOTFS_DIR}/usr/lib/systemd/system/"
+```
+
+### Build a root image
+```shell
+$ pushd  kata-containers/tools/osbuilder/image-builder
+$ script -fec 'sudo -E ./image_builder.sh "${ROOTFS_DIR}"'
+$ popd
+```
+
+### Install the rootfs image
+```shell
+$ pushd kata-containers/tools/osbuilder/image-builder
+$ commit="$(git log --format=%h -1 HEAD)"
+$ date="$(date +%Y-%m-%d-%T.%N%z)"
+$ rootfs="erofs"
+$ image="kata-containers-${rootfs}-${date}-${commit}"
+$ sudo install -o root -g root -m 0640 -D kata-containers.img "/usr/share/kata-containers/${image}"
+$ (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
+$ popd
+```
+
+### Use `EROFS` in the runtime
+```shell
+$ sudo sed -i -e 's/^# *\(rootfs_type\).*=.*$/\1 = erofs/g' /etc/kata-containers/configuration.toml
+```

docs/how-to/how-to-use-kata-containers-with-firecracker.md

@@ -104,7 +104,7 @@ sudo dmsetup create "${POOL_NAME}" \
 
 cat << EOF
 #
-# Add this to your config.toml configuration file and restart `containerd` daemon
+# Add this to your config.toml configuration file and restart containerd daemon
 #
 [plugins]
   [plugins.devmapper]
@@ -212,7 +212,7 @@ Next, we need to configure containerd. Add a file in your path (e.g. `/usr/local
 
 ```
 #!/bin/bash
-KATA_CONF_FILE=/etc/containers/configuration-fc.toml /usr/local/bin/containerd-shim-kata-v2 $@
+KATA_CONF_FILE=/etc/kata-containers/configuration-fc.toml /usr/local/bin/containerd-shim-kata-v2 $@
 ```
 > **Note:** You may need to edit the paths of the configuration file and the `containerd-shim-kata-v2` to correspond to your setup.
 

docs/install/README.md

@@ -19,7 +19,7 @@ Packaged installation methods uses your distribution's native package format (su
 |------------------------------------------------------|----------------------------------------------------------------------------------------------|-------------------|-----------------------------------------------------------------------------------------------|
 | [Using kata-deploy](#kata-deploy-installation)       | The preferred way to deploy the Kata Containers distributed binaries on a Kubernetes cluster | **No!**           | Best way to give it a try on kata-containers on an already up and running Kubernetes cluster. | 
 | [Using official distro packages](#official-packages) | Kata packages provided by Linux distributions official repositories                          | yes               | Recommended for most users.                                                                   |
-| [Using snap](#snap-installation)                     | Easy to install                                                                              | yes               | Good alternative to official distro packages.                                                 |
+| ~~[Using snap](#snap-installation)~~                     | ~~Easy to install~~                                                                              | ~~yes~~               | **Snap is unmaintained!** ~~Good alternative to official distro packages.~~                                                 |
 | [Automatic](#automatic-installation)                 | Run a single command to install a full system                                                | **No!**           | For those wanting the latest release quickly.                                                 |
 | [Manual](#manual-installation)                       | Follow a guide step-by-step to install a working system                                      | **No!**           | For those who want the latest release with more control.                                      |
 | [Build from source](#build-from-source-installation) | Build the software components manually                                                       | **No!**           | Power users and developers only.                                                              |
@@ -44,9 +44,24 @@ Kata packages are provided by official distribution repositories for:
 
 ### Snap Installation
 
-The snap installation is available for all distributions which support `snapd`.
+> **WARNING:**
+>
+> The Snap package method is **unmaintained** and only provides an old
+> version of Kata Containers:
+> The [latest Kata Containers snap](https://snapcraft.io/kata-containers)
+> provides Kata Containers
+> [version 2.4.2](https://github.com/kata-containers/kata-containers/releases/tag/2.4.2)
+> but the latest stable Kata Containers release at the time of writing is
+> [version 3.1.0](https://github.com/kata-containers/kata-containers/releases/tag/3.1.0).
+>
+> We recommend strongly that you switch to an alternative Kata Containers installation method.
+>
+> See: https://github.com/kata-containers/kata-containers/issues/6769
+> for further details.
 
-[Use snap](snap-installation-guide.md) to install Kata Containers from https://snapcraft.io.
+~~The snap installation is available for all distributions which support `snapd`.~~
+
+~~[Use snap](snap-installation-guide.md) to install Kata Containers from https://snapcraft.io. ~~
 
 ### Automatic Installation
 

docs/install/aws-installation-guide.md

@@ -123,7 +123,7 @@ Refer to [this guide](https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-l
 SSH into the machine
 
 ```bash
-$ ssh -i MyKeyPair.pen ubuntu@${IP}
+$ ssh -i MyKeyPair.pem ubuntu@${IP}
 ```
 
 Go onto the next step.

docs/install/kata-containers-3.0-rust-runtime-installation-guide.md

@@ -24,7 +24,7 @@ architectures:
 
 | Installation method                                  | Description                                                                                  | Automatic updates | Use case                                                                                      | Availability
 |------------------------------------------------------|----------------------------------------------------------------------------------------------|-------------------|-----------------------------------------------------------------------------------------------|----------- |
-| [Using kata-deploy](#kata-deploy-installation)       | The preferred way to deploy the Kata Containers distributed binaries on a Kubernetes cluster | **No!**           | Best way to give it a try on kata-containers on an already up and running Kubernetes cluster. | No |
+| [Using kata-deploy](#kata-deploy-installation)       | The preferred way to deploy the Kata Containers distributed binaries on a Kubernetes cluster | **No!**           | Best way to give it a try on kata-containers on an already up and running Kubernetes cluster. | Yes |
 | [Using official distro packages](#official-packages) | Kata packages provided by Linux distributions official repositories                          | yes               | Recommended for most users. | No |                                                                   
 | [Using snap](#snap-installation)                     | Easy to install                                                                              | yes               | Good alternative to official distro packages.                                                 | No |
 | [Automatic](#automatic-installation)                 | Run a single command to install a full system                                                | **No!**           | For those wanting the latest release quickly.                                                 | No |
@@ -32,7 +32,8 @@ architectures:
 | [Build from source](#build-from-source-installation) | Build the software components manually                                                       | **No!**           | Power users and developers only.  | Yes |              
 
 ### Kata Deploy Installation
-`ToDo`
+
+Follow the [`kata-deploy`](../../tools/packaging/kata-deploy/README.md).
 ### Official packages
 `ToDo`
 ### Snap Installation
@@ -48,14 +49,14 @@ architectures:
 
 * Download `Rustup` and install  `Rust`
     > **Notes:**
-    > Rust version 1.58 is needed
+    > For Rust version, please set `RUST_VERSION` to the value of `languages.rust.meta.newest-version key` in [`versions.yaml`](../../versions.yaml) or, if `yq` is available on your system, run `export RUST_VERSION=$(yq read versions.yaml languages.rust.meta.newest-version)`.
 
     Example for `x86_64`
     ```
     $ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
     $ source $HOME/.cargo/env
-    $ rustup install 1.58
-    $ rustup default 1.58-x86_64-unknown-linux-gnu
+    $ rustup install ${RUST_VERSION}
+    $ rustup default ${RUST_VERSION}-x86_64-unknown-linux-gnu
     ```
 
 * Musl support for fully static binary
@@ -83,7 +84,7 @@ $ git clone https://github.com/kata-containers/kata-containers.git
 $ cd kata-containers/src/runtime-rs
 $ make && sudo make install
 ```
-After running the command above, the default config file `configuration.toml` will be installed under `/usr/share/defaults/kata-containers/`,  the binary file `containerd-shim-kata-v2` will be installed under `/user/local/bin` .
+After running the command above, the default config file `configuration.toml` will be installed under `/usr/share/defaults/kata-containers/`,  the binary file `containerd-shim-kata-v2` will be installed under `/usr/local/bin/` .
 
 ### Build Kata Containers Kernel
 Follow the [Kernel installation guide](/tools/packaging/kernel/README.md).

docs/install/minikube-installation-guide.md

@@ -71,12 +71,6 @@ To use containerd, modify the `--container-runtime` argument:
 > **Notes:**
 > - Adjust the `--memory 6144` line to suit your environment and requirements. Kata Containers default to
 > requesting 2048MB per container. We recommended you supply more than that to the Minikube node.
-> - Prior to Minikube/Kubernetes v1.14, the beta `RuntimeClass` feature also needed enabling with
-> the following.
->
-> | what | why |
-> | ---- | --- |
-> | `--feature-gates=RuntimeClass=true` | Kata needs to use the `RuntimeClass` Kubernetes feature |
 
 The full command is therefore:
 
@@ -138,17 +132,9 @@ $ kubectl -n kube-system exec ${podname} -- ps -ef | fgrep infinity
 
 ## Enabling Kata Containers
 
-> **Note:** Only Minikube/Kubernetes versions <= 1.13 require this step. Since version
-> v1.14, the `RuntimeClass` is enabled by default. Performing this step on Kubernetes > v1.14 is
-> however benign.
-
 Now you have installed the Kata Containers components in the Minikube node. Next, you need to configure
 Kubernetes `RuntimeClass` to know when to use Kata Containers to run a pod.
 
-```sh
-$ kubectl apply -f https://raw.githubusercontent.com/kubernetes/node-api/master/manifests/runtimeclass_crd.yaml > runtimeclass_crd.yaml
-```
-
 ### Register the runtime
 
 Now register the `kata qemu` runtime with that class. This should result in no errors:

docs/install/snap-installation-guide.md

@@ -1,5 +1,20 @@
 # Kata Containers snap package
 
+> **WARNING:**
+>
+> The Snap package method is **unmaintained** and only provides an old
+> version of Kata Containers:
+> The [latest Kata Containers snap](https://snapcraft.io/kata-containers)
+> provides Kata Containers
+> [version 2.4.2](https://github.com/kata-containers/kata-containers/releases/tag/2.4.2)
+> but the latest stable Kata Containers release at the time of writing is
+> [version 3.1.0](https://github.com/kata-containers/kata-containers/releases/tag/3.1.0).
+>
+> We recommend strongly that you switch to an alternative Kata Containers installation method.
+>
+> See: https://github.com/kata-containers/kata-containers/issues/6769
+> for further details.
+
 ## Install Kata Containers
 
 Kata Containers can be installed in any Linux distribution that supports
@@ -7,6 +22,21 @@ Kata Containers can be installed in any Linux distribution that supports
 
 Run the following command to install **Kata Containers**:
 
+> **WARNING:**
+>
+> The Snap package method is **unmaintained** and only provides an old
+> version of Kata Containers:
+> The [latest Kata Containers snap](https://snapcraft.io/kata-containers)
+> provides Kata Containers
+> [version 2.4.2](https://github.com/kata-containers/kata-containers/releases/tag/2.4.2)
+> but the latest stable Kata Containers release at the time of writing is
+> [version 3.1.0](https://github.com/kata-containers/kata-containers/releases/tag/3.1.0).
+>
+> We recommend strongly that you switch to an alternative Kata Containers installation method.
+>
+> See: https://github.com/kata-containers/kata-containers/issues/6769
+> for further details.
+
 ```sh
 $ sudo snap install kata-containers --stable --classic
 ```

docs/use-cases/NVIDIA-GPU-passthrough-and-Kata.md

@@ -545,6 +545,12 @@ Create the hook execution file for Kata:
 /usr/bin/nvidia-container-toolkit -debug $@
 ```
 
+Make sure the hook shell is executable:
+
+```sh
+chmod +x $ROOTFS_DIR/usr/share/oci/hooks/prestart/nvidia-container-toolkit.sh
+```
+
 As the last step one can do some cleanup of files or package caches. Build the
 rootfs and configure it for use with Kata according to the development guide.
 

docs/use-cases/using-Intel-QAT-and-kata.md

@@ -49,7 +49,7 @@ the latest driver.
 $ export QAT_DRIVER_VER=qat1.7.l.4.14.0-00031.tar.gz
 $ export QAT_DRIVER_URL=https://downloadmirror.intel.com/30178/eng/${QAT_DRIVER_VER}
 $ export QAT_CONF_LOCATION=~/QAT_conf
-$ export QAT_DOCKERFILE=https://raw.githubusercontent.com/intel/intel-device-plugins-for-kubernetes/master/demo/openssl-qat-engine/Dockerfile
+$ export QAT_DOCKERFILE=https://raw.githubusercontent.com/intel/intel-device-plugins-for-kubernetes/main/demo/openssl-qat-engine/Dockerfile
 $ export QAT_SRC=~/src/QAT
 $ export GOPATH=~/src/go
 $ export KATA_KERNEL_LOCATION=~/kata

docs/use-cases/using-Intel-SGX-and-kata.md

@@ -61,6 +61,9 @@ spec:
           name: eosgx-demo-job-1
           image: oeciteam/oe-helloworld:latest
           imagePullPolicy: IfNotPresent
+          volumeMounts:
+          - mountPath: /dev
+            name: dev-mount
           securityContext:
             readOnlyRootFilesystem: true
             capabilities:

docs/use-cases/using-SPDK-vhostuser-and-kata.md

@@ -197,11 +197,6 @@ vhost_user_store_path = "<Path of the base directory for vhost-user device>"
 > under `[hypervisor.qemu]` section.
 
 
-For the subdirectories of `vhost_user_store_path`: `block` is used for block
-device; `block/sockets` is where we expect UNIX domain sockets for vhost-user
-block devices to live; `block/devices` is where simulated block device nodes
-for vhost-user block devices are created.
-
 For the subdirectories of `vhost_user_store_path`:
 -  `block` is used for block device;
 -  `block/sockets` is where we expect UNIX domain sockets for vhost-user

snap/snapcraft.yaml

@@ -34,78 +34,28 @@ parts:
       mkdir -p $(dirname ${kata_dir})
       ln -sf $(realpath "${SNAPCRAFT_STAGE}/..") ${kata_dir}
 
-  godeps:
+  docker:
     after: [metadata]
     plugin: nil
     prime:
       - -*
     build-packages:
+      - ca-certificates
+      - containerd
       - curl
+      - gnupg
+      - lsb-release
+      - runc
     override-build: |
       source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
 
-      # put everything in stage
-      cd "${SNAPCRAFT_STAGE}"
-
-      version="$(${yq} r ${kata_dir}/versions.yaml languages.golang.meta.newest-version)"
-      tarfile="go${version}.${goos}-${goarch}.tar.gz"
-      curl -LO https://golang.org/dl/${tarfile}
-      tar -xf ${tarfile} --strip-components=1
-
-  rustdeps:
-    after: [metadata]
-    plugin: nil
-    prime:
-      - -*
-    build-packages:
-      - curl
-    override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
-      # put everything in stage
-      cd "${SNAPCRAFT_STAGE}"
-
-      version="$(${yq} r ${kata_dir}/versions.yaml languages.rust.meta.newest-version)"
-      if ! command -v rustup > /dev/null; then
-        curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain ${version}
-      fi
-
-      export PATH=${PATH}:${HOME}/.cargo/bin
-      rustup toolchain install ${version}
-      rustup default ${version}
-      if [ "${arch}" == "ppc64le" ] || [ "${arch}" == "s390x" ] ; then
-        [ "${arch}" == "ppc64le" ] && arch="powerpc64le"
-        rustup target add ${arch}-unknown-linux-gnu
-      else
-        rustup target add ${arch}-unknown-linux-musl
-        $([ "$(whoami)" != "root" ] && echo sudo) ln -sf /usr/bin/g++ /bin/musl-g++
-      fi
-      rustup component add rustfmt
-
-  image:
-    after: [godeps, qemu, kernel]
-    plugin: nil
-    build-packages:
-      - docker.io
-      - cpio
-      - git
-      - iptables
-      - software-properties-common
-      - uidmap
-      - gnupg2
-    override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
-      [ "${arch}" = "ppc64le" ] || [ "${arch}" = "s390x" ] && sudo apt-get --no-install-recommends install -y protobuf-compiler
-
-      if [ -n "$http_proxy" ]; then
-        echo "Setting proxy $http_proxy"
-        sudo -E systemctl set-environment http_proxy="$http_proxy" || true
-        sudo -E systemctl set-environment https_proxy="$https_proxy" || true
-      fi
-
-      # Copy yq binary. It's used in the container
-      cp -a "${yq}" "${GOPATH}/bin/"
+      curl -fsSL https://download.docker.com/linux/ubuntu/gpg |\
+          sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+      distro_codename=$(lsb_release -cs)
+      echo "deb [arch=${dpkg_arch} signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu ${distro_codename} stable" |\
+          sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+      sudo apt-get -y update
+      sudo apt-get -y install docker-ce docker-ce-cli containerd.io
 
       echo "Unmasking docker service"
       sudo -E systemctl unmask docker.service || true
@@ -113,253 +63,102 @@ parts:
       echo "Adding $USER into docker group"
       sudo -E gpasswd -a $USER docker
       echo "Starting docker"
+      # docker may fail to start using "fd://" in docker.service
+      sudo sed -i 's/fd:\/\//unix:\/\//g' /lib/systemd/system/docker.service
+      sudo systemctl daemon-reload
       sudo -E systemctl start docker || true
 
-      cd "${kata_dir}/tools/osbuilder"
+  image:
+    after: [docker]
+    plugin: nil
+    override-build: |
+      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
 
-      # build image
-      export AGENT_INIT=yes
-      export USE_DOCKER=1
-      export DEBUG=1
-      initrd_distro=$(${yq} r -X ${kata_dir}/versions.yaml assets.initrd.architecture.${arch}.name)
-      image_distro=$(${yq} r -X ${kata_dir}/versions.yaml assets.image.architecture.${arch}.name)
-      case "$arch" in
-        x86_64)
-          # In some build systems it's impossible to build a rootfs image, try with the initrd image
-          sudo -E PATH=$PATH make image DISTRO="${image_distro}" || sudo -E PATH="$PATH" make initrd DISTRO="${initrd_distro}"
-        ;;
+      cd "${SNAPCRAFT_PROJECT_DIR}"
+      sudo -E NO_TTY=true make rootfs-image-tarball
 
-        aarch64|ppc64le|s390x)
-          sudo -E PATH="$PATH" make initrd DISTRO="${initrd_distro}"
-        ;;
+      tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-rootfs-image.tar.xz"
 
-        *) die "unsupported architecture: ${arch}" ;;
-      esac
+      tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}"
+
+
+      sudo -E NO_TTY=true make rootfs-initrd-tarball
+
+      tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-rootfs-initrd.tar.xz"
+
+      tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}"
 
-      # Install image
-      kata_image_dir="${SNAPCRAFT_PART_INSTALL}/usr/share/kata-containers"
-      mkdir -p "${kata_image_dir}"
-      cp kata-containers*.img "${kata_image_dir}"
 
   runtime:
-    after: [godeps, image, cloud-hypervisor]
+    after: [docker]
     plugin: nil
-    build-attributes: [no-patchelf]
     override-build: |
       source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
 
-      cd "${kata_dir}/src/runtime"
+      cd "${SNAPCRAFT_PROJECT_DIR}"
+      sudo -E NO_TTY=true make shim-v2-tarball
 
-      qemu_cmd="qemu-system-${qemu_arch}"
+      tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-shim-v2.tar.xz"
 
-      # build and install runtime
-      make \
-        PREFIX="/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr" \
-        SKIP_GO_VERSION_CHECK=1 \
-        QEMUCMD="${qemu_cmd}"
+      tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}"
 
-      make install \
-        PREFIX=/usr \
-        DESTDIR="${SNAPCRAFT_PART_INSTALL}" \
-        SKIP_GO_VERSION_CHECK=1 \
-        QEMUCMD="${qemu_cmd}"
-
-      if [ ! -f ${SNAPCRAFT_PART_INSTALL}/../../image/install/usr/share/kata-containers/kata-containers.img ]; then
-        sed -i -e "s|^image =.*|initrd = \"/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr/share/kata-containers/kata-containers-initrd.img\"|" \
-          ${SNAPCRAFT_PART_INSTALL}/usr/share/defaults/${SNAPCRAFT_PROJECT_NAME}/configuration.toml
-      fi
+      mkdir -p "${SNAPCRAFT_PART_INSTALL}/usr/bin"
+      ln -sf "${SNAPCRAFT_PART_INSTALL}/opt/kata/bin/containerd-shim-kata-v2" "${SNAPCRAFT_PART_INSTALL}/usr/bin/containerd-shim-kata-v2"
+      ln -sf "${SNAPCRAFT_PART_INSTALL}/opt/kata/bin/kata-runtime" "${SNAPCRAFT_PART_INSTALL}/usr/bin/kata-runtime"
+      ln -sf "${SNAPCRAFT_PART_INSTALL}/opt/kata/bin/kata-collect-data.sh" "${SNAPCRAFT_PART_INSTALL}/usr/bin/kata-collect-data.sh"
 
   kernel:
-    after: [godeps]
+    after: [docker]
     plugin: nil
-    build-packages:
-      - libelf-dev
-      - curl
-      - build-essential
-      - bison
-      - flex
     override-build: |
       source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
 
-      kernel_version="$(${yq} r $versions_file assets.kernel.version)"
-      #Remove extra 'v'
-      kernel_version="${kernel_version#v}"
+      cd "${SNAPCRAFT_PROJECT_DIR}"
+      sudo -E NO_TTY=true make kernel-tarball
 
-      [ "${arch}" = "s390x" ] && sudo apt-get --no-install-recommends install -y libssl-dev
+      tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-kernel.tar.xz"
 
-      cd "${kata_dir}/tools/packaging/kernel"
-      kernel_dir_prefix="kata-linux-"
-
-      # Setup and build kernel
-      ./build-kernel.sh -v "${kernel_version}" -d setup
-      cd ${kernel_dir_prefix}*
-      make -j $(nproc ${CI:+--ignore 1}) EXTRAVERSION=".container"
-
-      kernel_suffix="${kernel_version}.container"
-      kata_kernel_dir="${SNAPCRAFT_PART_INSTALL}/usr/share/kata-containers"
-      mkdir -p "${kata_kernel_dir}"
-
-      # Install bz kernel
-      make install INSTALL_PATH="${kata_kernel_dir}" EXTRAVERSION=".container" || true
-      vmlinuz_name="vmlinuz-${kernel_suffix}"
-      ln -sf "${vmlinuz_name}" "${kata_kernel_dir}/vmlinuz.container"
-
-      # Install raw kernel
-      vmlinux_path="vmlinux"
-      [ "${arch}" = "s390x" ] && vmlinux_path="arch/s390/boot/vmlinux"
-      vmlinux_name="vmlinux-${kernel_suffix}"
-      cp "${vmlinux_path}" "${kata_kernel_dir}/${vmlinux_name}"
-      ln -sf "${vmlinux_name}" "${kata_kernel_dir}/vmlinux.container"
+      tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}"
 
   qemu:
     plugin: make
-    after: [godeps]
-    build-packages:
-      - gcc
-      - python3
-      - zlib1g-dev
-      - libcap-ng-dev
-      - libglib2.0-dev
-      - libpixman-1-dev
-      - libnuma-dev
-      - libltdl-dev
-      - libcap-dev
-      - libattr1-dev
-      - libfdt-dev
-      - curl
-      - libcapstone-dev
-      - bc
-      - libblkid-dev
-      - libffi-dev
-      - libmount-dev
-      - libseccomp-dev
-      - libselinux1-dev
-      - ninja-build
+    after: [docker]
     override-build: |
       source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
 
-      branch="$(${yq} r ${versions_file} assets.hypervisor.qemu.version)"
-      url="$(${yq} r ${versions_file} assets.hypervisor.qemu.url)"
-      commit=""
-      patches_dir="${kata_dir}/tools/packaging/qemu/patches/$(echo ${branch} | sed -e 's/.[[:digit:]]*$//' -e 's/^v//').x"
-      patches_version_dir="${kata_dir}/tools/packaging/qemu/patches/tag_patches/${branch}"
+      cd "${SNAPCRAFT_PROJECT_DIR}"
+      sudo -E NO_TTY=true make qemu-tarball
 
-      # download source
-      qemu_dir="${SNAPCRAFT_STAGE}/qemu"
-      rm -rf "${qemu_dir}"
-      git clone --depth 1 --branch ${branch} --single-branch ${url} "${qemu_dir}"
-      cd "${qemu_dir}"
-      [ -z "${commit}" ] || git checkout "${commit}"
+      tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-qemu.tar.xz"
 
-      [ -n "$(ls -A ui/keycodemapdb)" ] || git clone --depth 1 https://github.com/qemu/keycodemapdb ui/keycodemapdb/
-      [ -n "$(ls -A capstone)" ] || git clone --depth 1 https://github.com/qemu/capstone capstone
-
-      # Apply branch patches
-      [ -d "${patches_version_dir}" ] || mkdir "${patches_version_dir}"
-      ${kata_dir}/tools/packaging/scripts/apply_patches.sh "${patches_dir}"
-      ${kata_dir}/tools/packaging/scripts/apply_patches.sh "${patches_version_dir}"
-
-      # Only x86_64 supports libpmem
-      [ "${arch}" = "x86_64" ] && sudo apt-get --no-install-recommends install -y apt-utils ca-certificates libpmem-dev
-
-      configure_hypervisor="${kata_dir}/tools/packaging/scripts/configure-hypervisor.sh"
-      chmod +x "${configure_hypervisor}"
-      # static build. The --prefix, --libdir, --libexecdir, --datadir arguments are
-      # based on PREFIX and set by configure-hypervisor.sh
-      echo "$(PREFIX=/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr ${configure_hypervisor} -s kata-qemu) \
-        --disable-rbd " \
-        | xargs ./configure
-
-      # Copy QEMU configurations (Kconfigs)
-      case "${branch}" in
-      "v5.1.0")
-        cp -a "${kata_dir}"/tools/packaging/qemu/default-configs/* default-configs
-        ;;
-
-      *)
-        cp -a "${kata_dir}"/tools/packaging/qemu/default-configs/* configs/devices/
-        ;;
-      esac
-
-      # build and install
-      make -j $(nproc ${CI:+--ignore 1})
-      make install DESTDIR="${SNAPCRAFT_PART_INSTALL}"
-    prime:
-      - -snap/
-      - -usr/bin/qemu-ga
-      - -usr/bin/qemu-pr-helper
-      - -usr/bin/virtfs-proxy-helper
-      - -usr/include/
-      - -usr/share/applications/
-      - -usr/share/icons/
-      - -usr/var/
-      - usr/*
-      - lib/*
-    organize:
-      # Hack: move qemu to /
-      "snap/kata-containers/current/": "./"
+      tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}"
 
   virtiofsd:
     plugin: nil
-    after: [godeps, rustdeps]
+    after: [docker]
     override-build: |
       source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
 
-      # Currently, powerpc makes use of the QEMU's C implementation.
-      # The other platforms make use of the new rust virtiofsd.
-      #
-      # See "tools/packaging/scripts/configure-hypervisor.sh".
-      if [ "${arch}" == "ppc64le" ]
-      then
-          echo "INFO: Building QEMU's C version of virtiofsd"
-          # Handled by the 'qemu' part, so nothing more to do here.
-          exit 0
-      else
-          echo "INFO: Building rust version of virtiofsd"
-      fi
+      cd "${SNAPCRAFT_PROJECT_DIR}"
+      sudo -E NO_TTY=true make virtiofsd-tarball
 
-      cd "${kata_dir}"
+      tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-virtiofsd.tar.xz"
 
-      export PATH=${PATH}:${HOME}/.cargo/bin
-      # Download the rust implementation of virtiofsd
-      tools/packaging/static-build/virtiofsd/build-static-virtiofsd.sh
-      sudo install \
-        --owner='root' \
-        --group='root' \
-        --mode=0755 \
-        -D \
-        --target-directory="${SNAPCRAFT_PART_INSTALL}/usr/libexec/" \
-        virtiofsd/virtiofsd
+      tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}"
 
   cloud-hypervisor:
     plugin: nil
-    after: [godeps]
+    after: [docker]
     override-build: |
       source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
 
       if [ "${arch}" == "aarch64" ] || [ "${arch}" == "x86_64" ]; then
-          sudo apt-get -y update
-          sudo apt-get -y install ca-certificates curl gnupg lsb-release
-          curl -fsSL https://download.docker.com/linux/ubuntu/gpg |\
-              sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
-          distro_codename=$(lsb_release -cs)
-          echo "deb [arch=${dpkg_arch} signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu ${distro_codename} stable" |\
-              sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
-          sudo apt-get -y update
-          sudo apt-get -y install docker-ce docker-ce-cli containerd.io
-          sudo systemctl start docker.socket
-
           cd "${SNAPCRAFT_PROJECT_DIR}"
           sudo -E NO_TTY=true make cloud-hypervisor-tarball
 
           tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-cloud-hypervisor.tar.xz"
-          tmpdir=$(mktemp -d)
 
-          tar -xvJpf "${tarfile}" -C "${tmpdir}"
-
-          install -D "${tmpdir}/opt/kata/bin/cloud-hypervisor" "${SNAPCRAFT_PART_INSTALL}/usr/bin/cloud-hypervisor"
-
-          rm -rf "${tmpdir}"
+          tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}"
       fi
 
 apps:

src/agent/Cargo.toml

@@ -10,8 +10,8 @@ oci = { path = "../libs/oci" }
 rustjail = { path = "rustjail" }
 protocols = { path = "../libs/protocols", features = ["async"] }
 lazy_static = "1.3.0"
-ttrpc = { version = "0.6.0", features = ["async"], default-features = false }
-protobuf = "2.27.0"
+ttrpc = { version = "0.7.1", features = ["async"], default-features = false }
+protobuf = "3.2.0"
 libc = "0.2.58"
 nix = "0.24.2"
 capctl = "0.2.0"
@@ -23,7 +23,6 @@ regex = "1.5.6"
 serial_test = "0.5.1"
 kata-sys-util = { path = "../libs/kata-sys-util" }
 kata-types = { path = "../libs/kata-types" }
-sysinfo = "0.23.0"
 
 # Async helpers
 async-trait = "0.1.42"
@@ -31,7 +30,7 @@ async-recursion = "0.3.2"
 futures = "0.3.17"
 
 # Async runtime
-tokio = { version = "1.14.0", features = ["full"] }
+tokio = { version = "1.28.1", features = ["full"] }
 tokio-vsock = "0.3.1"
 
 netlink-sys = { version = "0.7.0", features = ["tokio_socket",]}
@@ -49,10 +48,11 @@ slog-scope = "4.1.2"
 slog-stdlog = "4.0.0"
 log = "0.4.11"
 
+cfg-if = "1.0.0"
 prometheus = { version = "0.13.0", features = ["process"] }
 procfs = "0.12.0"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.2.10" }
+cgroups = { package = "cgroups-rs", version = "0.3.2" }
 
 # Tracing
 tracing = "0.1.26"
@@ -69,6 +69,7 @@ clap = { version = "3.0.1", features = ["derive"] }
 [dev-dependencies]
 tempfile = "3.1.0"
 test-utils = { path = "../libs/test-utils" }
+which = "4.3.0"
 
 [workspace]
 members = [

src/agent/Makefile

@@ -33,6 +33,12 @@ ifeq ($(SECCOMP),yes)
     override EXTRA_RUSTFEATURES += seccomp
 endif
 
+include ../../utils.mk
+
+ifeq ($(ARCH), ppc64le)
+    override ARCH = powerpc64le
+endif
+
 ##VAR STANDARD_OCI_RUNTIME=yes|no define if agent enables standard oci runtime feature
 STANDARD_OCI_RUNTIME := no
 
@@ -45,8 +51,6 @@ ifneq ($(EXTRA_RUSTFEATURES),)
     override EXTRA_RUSTFEATURES := --features "$(EXTRA_RUSTFEATURES)"
 endif
 
-include ../../utils.mk
-
 TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET)
 
 ##VAR DESTDIR=<path> is a directory prepended to each installed target file
@@ -107,6 +111,8 @@ endef
 ##TARGET default: build code
 default: $(TARGET) show-header
 
+static-checks-build: $(GENERATED_CODE)
+
 $(TARGET): $(GENERATED_CODE) $(TARGET_PATH)
 
 $(TARGET_PATH): show-summary

src/agent/rustjail/Cargo.toml

@@ -11,28 +11,32 @@ serde_json = "1.0.39"
 serde_derive = "1.0.91"
 oci = { path = "../../libs/oci" }
 protocols = { path ="../../libs/protocols" }
+kata-sys-util = { path = "../../libs/kata-sys-util" }
 caps = "0.5.0"
-nix = "0.23.0"
+nix = "0.24.2"
 scopeguard = "1.0.0"
 capctl = "0.2.0"
 lazy_static = "1.3.0"
 libc = "0.2.58"
-protobuf = "2.27.0"
+protobuf = "3.2.0"
 slog = "2.5.2"
 slog-scope = "4.1.2"
 scan_fmt = "0.2.6"
 regex = "1.5.6"
 path-absolutize = "1.2.0"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.2.10" }
+cgroups = { package = "cgroups-rs", version = "0.3.2" }
 rlimit = "0.5.3"
 cfg-if = "0.1.0"
 
-tokio = { version = "1.2.0", features = ["sync", "io-util", "process", "time", "macros", "rt"] }
+tokio = { version = "1.28.1", features = ["sync", "io-util", "process", "time", "macros", "rt"] }
 futures = "0.3.17"
 async-trait = "0.1.31"
 inotify = "0.9.2"
-libseccomp = { version = "0.2.3", optional = true }
+libseccomp = { version = "0.3.0", optional = true }
+zbus = "2.3.0"
+bit-vec= "0.6.3"
+xattr = "0.2.3"
 
 [dev-dependencies]
 serial_test = "0.5.0"

src/agent/rustjail/src/cgroups/fs/mod.rs

@@ -27,11 +27,12 @@ use oci::{
     LinuxNetwork, LinuxPids, LinuxResources,
 };
 
-use protobuf::{CachedSize, RepeatedField, SingularPtrField, UnknownFields};
+use protobuf::MessageField;
 use protocols::agent::{
     BlkioStats, BlkioStatsEntry, CgroupStats, CpuStats, CpuUsage, HugetlbStats, MemoryData,
     MemoryStats, PidsStats, ThrottlingData,
 };
+use std::any::Any;
 use std::collections::HashMap;
 use std::fs;
 use std::path::Path;
@@ -49,7 +50,7 @@ macro_rules! get_controller_or_return_singular_none {
     ($cg:ident) => {
         match $cg.controller_of() {
             Some(c) => c,
-            None => return SingularPtrField::none(),
+            None => return MessageField::none(),
         }
     };
 }
@@ -75,7 +76,7 @@ macro_rules! set_resource {
 
 impl CgroupManager for Manager {
     fn apply(&self, pid: pid_t) -> Result<()> {
-        self.cgroup.add_task(CgroupPid::from(pid as u64))?;
+        self.cgroup.add_task_by_tgid(CgroupPid::from(pid as u64))?;
         Ok(())
     }
 
@@ -133,11 +134,10 @@ impl CgroupManager for Manager {
 
         let throttling_data = get_cpu_stats(&self.cgroup);
 
-        let cpu_stats = SingularPtrField::some(CpuStats {
+        let cpu_stats = MessageField::some(CpuStats {
             cpu_usage,
             throttling_data,
-            unknown_fields: UnknownFields::default(),
-            cached_size: CachedSize::default(),
+            ..Default::default()
         });
 
         // Memorystats
@@ -159,8 +159,7 @@ impl CgroupManager for Manager {
             pids_stats,
             blkio_stats,
             hugetlb_stats,
-            unknown_fields: UnknownFields::default(),
-            cached_size: CachedSize::default(),
+            ..Default::default()
         })
     }
 
@@ -193,6 +192,83 @@ impl CgroupManager for Manager {
 
         Ok(result)
     }
+
+    fn update_cpuset_path(&self, guest_cpuset: &str, container_cpuset: &str) -> Result<()> {
+        if guest_cpuset.is_empty() {
+            return Ok(());
+        }
+        info!(sl!(), "update_cpuset_path to: {}", guest_cpuset);
+
+        let h = cgroups::hierarchies::auto();
+        let root_cg = h.root_control_group();
+
+        let root_cpuset_controller: &CpuSetController = root_cg.controller_of().unwrap();
+        let path = root_cpuset_controller.path();
+        let root_path = Path::new(path);
+        info!(sl!(), "root cpuset path: {:?}", &path);
+
+        let container_cpuset_controller: &CpuSetController = self.cgroup.controller_of().unwrap();
+        let path = container_cpuset_controller.path();
+        let container_path = Path::new(path);
+        info!(sl!(), "container cpuset path: {:?}", &path);
+
+        let mut paths = vec![];
+        for ancestor in container_path.ancestors() {
+            if ancestor == root_path {
+                break;
+            }
+            paths.push(ancestor);
+        }
+        info!(sl!(), "parent paths to update cpuset: {:?}", &paths);
+
+        let mut i = paths.len();
+        loop {
+            if i == 0 {
+                break;
+            }
+            i -= 1;
+
+            // remove cgroup root from path
+            let r_path = &paths[i]
+                .to_str()
+                .unwrap()
+                .trim_start_matches(root_path.to_str().unwrap());
+            info!(sl!(), "updating cpuset for parent path {:?}", &r_path);
+            let cg = new_cgroup(cgroups::hierarchies::auto(), r_path)?;
+            let cpuset_controller: &CpuSetController = cg.controller_of().unwrap();
+            cpuset_controller.set_cpus(guest_cpuset)?;
+        }
+
+        if !container_cpuset.is_empty() {
+            info!(
+                sl!(),
+                "updating cpuset for container path: {:?} cpuset: {}",
+                &container_path,
+                container_cpuset
+            );
+            container_cpuset_controller.set_cpus(container_cpuset)?;
+        }
+
+        Ok(())
+    }
+
+    fn get_cgroup_path(&self, cg: &str) -> Result<String> {
+        if cgroups::hierarchies::is_cgroup2_unified_mode() {
+            let cg_path = format!("/sys/fs/cgroup/{}", self.cpath);
+            return Ok(cg_path);
+        }
+
+        // for cgroup v1
+        Ok(self.paths.get(cg).map(|s| s.to_string()).unwrap())
+    }
+
+    fn as_any(&self) -> Result<&dyn Any> {
+        Ok(self)
+    }
+
+    fn name(&self) -> &str {
+        "cgroupfs"
+    }
 }
 
 fn set_network_resources(
@@ -252,19 +328,28 @@ fn set_devices_resources(
 }
 
 fn set_hugepages_resources(
-    _cg: &cgroups::Cgroup,
+    cg: &cgroups::Cgroup,
     hugepage_limits: &[LinuxHugepageLimit],
     res: &mut cgroups::Resources,
 ) {
     info!(sl!(), "cgroup manager set hugepage");
     let mut limits = vec![];
+    let hugetlb_controller = cg.controller_of::<HugeTlbController>();
 
     for l in hugepage_limits.iter() {
-        let hr = HugePageResource {
-            size: l.page_size.clone(),
-            limit: l.limit,
-        };
-        limits.push(hr);
+        if hugetlb_controller.is_some() && hugetlb_controller.unwrap().size_supported(&l.page_size)
+        {
+            let hr = HugePageResource {
+                size: l.page_size.clone(),
+                limit: l.limit,
+            };
+            limits.push(hr);
+        } else {
+            warn!(
+                sl!(),
+                "{} page size support cannot be verified, dropping requested limit", l.page_size
+            );
+        }
     }
     res.hugepages.limits = limits;
 }
@@ -359,14 +444,14 @@ fn set_memory_resources(cg: &cgroups::Cgroup, memory: &LinuxMemory, update: bool
         let memstat = get_memory_stats(cg)
             .into_option()
             .ok_or_else(|| anyhow!("failed to get the cgroup memory stats"))?;
-        let memusage = memstat.get_usage();
+        let memusage = memstat.usage();
 
         // When update memory limit, the kernel would check the current memory limit
         // set against the new swap setting, if the current memory limit is large than
         // the new swap, then set limit first, otherwise the kernel would complain and
         // refused to set; on the other hand, if the current memory limit is smaller than
         // the new swap, then we should set the swap first and then set the memor limit.
-        if swap == -1 || memusage.get_limit() < swap as u64 {
+        if swap == -1 || memusage.limit() < swap as u64 {
             mem_controller.set_memswap_limit(swap)?;
             set_resource!(mem_controller, set_limit, memory, limit);
         } else {
@@ -570,21 +655,20 @@ lazy_static! {
     };
 }
 
-fn get_cpu_stats(cg: &cgroups::Cgroup) -> SingularPtrField<ThrottlingData> {
+fn get_cpu_stats(cg: &cgroups::Cgroup) -> MessageField<ThrottlingData> {
     let cpu_controller: &CpuController = get_controller_or_return_singular_none!(cg);
     let stat = cpu_controller.cpu().stat;
     let h = lines_to_map(&stat);
 
-    SingularPtrField::some(ThrottlingData {
+    MessageField::some(ThrottlingData {
         periods: *h.get("nr_periods").unwrap_or(&0),
         throttled_periods: *h.get("nr_throttled").unwrap_or(&0),
         throttled_time: *h.get("throttled_time").unwrap_or(&0),
-        unknown_fields: UnknownFields::default(),
-        cached_size: CachedSize::default(),
+        ..Default::default()
     })
 }
 
-fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> SingularPtrField<CpuUsage> {
+fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> MessageField<CpuUsage> {
     if let Some(cpuacct_controller) = cg.controller_of::<CpuAcctController>() {
         let cpuacct = cpuacct_controller.cpuacct();
 
@@ -598,24 +682,12 @@ fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> SingularPtrField<CpuUsage> {
 
         let percpu_usage = line_to_vec(&cpuacct.usage_percpu);
 
-        return SingularPtrField::some(CpuUsage {
+        return MessageField::some(CpuUsage {
             total_usage,
             percpu_usage,
             usage_in_kernelmode,
             usage_in_usermode,
-            unknown_fields: UnknownFields::default(),
-            cached_size: CachedSize::default(),
-        });
-    }
-
-    if cg.v2() {
-        return SingularPtrField::some(CpuUsage {
-            total_usage: 0,
-            percpu_usage: vec![],
-            usage_in_kernelmode: 0,
-            usage_in_usermode: 0,
-            unknown_fields: UnknownFields::default(),
-            cached_size: CachedSize::default(),
+            ..Default::default()
         });
     }
 
@@ -628,17 +700,16 @@ fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> SingularPtrField<CpuUsage> {
     let total_usage = *h.get("usage_usec").unwrap_or(&0);
     let percpu_usage = vec![];
 
-    SingularPtrField::some(CpuUsage {
+    MessageField::some(CpuUsage {
         total_usage,
         percpu_usage,
         usage_in_kernelmode,
         usage_in_usermode,
-        unknown_fields: UnknownFields::default(),
-        cached_size: CachedSize::default(),
+        ..Default::default()
     })
 }
 
-fn get_memory_stats(cg: &cgroups::Cgroup) -> SingularPtrField<MemoryStats> {
+fn get_memory_stats(cg: &cgroups::Cgroup) -> MessageField<MemoryStats> {
     let memory_controller: &MemController = get_controller_or_return_singular_none!(cg);
 
     // cache from memory stat
@@ -649,53 +720,49 @@ fn get_memory_stats(cg: &cgroups::Cgroup) -> SingularPtrField<MemoryStats> {
     let value = memory.use_hierarchy;
     let use_hierarchy = value == 1;
 
-    // gte memory datas
-    let usage = SingularPtrField::some(MemoryData {
+    // get memory data
+    let usage = MessageField::some(MemoryData {
         usage: memory.usage_in_bytes,
         max_usage: memory.max_usage_in_bytes,
         failcnt: memory.fail_cnt,
         limit: memory.limit_in_bytes as u64,
-        unknown_fields: UnknownFields::default(),
-        cached_size: CachedSize::default(),
+        ..Default::default()
     });
 
     // get swap usage
     let memswap = memory_controller.memswap();
 
-    let swap_usage = SingularPtrField::some(MemoryData {
+    let swap_usage = MessageField::some(MemoryData {
         usage: memswap.usage_in_bytes,
         max_usage: memswap.max_usage_in_bytes,
         failcnt: memswap.fail_cnt,
         limit: memswap.limit_in_bytes as u64,
-        unknown_fields: UnknownFields::default(),
-        cached_size: CachedSize::default(),
+        ..Default::default()
     });
 
     // get kernel usage
     let kmem_stat = memory_controller.kmem_stat();
 
-    let kernel_usage = SingularPtrField::some(MemoryData {
+    let kernel_usage = MessageField::some(MemoryData {
         usage: kmem_stat.usage_in_bytes,
         max_usage: kmem_stat.max_usage_in_bytes,
         failcnt: kmem_stat.fail_cnt,
         limit: kmem_stat.limit_in_bytes as u64,
-        unknown_fields: UnknownFields::default(),
-        cached_size: CachedSize::default(),
+        ..Default::default()
     });
 
-    SingularPtrField::some(MemoryStats {
+    MessageField::some(MemoryStats {
         cache,
         usage,
         swap_usage,
         kernel_usage,
         use_hierarchy,
         stats: memory.stat.raw,
-        unknown_fields: UnknownFields::default(),
-        cached_size: CachedSize::default(),
+        ..Default::default()
     })
 }
 
-fn get_pids_stats(cg: &cgroups::Cgroup) -> SingularPtrField<PidsStats> {
+fn get_pids_stats(cg: &cgroups::Cgroup) -> MessageField<PidsStats> {
     let pid_controller: &PidController = get_controller_or_return_singular_none!(cg);
 
     let current = pid_controller.get_pid_current().unwrap_or(0);
@@ -709,11 +776,10 @@ fn get_pids_stats(cg: &cgroups::Cgroup) -> SingularPtrField<PidsStats> {
         },
     } as u64;
 
-    SingularPtrField::some(PidsStats {
+    MessageField::some(PidsStats {
         current,
         limit,
-        unknown_fields: UnknownFields::default(),
-        cached_size: CachedSize::default(),
+        ..Default::default()
     })
 }
 
@@ -749,8 +815,8 @@ https://github.com/opencontainers/runc/blob/a5847db387ae28c0ca4ebe4beee1a76900c8
     Total 0
 */
 
-fn get_blkio_stat_blkiodata(blkiodata: &[BlkIoData]) -> RepeatedField<BlkioStatsEntry> {
-    let mut m = RepeatedField::new();
+fn get_blkio_stat_blkiodata(blkiodata: &[BlkIoData]) -> Vec<BlkioStatsEntry> {
+    let mut m = Vec::new();
     if blkiodata.is_empty() {
         return m;
     }
@@ -763,16 +829,15 @@ fn get_blkio_stat_blkiodata(blkiodata: &[BlkIoData]) -> RepeatedField<BlkioStats
             minor: d.minor as u64,
             op: op.clone(),
             value: d.data,
-            unknown_fields: UnknownFields::default(),
-            cached_size: CachedSize::default(),
+            ..Default::default()
         });
     }
 
     m
 }
 
-fn get_blkio_stat_ioservice(services: &[IoService]) -> RepeatedField<BlkioStatsEntry> {
-    let mut m = RepeatedField::new();
+fn get_blkio_stat_ioservice(services: &[IoService]) -> Vec<BlkioStatsEntry> {
+    let mut m = Vec::new();
 
     if services.is_empty() {
         return m;
@@ -796,17 +861,16 @@ fn build_blkio_stats_entry(major: i16, minor: i16, op: &str, value: u64) -> Blki
         minor: minor as u64,
         op: op.to_string(),
         value,
-        unknown_fields: UnknownFields::default(),
-        cached_size: CachedSize::default(),
+        ..Default::default()
     }
 }
 
-fn get_blkio_stats_v2(cg: &cgroups::Cgroup) -> SingularPtrField<BlkioStats> {
+fn get_blkio_stats_v2(cg: &cgroups::Cgroup) -> MessageField<BlkioStats> {
     let blkio_controller: &BlkIoController = get_controller_or_return_singular_none!(cg);
     let blkio = blkio_controller.blkio();
 
     let mut resp = BlkioStats::new();
-    let mut blkio_stats = RepeatedField::new();
+    let mut blkio_stats = Vec::new();
 
     let stat = blkio.io_stat;
     for s in stat {
@@ -822,10 +886,10 @@ fn get_blkio_stats_v2(cg: &cgroups::Cgroup) -> SingularPtrField<BlkioStats> {
 
     resp.io_service_bytes_recursive = blkio_stats;
 
-    SingularPtrField::some(resp)
+    MessageField::some(resp)
 }
 
-fn get_blkio_stats(cg: &cgroups::Cgroup) -> SingularPtrField<BlkioStats> {
+fn get_blkio_stats(cg: &cgroups::Cgroup) -> MessageField<BlkioStats> {
     if cg.v2() {
         return get_blkio_stats_v2(cg);
     }
@@ -858,7 +922,7 @@ fn get_blkio_stats(cg: &cgroups::Cgroup) -> SingularPtrField<BlkioStats> {
         m.sectors_recursive = get_blkio_stat_blkiodata(&blkio.sectors_recursive);
     }
 
-    SingularPtrField::some(m)
+    MessageField::some(m)
 }
 
 fn get_hugetlb_stats(cg: &cgroups::Cgroup) -> HashMap<String, HugetlbStats> {
@@ -882,8 +946,7 @@ fn get_hugetlb_stats(cg: &cgroups::Cgroup) -> HashMap<String, HugetlbStats> {
                 usage,
                 max_usage,
                 failcnt,
-                unknown_fields: UnknownFields::default(),
-                cached_size: CachedSize::default(),
+                ..Default::default()
             },
         );
     }
@@ -940,9 +1003,9 @@ pub fn get_mounts(paths: &HashMap<String, String>) -> Result<HashMap<String, Str
     Ok(m)
 }
 
-fn new_cgroup(h: Box<dyn cgroups::Hierarchy>, path: &str) -> Cgroup {
+fn new_cgroup(h: Box<dyn cgroups::Hierarchy>, path: &str) -> Result<Cgroup> {
     let valid_path = path.trim_start_matches('/').to_string();
-    cgroups::Cgroup::new(h, valid_path.as_str())
+    cgroups::Cgroup::new(h, valid_path.as_str()).map_err(anyhow::Error::from)
 }
 
 impl Manager {
@@ -964,83 +1027,16 @@ impl Manager {
             m.insert(key.to_string(), p);
         }
 
+        let cg = new_cgroup(cgroups::hierarchies::auto(), cpath)?;
+
         Ok(Self {
             paths: m,
             mounts,
             // rels: paths,
             cpath: cpath.to_string(),
-            cgroup: new_cgroup(cgroups::hierarchies::auto(), cpath),
+            cgroup: cg,
         })
     }
-
-    pub fn update_cpuset_path(&self, guest_cpuset: &str, container_cpuset: &str) -> Result<()> {
-        if guest_cpuset.is_empty() {
-            return Ok(());
-        }
-        info!(sl!(), "update_cpuset_path to: {}", guest_cpuset);
-
-        let h = cgroups::hierarchies::auto();
-        let root_cg = h.root_control_group();
-
-        let root_cpuset_controller: &CpuSetController = root_cg.controller_of().unwrap();
-        let path = root_cpuset_controller.path();
-        let root_path = Path::new(path);
-        info!(sl!(), "root cpuset path: {:?}", &path);
-
-        let container_cpuset_controller: &CpuSetController = self.cgroup.controller_of().unwrap();
-        let path = container_cpuset_controller.path();
-        let container_path = Path::new(path);
-        info!(sl!(), "container cpuset path: {:?}", &path);
-
-        let mut paths = vec![];
-        for ancestor in container_path.ancestors() {
-            if ancestor == root_path {
-                break;
-            }
-            paths.push(ancestor);
-        }
-        info!(sl!(), "parent paths to update cpuset: {:?}", &paths);
-
-        let mut i = paths.len();
-        loop {
-            if i == 0 {
-                break;
-            }
-            i -= 1;
-
-            // remove cgroup root from path
-            let r_path = &paths[i]
-                .to_str()
-                .unwrap()
-                .trim_start_matches(root_path.to_str().unwrap());
-            info!(sl!(), "updating cpuset for parent path {:?}", &r_path);
-            let cg = new_cgroup(cgroups::hierarchies::auto(), r_path);
-            let cpuset_controller: &CpuSetController = cg.controller_of().unwrap();
-            cpuset_controller.set_cpus(guest_cpuset)?;
-        }
-
-        if !container_cpuset.is_empty() {
-            info!(
-                sl!(),
-                "updating cpuset for container path: {:?} cpuset: {}",
-                &container_path,
-                container_cpuset
-            );
-            container_cpuset_controller.set_cpus(container_cpuset)?;
-        }
-
-        Ok(())
-    }
-
-    pub fn get_cg_path(&self, cg: &str) -> Option<String> {
-        if cgroups::hierarchies::is_cgroup2_unified_mode() {
-            let cg_path = format!("/sys/fs/cgroup/{}", self.cpath);
-            return Some(cg_path);
-        }
-
-        // for cgroup v1
-        self.paths.get(cg).map(|s| s.to_string())
-    }
 }
 
 // get the guest's online cpus.

src/agent/rustjail/src/cgroups/mock.rs

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use protobuf::{CachedSize, SingularPtrField, UnknownFields};
+use protobuf::MessageField;
 
 use crate::cgroups::Manager as CgroupManager;
 use crate::protocols::agent::{BlkioStats, CgroupStats, CpuStats, MemoryStats, PidsStats};
@@ -11,6 +11,7 @@ use anyhow::Result;
 use cgroups::freezer::FreezerState;
 use libc::{self, pid_t};
 use oci::LinuxResources;
+use std::any::Any;
 use std::collections::HashMap;
 use std::string::String;
 
@@ -32,13 +33,12 @@ impl CgroupManager for Manager {
 
     fn get_stats(&self) -> Result<CgroupStats> {
         Ok(CgroupStats {
-            cpu_stats: SingularPtrField::some(CpuStats::default()),
-            memory_stats: SingularPtrField::some(MemoryStats::new()),
-            pids_stats: SingularPtrField::some(PidsStats::new()),
-            blkio_stats: SingularPtrField::some(BlkioStats::new()),
+            cpu_stats: MessageField::some(CpuStats::default()),
+            memory_stats: MessageField::some(MemoryStats::new()),
+            pids_stats: MessageField::some(PidsStats::new()),
+            blkio_stats: MessageField::some(BlkioStats::new()),
             hugetlb_stats: HashMap::new(),
-            unknown_fields: UnknownFields::default(),
-            cached_size: CachedSize::default(),
+            ..Default::default()
         })
     }
 
@@ -53,6 +53,22 @@ impl CgroupManager for Manager {
     fn get_pids(&self) -> Result<Vec<pid_t>> {
         Ok(Vec::new())
     }
+
+    fn update_cpuset_path(&self, _: &str, _: &str) -> Result<()> {
+        Ok(())
+    }
+
+    fn get_cgroup_path(&self, _: &str) -> Result<String> {
+        Ok("".to_string())
+    }
+
+    fn as_any(&self) -> Result<&dyn Any> {
+        Ok(self)
+    }
+
+    fn name(&self) -> &str {
+        "mock"
+    }
 }
 
 impl Manager {
@@ -63,12 +79,4 @@ impl Manager {
             cpath: cpath.to_string(),
         })
     }
-
-    pub fn update_cpuset_path(&self, _: &str, _: &str) -> Result<()> {
-        Ok(())
-    }
-
-    pub fn get_cg_path(&self, _: &str) -> Option<String> {
-        Some("".to_string())
-    }
 }

src/agent/rustjail/src/cgroups/mod.rs

@@ -4,8 +4,10 @@
 //
 
 use anyhow::{anyhow, Result};
+use core::fmt::Debug;
 use oci::LinuxResources;
 use protocols::agent::CgroupStats;
+use std::any::Any;
 
 use cgroups::freezer::FreezerState;
 
@@ -38,4 +40,24 @@ pub trait Manager {
     fn set(&self, _container: &LinuxResources, _update: bool) -> Result<()> {
         Err(anyhow!("not supported!"))
     }
+
+    fn update_cpuset_path(&self, _: &str, _: &str) -> Result<()> {
+        Err(anyhow!("not supported!"))
+    }
+
+    fn get_cgroup_path(&self, _: &str) -> Result<String> {
+        Err(anyhow!("not supported!"))
+    }
+
+    fn as_any(&self) -> Result<&dyn Any> {
+        Err(anyhow!("not supported!"))
+    }
+
+    fn name(&self) -> &str;
+}
+
+impl Debug for dyn Manager + Send + Sync {
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+        write!(f, "{}", self.name())
+    }
 }

src/agent/rustjail/src/cgroups/systemd.rs

@@ -1,10 +0,0 @@
-// Copyright (c) 2019 Ant Financial
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-use crate::cgroups::Manager as CgroupManager;
-
-pub struct Manager {}
-
-impl CgroupManager for Manager {}

src/agent/rustjail/src/cgroups/systemd/cgroups_path.rs

@@ -0,0 +1,95 @@
+// Copyright 2021-2022 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use anyhow::{anyhow, Result};
+
+use super::common::{DEFAULT_SLICE, SCOPE_SUFFIX, SLICE_SUFFIX};
+use std::string::String;
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct CgroupsPath {
+    pub slice: String,
+    pub prefix: String,
+    pub name: String,
+}
+
+impl CgroupsPath {
+    pub fn new(cgroups_path_str: &str) -> Result<Self> {
+        let path_vec: Vec<&str> = cgroups_path_str.split(':').collect();
+        if path_vec.len() != 3 {
+            return Err(anyhow!("invalid cpath: {:?}", cgroups_path_str));
+        }
+
+        Ok(CgroupsPath {
+            slice: if path_vec[0].is_empty() {
+                DEFAULT_SLICE.to_string()
+            } else {
+                path_vec[0].to_owned()
+            },
+            prefix: path_vec[1].to_owned(),
+            name: path_vec[2].to_owned(),
+        })
+    }
+
+    // ref: https://github.com/opencontainers/runc/blob/main/docs/systemd.md
+    // return: (parent_slice, unit_name)
+    pub fn parse(&self) -> Result<(String, String)> {
+        Ok((
+            parse_parent(self.slice.to_owned())?,
+            get_unit_name(self.prefix.to_owned(), self.name.to_owned()),
+        ))
+    }
+}
+
+fn parse_parent(slice: String) -> Result<String> {
+    if !slice.ends_with(SLICE_SUFFIX) || slice.contains('/') {
+        return Err(anyhow!("invalid slice name: {}", slice));
+    } else if slice == "-.slice" {
+        return Ok(String::new());
+    }
+
+    let mut slice_path = String::new();
+    let mut prefix = String::new();
+    for subslice in slice.trim_end_matches(SLICE_SUFFIX).split('-') {
+        if subslice.is_empty() {
+            return Err(anyhow!("invalid slice name: {}", slice));
+        }
+        slice_path = format!("{}/{}{}{}", slice_path, prefix, subslice, SLICE_SUFFIX);
+        prefix = format!("{}{}-", prefix, subslice);
+    }
+    slice_path.remove(0);
+    Ok(slice_path)
+}
+
+fn get_unit_name(prefix: String, name: String) -> String {
+    if name.ends_with(SLICE_SUFFIX) {
+        name
+    } else if prefix.is_empty() {
+        format!("{}{}", name, SCOPE_SUFFIX)
+    } else {
+        format!("{}-{}{}", prefix, name, SCOPE_SUFFIX)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::CgroupsPath;
+
+    #[test]
+    fn test_cgroup_path_parse() {
+        let slice = "system.slice";
+        let prefix = "kata_agent";
+        let name = "123";
+        let cgroups_path =
+            CgroupsPath::new(format!("{}:{}:{}", slice, prefix, name).as_str()).unwrap();
+        assert_eq!(slice, cgroups_path.slice.as_str());
+        assert_eq!(prefix, cgroups_path.prefix.as_str());
+        assert_eq!(name, cgroups_path.name.as_str());
+
+        let (parent_slice, unit_name) = cgroups_path.parse().unwrap();
+        assert_eq!(format!("{}", slice), parent_slice);
+        assert_eq!(format!("{}-{}.scope", prefix, name), unit_name);
+    }
+}

src/agent/rustjail/src/cgroups/systemd/common.rs

@@ -0,0 +1,17 @@
+// Copyright 2021-2022 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+pub const DEFAULT_SLICE: &str = "system.slice";
+pub const SLICE_SUFFIX: &str = ".slice";
+pub const SCOPE_SUFFIX: &str = ".scope";
+pub const UNIT_MODE: &str = "replace";
+
+pub type Properties<'a> = Vec<(&'a str, zbus::zvariant::Value<'a>)>;
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub enum CgroupHierarchy {
+    Legacy,
+    Unified,
+}

src/agent/rustjail/src/cgroups/systemd/dbus_client.rs

@@ -0,0 +1,129 @@
+// Copyright 2021-2022 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::vec;
+
+use super::common::CgroupHierarchy;
+use super::common::{Properties, SLICE_SUFFIX, UNIT_MODE};
+use super::interface::system::ManagerProxyBlocking as SystemManager;
+use anyhow::{Context, Result};
+use zbus::zvariant::Value;
+
+pub trait SystemdInterface {
+    fn start_unit(
+        &self,
+        pid: i32,
+        parent: &str,
+        unit_name: &str,
+        cg_hierarchy: &CgroupHierarchy,
+    ) -> Result<()>;
+
+    fn set_properties(&self, unit_name: &str, properties: &Properties) -> Result<()>;
+
+    fn stop_unit(&self, unit_name: &str) -> Result<()>;
+
+    fn get_version(&self) -> Result<String>;
+
+    fn unit_exists(&self, unit_name: &str) -> Result<bool>;
+
+    fn add_process(&self, pid: i32, unit_name: &str) -> Result<()>;
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct DBusClient {}
+
+impl DBusClient {
+    fn build_proxy(&self) -> Result<SystemManager<'static>> {
+        let connection =
+            zbus::blocking::Connection::system().context("Establishing a D-Bus connection")?;
+        let proxy = SystemManager::new(&connection).context("Building a D-Bus proxy manager")?;
+        Ok(proxy)
+    }
+}
+
+impl SystemdInterface for DBusClient {
+    fn start_unit(
+        &self,
+        pid: i32,
+        parent: &str,
+        unit_name: &str,
+        cg_hierarchy: &CgroupHierarchy,
+    ) -> Result<()> {
+        let proxy = self.build_proxy()?;
+
+        // enable CPUAccounting & MemoryAccounting & (Block)IOAccounting by default
+        let mut properties: Properties = vec![
+            ("CPUAccounting", Value::Bool(true)),
+            ("DefaultDependencies", Value::Bool(false)),
+            ("MemoryAccounting", Value::Bool(true)),
+            ("TasksAccounting", Value::Bool(true)),
+            ("Description", Value::Str("kata-agent container".into())),
+            ("PIDs", Value::Array(vec![pid as u32].into())),
+        ];
+
+        match *cg_hierarchy {
+            CgroupHierarchy::Legacy => properties.push(("IOAccounting", Value::Bool(true))),
+            CgroupHierarchy::Unified => properties.push(("BlockIOAccounting", Value::Bool(true))),
+        }
+
+        if unit_name.ends_with(SLICE_SUFFIX) {
+            properties.push(("Wants", Value::Str(parent.into())));
+        } else {
+            properties.push(("Slice", Value::Str(parent.into())));
+            properties.push(("Delegate", Value::Bool(true)));
+        }
+
+        proxy
+            .start_transient_unit(unit_name, UNIT_MODE, &properties, &[])
+            .with_context(|| format!("failed to start transient unit {}", unit_name))?;
+        Ok(())
+    }
+
+    fn set_properties(&self, unit_name: &str, properties: &Properties) -> Result<()> {
+        let proxy = self.build_proxy()?;
+
+        proxy
+            .set_unit_properties(unit_name, true, properties)
+            .with_context(|| format!("failed to set unit properties {}", unit_name))?;
+
+        Ok(())
+    }
+
+    fn stop_unit(&self, unit_name: &str) -> Result<()> {
+        let proxy = self.build_proxy()?;
+
+        proxy
+            .stop_unit(unit_name, UNIT_MODE)
+            .with_context(|| format!("failed to stop unit {}", unit_name))?;
+        Ok(())
+    }
+
+    fn get_version(&self) -> Result<String> {
+        let proxy = self.build_proxy()?;
+
+        let systemd_version = proxy
+            .version()
+            .with_context(|| "failed to get systemd version".to_string())?;
+        Ok(systemd_version)
+    }
+
+    fn unit_exists(&self, unit_name: &str) -> Result<bool> {
+        let proxy = self
+            .build_proxy()
+            .with_context(|| format!("Checking if systemd unit {} exists", unit_name))?;
+
+        Ok(proxy.get_unit(unit_name).is_ok())
+    }
+
+    fn add_process(&self, pid: i32, unit_name: &str) -> Result<()> {
+        let proxy = self.build_proxy()?;
+
+        proxy
+            .attach_processes_to_unit(unit_name, "/", &[pid as u32])
+            .with_context(|| format!("failed to add process {}", unit_name))?;
+
+        Ok(())
+    }
+}

src/agent/rustjail/src/cgroups/systemd/interface/mod.rs

@@ -0,0 +1,7 @@
+// Copyright 2021-2022 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+pub(crate) mod session;
+pub(crate) mod system;

src/agent/rustjail/src/cgroups/systemd/manager.rs

@@ -0,0 +1,133 @@
+// Copyright 2021-2022 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use crate::cgroups::Manager as CgroupManager;
+use crate::protocols::agent::CgroupStats;
+use anyhow::Result;
+use cgroups::freezer::FreezerState;
+use libc::{self, pid_t};
+use oci::LinuxResources;
+use std::any::Any;
+use std::collections::HashMap;
+use std::convert::TryInto;
+use std::string::String;
+use std::vec;
+
+use super::super::fs::Manager as FsManager;
+
+use super::cgroups_path::CgroupsPath;
+use super::common::{CgroupHierarchy, Properties};
+use super::dbus_client::{DBusClient, SystemdInterface};
+use super::subsystem::transformer::Transformer;
+use super::subsystem::{cpu::Cpu, cpuset::CpuSet, memory::Memory, pids::Pids};
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct Manager {
+    pub paths: HashMap<String, String>,
+    pub mounts: HashMap<String, String>,
+    pub cgroups_path: CgroupsPath,
+    pub cpath: String,
+    pub unit_name: String,
+    // dbus client for set properties
+    dbus_client: DBusClient,
+    // fs manager for get properties
+    fs_manager: FsManager,
+    // cgroup version for different dbus properties
+    cg_hierarchy: CgroupHierarchy,
+}
+
+impl CgroupManager for Manager {
+    fn apply(&self, pid: pid_t) -> Result<()> {
+        let unit_name = self.unit_name.as_str();
+        if self.dbus_client.unit_exists(unit_name)? {
+            self.dbus_client.add_process(pid, self.unit_name.as_str())?;
+        } else {
+            self.dbus_client.start_unit(
+                (pid as u32).try_into().unwrap(),
+                self.cgroups_path.slice.as_str(),
+                self.unit_name.as_str(),
+                &self.cg_hierarchy,
+            )?;
+        }
+
+        Ok(())
+    }
+
+    fn set(&self, r: &LinuxResources, _: bool) -> Result<()> {
+        let mut properties: Properties = vec![];
+
+        let systemd_version = self.dbus_client.get_version()?;
+        let systemd_version_str = systemd_version.as_str();
+
+        Cpu::apply(r, &mut properties, &self.cg_hierarchy, systemd_version_str)?;
+        Memory::apply(r, &mut properties, &self.cg_hierarchy, systemd_version_str)?;
+        Pids::apply(r, &mut properties, &self.cg_hierarchy, systemd_version_str)?;
+        CpuSet::apply(r, &mut properties, &self.cg_hierarchy, systemd_version_str)?;
+
+        self.dbus_client
+            .set_properties(self.unit_name.as_str(), &properties)?;
+
+        Ok(())
+    }
+
+    fn get_stats(&self) -> Result<CgroupStats> {
+        self.fs_manager.get_stats()
+    }
+
+    fn freeze(&self, state: FreezerState) -> Result<()> {
+        self.fs_manager.freeze(state)
+    }
+
+    fn destroy(&mut self) -> Result<()> {
+        self.dbus_client.stop_unit(self.unit_name.as_str())?;
+        self.fs_manager.destroy()
+    }
+
+    fn get_pids(&self) -> Result<Vec<pid_t>> {
+        self.fs_manager.get_pids()
+    }
+
+    fn update_cpuset_path(&self, guest_cpuset: &str, container_cpuset: &str) -> Result<()> {
+        self.fs_manager
+            .update_cpuset_path(guest_cpuset, container_cpuset)
+    }
+
+    fn get_cgroup_path(&self, cg: &str) -> Result<String> {
+        self.fs_manager.get_cgroup_path(cg)
+    }
+
+    fn as_any(&self) -> Result<&dyn Any> {
+        Ok(self)
+    }
+
+    fn name(&self) -> &str {
+        "systemd"
+    }
+}
+
+impl Manager {
+    pub fn new(cgroups_path_str: &str) -> Result<Self> {
+        let cgroups_path = CgroupsPath::new(cgroups_path_str)?;
+        let (parent_slice, unit_name) = cgroups_path.parse()?;
+        let cpath = parent_slice + "/" + &unit_name;
+
+        let fs_manager = FsManager::new(cpath.as_str())?;
+
+        Ok(Manager {
+            paths: fs_manager.paths.clone(),
+            mounts: fs_manager.mounts.clone(),
+            cgroups_path,
+            cpath,
+            unit_name,
+            dbus_client: DBusClient {},
+            fs_manager,
+            cg_hierarchy: if cgroups::hierarchies::is_cgroup2_unified_mode() {
+                CgroupHierarchy::Unified
+            } else {
+                CgroupHierarchy::Legacy
+            },
+        })
+    }
+}

src/agent/rustjail/src/cgroups/systemd/mod.rs

@@ -0,0 +1,12 @@
+// Copyright 2021-2022 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+pub mod manager;
+
+mod cgroups_path;
+mod common;
+mod dbus_client;
+mod interface;
+mod subsystem;

src/agent/rustjail/src/cgroups/systemd/subsystem/cpu.rs

@@ -0,0 +1,139 @@
+// Copyright 2021-2022 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use super::super::common::{CgroupHierarchy, Properties};
+use super::transformer::Transformer;
+
+use anyhow::Result;
+use oci::{LinuxCpu, LinuxResources};
+use zbus::zvariant::Value;
+
+const BASIC_SYSTEMD_VERSION: &str = "242";
+const DEFAULT_CPUQUOTAPERIOD: u64 = 100 * 1000;
+const SEC2MICROSEC: u64 = 1000 * 1000;
+const BASIC_INTERVAL: u64 = 10 * 1000;
+
+pub struct Cpu {}
+
+impl Transformer for Cpu {
+    fn apply(
+        r: &LinuxResources,
+        properties: &mut Properties,
+        cgroup_hierarchy: &CgroupHierarchy,
+        systemd_version: &str,
+    ) -> Result<()> {
+        if let Some(cpu_resources) = &r.cpu {
+            match cgroup_hierarchy {
+                CgroupHierarchy::Legacy => {
+                    Self::legacy_apply(cpu_resources, properties, systemd_version)?
+                }
+                CgroupHierarchy::Unified => {
+                    Self::unified_apply(cpu_resources, properties, systemd_version)?
+                }
+            }
+        }
+
+        Ok(())
+    }
+}
+
+impl Cpu {
+    // v1:
+    // cpu.shares <-> CPUShares
+    // cpu.period <-> CPUQuotaPeriodUSec
+    // cpu.period & cpu.quota <-> CPUQuotaPerSecUSec
+    fn legacy_apply(
+        cpu_resources: &LinuxCpu,
+        properties: &mut Properties,
+        systemd_version: &str,
+    ) -> Result<()> {
+        if let Some(shares) = cpu_resources.shares {
+            properties.push(("CPUShares", Value::U64(shares)));
+        }
+
+        if let Some(period) = cpu_resources.period {
+            if period != 0 && systemd_version >= BASIC_SYSTEMD_VERSION {
+                properties.push(("CPUQuotaPeriodUSec", Value::U64(period)));
+            }
+        }
+
+        if let Some(quota) = cpu_resources.quota {
+            let period = cpu_resources.period.unwrap_or(DEFAULT_CPUQUOTAPERIOD);
+            if period != 0 {
+                let cpu_quota_per_sec_usec = resolve_cpuquota(quota, period);
+                properties.push(("CPUQuotaPerSecUSec", Value::U64(cpu_quota_per_sec_usec)));
+            }
+        }
+
+        Ok(())
+    }
+
+    // v2:
+    // cpu.shares <-> CPUWeight
+    // cpu.period <-> CPUQuotaPeriodUSec
+    // cpu.period & cpu.quota <-> CPUQuotaPerSecUSec
+    fn unified_apply(
+        cpu_resources: &LinuxCpu,
+        properties: &mut Properties,
+        systemd_version: &str,
+    ) -> Result<()> {
+        if let Some(shares) = cpu_resources.shares {
+            let weight = shares_to_weight(shares);
+            properties.push(("CPUWeight", Value::U64(weight)));
+        }
+
+        if let Some(period) = cpu_resources.period {
+            if period != 0 && systemd_version >= BASIC_SYSTEMD_VERSION {
+                properties.push(("CPUQuotaPeriodUSec", Value::U64(period)));
+            }
+        }
+
+        if let Some(quota) = cpu_resources.quota {
+            let period = cpu_resources.period.unwrap_or(DEFAULT_CPUQUOTAPERIOD);
+            if period != 0 {
+                let cpu_quota_per_sec_usec = resolve_cpuquota(quota, period);
+                properties.push(("CPUQuotaPerSecUSec", Value::U64(cpu_quota_per_sec_usec)));
+            }
+        }
+
+        Ok(())
+    }
+}
+
+// ref: https://github.com/containers/crun/blob/main/crun.1.md#cgroup-v2
+// [2-262144] to [1-10000]
+fn shares_to_weight(shares: u64) -> u64 {
+    if shares == 0 {
+        return 100;
+    }
+
+    1 + ((shares - 2) * 9999) / 262142
+}
+
+fn resolve_cpuquota(quota: i64, period: u64) -> u64 {
+    let mut cpu_quota_per_sec_usec = u64::MAX;
+    if quota > 0 {
+        cpu_quota_per_sec_usec = (quota as u64) * SEC2MICROSEC / period;
+        if cpu_quota_per_sec_usec % BASIC_INTERVAL != 0 {
+            cpu_quota_per_sec_usec =
+                ((cpu_quota_per_sec_usec / BASIC_INTERVAL) + 1) * BASIC_INTERVAL;
+        }
+    }
+    cpu_quota_per_sec_usec
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::cgroups::systemd::subsystem::cpu::resolve_cpuquota;
+
+    #[test]
+    fn test_unified_cpuquota() {
+        let quota: i64 = 1000000;
+        let period: u64 = 500000;
+        let cpu_quota_per_sec_usec = resolve_cpuquota(quota, period);
+
+        assert_eq!(2000000, cpu_quota_per_sec_usec);
+    }
+}

src/agent/rustjail/src/cgroups/systemd/subsystem/cpuset.rs

@@ -0,0 +1,124 @@
+// Copyright 2021-2022 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use super::super::common::{CgroupHierarchy, Properties};
+
+use super::transformer::Transformer;
+
+use anyhow::{bail, Result};
+use bit_vec::BitVec;
+use oci::{LinuxCpu, LinuxResources};
+use std::convert::{TryFrom, TryInto};
+use zbus::zvariant::Value;
+
+const BASIC_SYSTEMD_VERSION: &str = "244";
+
+pub struct CpuSet {}
+
+impl Transformer for CpuSet {
+    fn apply(
+        r: &LinuxResources,
+        properties: &mut Properties,
+        _: &CgroupHierarchy,
+        systemd_version: &str,
+    ) -> Result<()> {
+        if let Some(cpuset_resources) = &r.cpu {
+            Self::apply(cpuset_resources, properties, systemd_version)?;
+        }
+
+        Ok(())
+    }
+}
+
+// v1 & v2:
+// cpuset.cpus <-> AllowedCPUs (v244)
+// cpuset.mems <-> AllowedMemoryNodes (v244)
+impl CpuSet {
+    fn apply(
+        cpuset_resources: &LinuxCpu,
+        properties: &mut Properties,
+        systemd_version: &str,
+    ) -> Result<()> {
+        if systemd_version < BASIC_SYSTEMD_VERSION {
+            return Ok(());
+        }
+
+        let cpus = cpuset_resources.cpus.as_str();
+        if !cpus.is_empty() {
+            let cpus_vec: BitMask = cpus.try_into()?;
+            properties.push(("AllowedCPUs", Value::Array(cpus_vec.0.into())));
+        }
+
+        let mems = cpuset_resources.mems.as_str();
+        if !mems.is_empty() {
+            let mems_vec: BitMask = mems.try_into()?;
+            properties.push(("AllowedMemoryNodes", Value::Array(mems_vec.0.into())));
+        }
+
+        Ok(())
+    }
+}
+
+struct BitMask(Vec<u8>);
+
+impl TryFrom<&str> for BitMask {
+    type Error = anyhow::Error;
+
+    fn try_from(bitmask_str: &str) -> Result<Self, Self::Error> {
+        let mut bitmask_vec = BitVec::from_elem(8, false);
+        let bitmask_str_vec: Vec<&str> = bitmask_str.split(',').collect();
+        for bitmask in bitmask_str_vec.iter() {
+            let range: Vec<&str> = bitmask.split('-').collect();
+            match range.len() {
+                1 => {
+                    let idx: usize = range[0].parse()?;
+                    while idx >= bitmask_vec.len() {
+                        bitmask_vec.grow(8, false);
+                    }
+                    bitmask_vec.set(adjust_index(idx), true);
+                }
+                2 => {
+                    let left_index = range[0].parse()?;
+                    let right_index = range[1].parse()?;
+                    while right_index >= bitmask_vec.len() {
+                        bitmask_vec.grow(8, false);
+                    }
+                    for idx in left_index..=right_index {
+                        bitmask_vec.set(adjust_index(idx), true);
+                    }
+                }
+                _ => bail!("invalid bitmask str {}", bitmask_str),
+            }
+        }
+        let mut result_vec = bitmask_vec.to_bytes();
+        result_vec.reverse();
+
+        Ok(BitMask(result_vec))
+    }
+}
+
+#[inline(always)]
+fn adjust_index(idx: usize) -> usize {
+    idx / 8 * 8 + 7 - idx % 8
+}
+
+#[cfg(test)]
+mod tests {
+    use std::convert::TryInto;
+
+    use crate::cgroups::systemd::subsystem::cpuset::BitMask;
+
+    #[test]
+    fn test_bitmask_conversion() {
+        let cpus_vec: BitMask = "2-4".try_into().unwrap();
+        assert_eq!(vec![0b11100 as u8], cpus_vec.0);
+
+        let cpus_vec: BitMask = "1,7".try_into().unwrap();
+        assert_eq!(vec![0b10000010 as u8], cpus_vec.0);
+
+        let cpus_vec: BitMask = "0,2-3,7".try_into().unwrap();
+        assert_eq!(vec![0b10001101 as u8], cpus_vec.0);
+    }
+}

src/agent/rustjail/src/cgroups/systemd/subsystem/memory.rs

@@ -0,0 +1,117 @@
+// Copyright 2021-2022 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use super::super::common::{CgroupHierarchy, Properties};
+
+use super::transformer::Transformer;
+
+use anyhow::{bail, Result};
+use oci::{LinuxMemory, LinuxResources};
+use zbus::zvariant::Value;
+
+pub struct Memory {}
+
+impl Transformer for Memory {
+    fn apply(
+        r: &LinuxResources,
+        properties: &mut Properties,
+        cgroup_hierarchy: &CgroupHierarchy,
+        _: &str,
+    ) -> Result<()> {
+        if let Some(memory_resources) = &r.memory {
+            match cgroup_hierarchy {
+                CgroupHierarchy::Legacy => Self::legacy_apply(memory_resources, properties)?,
+                CgroupHierarchy::Unified => Self::unified_apply(memory_resources, properties)?,
+            }
+        }
+
+        Ok(())
+    }
+}
+
+impl Memory {
+    // v1:
+    // memory.limit <-> MemoryLimit
+    fn legacy_apply(memory_resources: &LinuxMemory, properties: &mut Properties) -> Result<()> {
+        if let Some(limit) = memory_resources.limit {
+            let limit = match limit {
+                1..=i64::MAX => limit as u64,
+                0 => u64::MAX,
+                _ => bail!("invalid memory.limit"),
+            };
+            properties.push(("MemoryLimit", Value::U64(limit)));
+        }
+
+        Ok(())
+    }
+
+    // v2:
+    // memory.low <-> MemoryLow
+    // memory.max <-> MemoryMax
+    // memory.swap & memory.limit <-> MemorySwapMax
+    fn unified_apply(memory_resources: &LinuxMemory, properties: &mut Properties) -> Result<()> {
+        if let Some(limit) = memory_resources.limit {
+            let limit = match limit {
+                1..=i64::MAX => limit as u64,
+                0 => u64::MAX,
+                _ => bail!("invalid memory.limit: {}", limit),
+            };
+            properties.push(("MemoryMax", Value::U64(limit)));
+        }
+
+        if let Some(reservation) = memory_resources.reservation {
+            let reservation = match reservation {
+                1..=i64::MAX => reservation as u64,
+                0 => u64::MAX,
+                _ => bail!("invalid memory.reservation: {}", reservation),
+            };
+            properties.push(("MemoryLow", Value::U64(reservation)));
+        }
+
+        let swap = match memory_resources.swap {
+            Some(0) => u64::MAX,
+            Some(1..=i64::MAX) => match memory_resources.limit {
+                Some(1..=i64::MAX) => {
+                    (memory_resources.limit.unwrap() - memory_resources.swap.unwrap()) as u64
+                }
+                _ => bail!("invalid memory.limit when memory.swap specified"),
+            },
+            None => u64::MAX,
+            _ => bail!("invalid memory.swap"),
+        };
+
+        properties.push(("MemorySwapMax", Value::U64(swap)));
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Memory;
+    use super::Properties;
+    use super::Value;
+
+    #[test]
+    fn test_unified_memory() {
+        let memory_resources = oci::LinuxMemory {
+            limit: Some(736870912),
+            reservation: Some(536870912),
+            swap: Some(536870912),
+            kernel: Some(0),
+            kernel_tcp: Some(0),
+            swappiness: Some(0),
+            disable_oom_killer: Some(false),
+        };
+        let mut properties: Properties = vec![];
+
+        assert_eq!(
+            true,
+            Memory::unified_apply(&memory_resources, &mut properties).is_ok()
+        );
+
+        assert_eq!(Value::U64(200000000), properties[2].1);
+    }
+}

src/agent/rustjail/src/cgroups/systemd/subsystem/mod.rs

@@ -0,0 +1,10 @@
+// Copyright 2021-2022 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+pub mod cpu;
+pub mod cpuset;
+pub mod memory;
+pub mod pids;
+pub mod transformer;

src/agent/rustjail/src/cgroups/systemd/subsystem/pids.rs

@@ -0,0 +1,60 @@
+// Copyright 2021-2022 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use super::super::common::{CgroupHierarchy, Properties};
+
+use super::transformer::Transformer;
+
+use anyhow::Result;
+use oci::{LinuxPids, LinuxResources};
+use zbus::zvariant::Value;
+
+pub struct Pids {}
+
+impl Transformer for Pids {
+    fn apply(
+        r: &LinuxResources,
+        properties: &mut Properties,
+        _: &CgroupHierarchy,
+        _: &str,
+    ) -> Result<()> {
+        if let Some(pids_resources) = &r.pids {
+            Self::apply(pids_resources, properties)?;
+        }
+
+        Ok(())
+    }
+}
+
+// pids.limit <-> TasksMax
+impl Pids {
+    fn apply(pids_resources: &LinuxPids, properties: &mut Properties) -> Result<()> {
+        let limit = if pids_resources.limit > 0 {
+            pids_resources.limit as u64
+        } else {
+            u64::MAX
+        };
+
+        properties.push(("TasksMax", Value::U64(limit)));
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Pids;
+    use super::Properties;
+    use super::Value;
+
+    #[test]
+    fn test_subsystem_workflow() {
+        let pids_resources = oci::LinuxPids { limit: 0 };
+        let mut properties: Properties = vec![];
+
+        assert_eq!(true, Pids::apply(&pids_resources, &mut properties).is_ok());
+
+        assert_eq!(Value::U64(u64::MAX), properties[0].1);
+    }
+}

src/agent/rustjail/src/cgroups/systemd/subsystem/transformer.rs

@@ -0,0 +1,17 @@
+// Copyright 2021-2022 Kata Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use super::super::common::{CgroupHierarchy, Properties};
+use anyhow::Result;
+use oci::LinuxResources;
+
+pub trait Transformer {
+    fn apply(
+        r: &LinuxResources,
+        properties: &mut Properties,
+        cgroup_hierarchy: &CgroupHierarchy,
+        systemd_version: &str,
+    ) -> Result<()>;
+}

src/agent/rustjail/src/console.rs

@@ -6,8 +6,9 @@
 use anyhow::{anyhow, Result};
 use nix::errno::Errno;
 use nix::pty;
-use nix::sys::{socket, uio};
+use nix::sys::socket;
 use nix::unistd::{self, dup2};
+use std::io::IoSlice;
 use std::os::unix::io::{AsRawFd, RawFd};
 use std::path::Path;
 
@@ -23,10 +24,7 @@ pub fn setup_console_socket(csocket_path: &str) -> Result<Option<RawFd>> {
         None,
     )?;
 
-    match socket::connect(
-        socket_fd,
-        &socket::SockAddr::Unix(socket::UnixAddr::new(Path::new(csocket_path))?),
-    ) {
+    match socket::connect(socket_fd, &socket::UnixAddr::new(Path::new(csocket_path))?) {
         Ok(()) => Ok(Some(socket_fd)),
         Err(errno) => Err(anyhow!("failed to open console fd: {}", errno)),
     }
@@ -36,11 +34,11 @@ pub fn setup_master_console(socket_fd: RawFd) -> Result<()> {
     let pseudo = pty::openpty(None, None)?;
 
     let pty_name: &[u8] = b"/dev/ptmx";
-    let iov = [uio::IoVec::from_slice(pty_name)];
+    let iov = [IoSlice::new(pty_name)];
     let fds = [pseudo.master];
     let cmsg = socket::ControlMessage::ScmRights(&fds);
 
-    socket::sendmsg(socket_fd, &iov, &[cmsg], socket::MsgFlags::empty(), None)?;
+    socket::sendmsg::<()>(socket_fd, &iov, &[cmsg], socket::MsgFlags::empty(), None)?;
 
     unistd::setsid()?;
     let ret = unsafe { libc::ioctl(pseudo.slave, libc::TIOCSCTTY) };

src/agent/rustjail/src/container.rs

@@ -6,7 +6,7 @@
 use anyhow::{anyhow, Context, Result};
 use libc::pid_t;
 use oci::{ContainerState, LinuxDevice, LinuxIdMapping};
-use oci::{Hook, Linux, LinuxNamespace, LinuxResources, Spec};
+use oci::{Linux, LinuxNamespace, LinuxResources, Spec};
 use std::clone::Clone;
 use std::ffi::CString;
 use std::fmt::Display;
@@ -22,6 +22,7 @@ use crate::capabilities;
 use crate::cgroups::fs::Manager as FsManager;
 #[cfg(test)]
 use crate::cgroups::mock::Manager as FsManager;
+use crate::cgroups::systemd::manager::Manager as SystemdManager;
 use crate::cgroups::Manager;
 #[cfg(feature = "standard-oci-runtime")]
 use crate::console;
@@ -29,6 +30,7 @@ use crate::log_child;
 use crate::process::Process;
 #[cfg(feature = "seccomp")]
 use crate::seccomp;
+use crate::selinux;
 use crate::specconv::CreateOpts;
 use crate::{mount, validator};
 
@@ -46,9 +48,10 @@ use nix::unistd::{self, fork, ForkResult, Gid, Pid, Uid, User};
 use std::os::unix::fs::MetadataExt;
 use std::os::unix::io::AsRawFd;
 
-use protobuf::SingularPtrField;
+use protobuf::MessageField;
 
 use oci::State as OCIState;
+use regex::Regex;
 use std::collections::HashMap;
 use std::os::unix::io::FromRawFd;
 use std::str::FromStr;
@@ -64,6 +67,9 @@ use rlimit::{setrlimit, Resource, Rlim};
 use tokio::io::AsyncBufReadExt;
 use tokio::sync::Mutex;
 
+use kata_sys_util::hooks::HookStates;
+use kata_sys_util::validate::valid_env;
+
 pub const EXEC_FIFO_FILENAME: &str = "exec.fifo";
 
 const INIT: &str = "INIT";
@@ -107,7 +113,6 @@ impl Default for ContainerStatus {
 }
 
 // We might want to change this to thiserror in the future
-const MissingCGroupManager: &str = "failed to get container's cgroup Manager";
 const MissingLinux: &str = "no linux config";
 const InvalidNamespace: &str = "invalid namespace type";
 
@@ -201,6 +206,8 @@ lazy_static! {
             },
         ]
     };
+
+    pub static ref SYSTEMD_CGROUP_PATH_FORMAT:Regex = Regex::new(r"^[\w\-.]*:[\w\-.]*:[\w\-.]*$").unwrap();
 }
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -239,7 +246,7 @@ pub struct LinuxContainer {
     pub id: String,
     pub root: String,
     pub config: Config,
-    pub cgroup_manager: Option<FsManager>,
+    pub cgroup_manager: Box<dyn Manager + Send + Sync>,
     pub init_process_pid: pid_t,
     pub init_process_start_time: u64,
     pub uid_map_path: String,
@@ -288,16 +295,11 @@ impl Container for LinuxContainer {
             ));
         }
 
-        if self.cgroup_manager.is_some() {
-            self.cgroup_manager
-                .as_ref()
-                .unwrap()
-                .freeze(FreezerState::Frozen)?;
+        self.cgroup_manager.as_ref().freeze(FreezerState::Frozen)?;
 
-            self.status.transition(ContainerState::Paused);
-            return Ok(());
-        }
-        Err(anyhow!(MissingCGroupManager))
+        self.status.transition(ContainerState::Paused);
+
+        Ok(())
     }
 
     fn resume(&mut self) -> Result<()> {
@@ -306,16 +308,11 @@ impl Container for LinuxContainer {
             return Err(anyhow!("container status is: {:?}, not paused", status));
         }
 
-        if self.cgroup_manager.is_some() {
-            self.cgroup_manager
-                .as_ref()
-                .unwrap()
-                .freeze(FreezerState::Thawed)?;
+        self.cgroup_manager.as_ref().freeze(FreezerState::Thawed)?;
 
-            self.status.transition(ContainerState::Running);
-            return Ok(());
-        }
-        Err(anyhow!(MissingCGroupManager))
+        self.status.transition(ContainerState::Running);
+
+        Ok(())
     }
 }
 
@@ -377,20 +374,27 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     let buf = read_sync(crfd)?;
     let spec_str = std::str::from_utf8(&buf)?;
     let spec: oci::Spec = serde_json::from_str(spec_str)?;
-
     log_child!(cfd_log, "notify parent to send oci process");
     write_sync(cwfd, SYNC_SUCCESS, "")?;
 
     let buf = read_sync(crfd)?;
     let process_str = std::str::from_utf8(&buf)?;
     let oci_process: oci::Process = serde_json::from_str(process_str)?;
+    log_child!(cfd_log, "notify parent to send oci state");
+    write_sync(cwfd, SYNC_SUCCESS, "")?;
+
+    let buf = read_sync(crfd)?;
+    let state_str = std::str::from_utf8(&buf)?;
+    let mut state: oci::State = serde_json::from_str(state_str)?;
     log_child!(cfd_log, "notify parent to send cgroup manager");
     write_sync(cwfd, SYNC_SUCCESS, "")?;
 
     let buf = read_sync(crfd)?;
     let cm_str = std::str::from_utf8(&buf)?;
 
-    let cm: FsManager = serde_json::from_str(cm_str)?;
+    // deserialize cm_str into FsManager and SystemdManager separately
+    let fs_cm: Result<FsManager, serde_json::Error> = serde_json::from_str(cm_str);
+    let systemd_cm: Result<SystemdManager, serde_json::Error> = serde_json::from_str(cm_str);
 
     #[cfg(feature = "standard-oci-runtime")]
     let csocket_fd = console::setup_console_socket(&std::env::var(CONSOLE_SOCKET_FD)?)?;
@@ -531,6 +535,8 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         }
     }
 
+    let selinux_enabled = selinux::is_enabled()?;
+
     sched::unshare(to_new & !CloneFlags::CLONE_NEWUSER)?;
 
     if userns {
@@ -548,7 +554,18 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
 
     if to_new.contains(CloneFlags::CLONE_NEWNS) {
         // setup rootfs
-        mount::init_rootfs(cfd_log, &spec, &cm.paths, &cm.mounts, bind_device)?;
+        if let Ok(systemd_cm) = systemd_cm {
+            mount::init_rootfs(
+                cfd_log,
+                &spec,
+                &systemd_cm.paths,
+                &systemd_cm.mounts,
+                bind_device,
+            )?;
+        } else {
+            let fs_cm = fs_cm.unwrap();
+            mount::init_rootfs(cfd_log, &spec, &fs_cm.paths, &fs_cm.mounts, bind_device)?;
+        }
     }
 
     if init {
@@ -621,6 +638,18 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         capctl::prctl::set_no_new_privs().map_err(|_| anyhow!("cannot set no new privileges"))?;
     }
 
+    // Set SELinux label
+    if !oci_process.selinux_label.is_empty() {
+        if !selinux_enabled {
+            return Err(anyhow!(
+                "SELinux label for the process is provided but SELinux is not enabled on the running kernel"
+            ));
+        }
+
+        log_child!(cfd_log, "Set SELinux label to the container process");
+        selinux::set_exec_label(&oci_process.selinux_label)?;
+    }
+
     // Log unknown seccomp system calls in advance before the log file descriptor closes.
     #[cfg(feature = "seccomp")]
     if let Some(ref scmp) = linux.seccomp {
@@ -719,6 +748,19 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         unistd::read(fd, buf)?;
     }
 
+    if init {
+        // StartContainer Hooks:
+        // * should be run in container namespace
+        // * should be run after container is created and before container is started (before user-specific command is executed)
+        // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#startcontainer-hooks
+        state.pid = std::process::id() as i32;
+        state.status = oci::ContainerState::Created;
+        if let Some(hooks) = spec.hooks.as_ref() {
+            let mut start_container_states = HookStates::new();
+            start_container_states.execute_hooks(&hooks.start_container, Some(state))?;
+        }
+    }
+
     // With NoNewPrivileges, we should set seccomp as close to
     // do_exec as possible in order to reduce the amount of
     // system calls in the seccomp profiles.
@@ -830,22 +872,17 @@ impl BaseContainer for LinuxContainer {
     }
 
     fn stats(&self) -> Result<StatsContainerResponse> {
-        let mut r = StatsContainerResponse::default();
-
-        if self.cgroup_manager.is_some() {
-            r.cgroup_stats =
-                SingularPtrField::some(self.cgroup_manager.as_ref().unwrap().get_stats()?);
-        }
-
         // what about network interface stats?
 
-        Ok(r)
+        Ok(StatsContainerResponse {
+            cgroup_stats: MessageField::some(self.cgroup_manager.as_ref().get_stats()?),
+            ..Default::default()
+        })
     }
 
     fn set(&mut self, r: LinuxResources) -> Result<()> {
-        if self.cgroup_manager.is_some() {
-            self.cgroup_manager.as_ref().unwrap().set(&r, true)?;
-        }
+        self.cgroup_manager.as_ref().set(&r, true)?;
+
         self.config
             .spec
             .as_mut()
@@ -1018,7 +1055,8 @@ impl BaseContainer for LinuxContainer {
             &logger,
             spec,
             &p,
-            self.cgroup_manager.as_ref().unwrap(),
+            self.cgroup_manager.as_ref(),
+            self.config.use_systemd_cgroup,
             &st,
             &mut pipe_w,
             &mut pipe_r,
@@ -1081,12 +1119,14 @@ impl BaseContainer for LinuxContainer {
             }
         }
 
-        if spec.hooks.is_some() {
-            info!(self.logger, "poststop");
-            let hooks = spec.hooks.as_ref().unwrap();
-            for h in hooks.poststop.iter() {
-                execute_hook(&self.logger, h, &st).await?;
-            }
+        // guest Poststop hook
+        // * should be executed after the container is deleted but before the delete operation returns
+        // * the executable file is in agent namespace
+        // * should also be executed in agent namespace.
+        if let Some(hooks) = spec.hooks.as_ref() {
+            info!(self.logger, "guest Poststop hook");
+            let mut hook_states = HookStates::new();
+            hook_states.execute_hooks(&hooks.poststop, Some(st))?;
         }
 
         self.status.transition(ContainerState::Stopped);
@@ -1096,19 +1136,19 @@ impl BaseContainer for LinuxContainer {
         )?;
         fs::remove_dir_all(&self.root)?;
 
-        if let Some(cgm) = self.cgroup_manager.as_mut() {
-            // Kill all of the processes created in this container to prevent
-            // the leak of some daemon process when this container shared pidns
-            // with the sandbox.
-            let pids = cgm.get_pids().context("get cgroup pids")?;
-            for i in pids {
-                if let Err(e) = signal::kill(Pid::from_raw(i), Signal::SIGKILL) {
-                    warn!(self.logger, "kill the process {} error: {:?}", i, e);
-                }
+        let cgm = self.cgroup_manager.as_mut();
+        // Kill all of the processes created in this container to prevent
+        // the leak of some daemon process when this container shared pidns
+        // with the sandbox.
+        let pids = cgm.get_pids().context("get cgroup pids")?;
+        for i in pids {
+            if let Err(e) = signal::kill(Pid::from_raw(i), Signal::SIGKILL) {
+                warn!(self.logger, "kill the process {} error: {:?}", i, e);
             }
-
-            cgm.destroy().context("destroy cgroups")?;
         }
+
+        cgm.destroy().context("destroy cgroups")?;
+
         Ok(())
     }
 
@@ -1132,16 +1172,14 @@ impl BaseContainer for LinuxContainer {
             .ok_or_else(|| anyhow!("OCI spec was not found"))?;
         let st = self.oci_state()?;
 
-        // run poststart hook
-        if spec.hooks.is_some() {
-            info!(self.logger, "poststart hook");
-            let hooks = spec
-                .hooks
-                .as_ref()
-                .ok_or_else(|| anyhow!("OCI hooks were not found"))?;
-            for h in hooks.poststart.iter() {
-                execute_hook(&self.logger, h, &st).await?;
-            }
+        // guest Poststart hook
+        // * should be executed after the container is started but before the delete operation returns
+        // * the executable file is in agent namespace
+        // * should also be executed in agent namespace.
+        if let Some(hooks) = spec.hooks.as_ref() {
+            info!(self.logger, "guest Poststart hook");
+            let mut hook_states = HookStates::new();
+            hook_states.execute_hooks(&hooks.poststart, Some(st))?;
         }
 
         unistd::close(fd)?;
@@ -1280,11 +1318,13 @@ pub fn setup_child_logger(fd: RawFd, child_logger: Logger) -> tokio::task::JoinH
     })
 }
 
+#[allow(clippy::too_many_arguments)]
 async fn join_namespaces(
     logger: &Logger,
     spec: &Spec,
     p: &Process,
-    cm: &FsManager,
+    cm: &(dyn Manager + Send + Sync),
+    use_systemd_cgroup: bool,
     st: &OCIState,
     pipe_w: &mut PipeStream,
     pipe_r: &mut PipeStream,
@@ -1301,7 +1341,6 @@ async fn join_namespaces(
     write_async(pipe_w, SYNC_DATA, spec_str.as_str()).await?;
 
     info!(logger, "wait child received oci spec");
-
     read_async(pipe_r).await?;
 
     info!(logger, "send oci process from parent to child");
@@ -1311,7 +1350,18 @@ async fn join_namespaces(
     info!(logger, "wait child received oci process");
     read_async(pipe_r).await?;
 
-    let cm_str = serde_json::to_string(cm)?;
+    info!(logger, "try to send state from parent to child");
+    let state_str = serde_json::to_string(st)?;
+    write_async(pipe_w, SYNC_DATA, state_str.as_str()).await?;
+
+    info!(logger, "wait child received oci state");
+    read_async(pipe_r).await?;
+
+    let cm_str = if use_systemd_cgroup {
+        serde_json::to_string(cm.as_any()?.downcast_ref::<SystemdManager>().unwrap())
+    } else {
+        serde_json::to_string(cm.as_any()?.downcast_ref::<FsManager>().unwrap())
+    }?;
     write_async(pipe_w, SYNC_DATA, cm_str.as_str()).await?;
 
     // wait child setup user namespace
@@ -1334,13 +1384,16 @@ async fn join_namespaces(
     }
 
     // apply cgroups
-    if p.init && res.is_some() {
-        info!(logger, "apply cgroups!");
-        cm.set(res.unwrap(), false)?;
+    // For FsManger, it's no matter about the order of apply and set.
+    // For SystemdManger, apply must be precede set because we can only create a systemd unit with specific processes(pids).
+    if res.is_some() {
+        info!(logger, "apply processes to cgroups!");
+        cm.apply(p.pid)?;
     }
 
-    if res.is_some() {
-        cm.apply(p.pid)?;
+    if p.init && res.is_some() {
+        info!(logger, "set properties to cgroups!");
+        cm.set(res.unwrap(), false)?;
     }
 
     info!(logger, "notify child to continue");
@@ -1353,13 +1406,14 @@ async fn join_namespaces(
 
         info!(logger, "get ready to run prestart hook!");
 
-        // run prestart hook
-        if spec.hooks.is_some() {
-            info!(logger, "prestart hook");
-            let hooks = spec.hooks.as_ref().unwrap();
-            for h in hooks.prestart.iter() {
-                execute_hook(&logger, h, st).await?;
-            }
+        // guest Prestart hook
+        // * should be executed during the start operation, and before the container command is executed
+        // * the executable file is in agent namespace
+        // * should also be executed in agent namespace.
+        if let Some(hooks) = spec.hooks.as_ref() {
+            info!(logger, "guest Prestart hook");
+            let mut hook_states = HookStates::new();
+            hook_states.execute_hooks(&hooks.prestart, Some(st.clone()))?;
         }
 
         // notify child run prestart hooks completed
@@ -1445,27 +1499,41 @@ impl LinuxContainer {
         .context(format!("Cannot change owner of container {} root", id))?;
 
         let spec = config.spec.as_ref().unwrap();
-
         let linux = spec.linux.as_ref().unwrap();
-
-        let cpath = if linux.cgroups_path.is_empty() {
+        let cpath = if config.use_systemd_cgroup {
+            if linux.cgroups_path.len() == 2 {
+                format!("system.slice:kata_agent:{}", id.as_str())
+            } else {
+                linux.cgroups_path.clone()
+            }
+        } else if linux.cgroups_path.is_empty() {
             format!("/{}", id.as_str())
         } else {
-            linux.cgroups_path.clone()
+            // if we have a systemd cgroup path we need to convert it to a fs cgroup path
+            linux.cgroups_path.replace(':', "/")
         };
 
-        let cgroup_manager = FsManager::new(cpath.as_str()).map_err(|e| {
-            anyhow!(format!(
-                "fail to create cgroup manager with path {}: {:}",
-                cpath, e
-            ))
-        })?;
+        let cgroup_manager: Box<dyn Manager + Send + Sync> = if config.use_systemd_cgroup {
+            Box::new(SystemdManager::new(cpath.as_str()).map_err(|e| {
+                anyhow!(format!(
+                    "fail to create cgroup manager with path {}: {:}",
+                    cpath, e
+                ))
+            })?)
+        } else {
+            Box::new(FsManager::new(cpath.as_str()).map_err(|e| {
+                anyhow!(format!(
+                    "fail to create cgroup manager with path {}: {:}",
+                    cpath, e
+                ))
+            })?)
+        };
         info!(logger, "new cgroup_manager {:?}", &cgroup_manager);
 
         Ok(LinuxContainer {
             id: id.clone(),
             root,
-            cgroup_manager: Some(cgroup_manager),
+            cgroup_manager,
             status: ContainerStatus::new(),
             uid_map_path: String::from(""),
             gid_map_path: "".to_string(),
@@ -1517,143 +1585,6 @@ fn set_sysctls(sysctls: &HashMap<String, String>) -> Result<()> {
     Ok(())
 }
 
-use std::process::Stdio;
-use std::time::Duration;
-use tokio::io::{AsyncReadExt, AsyncWriteExt};
-
-pub async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
-    let logger = logger.new(o!("action" => "execute-hook"));
-
-    let binary = PathBuf::from(h.path.as_str());
-    let path = binary.canonicalize()?;
-    if !path.exists() {
-        return Err(anyhow!("Path {:?} does not exist", path));
-    }
-
-    let mut args = h.args.clone();
-    // the hook.args[0] is the hook binary name which shouldn't be included
-    // in the Command.args
-    if args.len() > 1 {
-        args.remove(0);
-    }
-
-    // all invalid envs will be omitted, only valid envs will be passed to hook.
-    let env: HashMap<&str, &str> = h.env.iter().filter_map(|e| valid_env(e)).collect();
-
-    // Avoid the exit signal to be reaped by the global reaper.
-    let _wait_locker = WAIT_PID_LOCKER.lock().await;
-    let mut child = tokio::process::Command::new(path)
-        .args(args.iter())
-        .envs(env.iter())
-        .kill_on_drop(true)
-        .stdin(Stdio::piped())
-        .stdout(Stdio::piped())
-        .stderr(Stdio::piped())
-        .spawn()?;
-
-    // default timeout 10s
-    let mut timeout: u64 = 10;
-
-    // if timeout is set if hook, then use the specified value
-    if let Some(t) = h.timeout {
-        if t > 0 {
-            timeout = t as u64;
-        }
-    }
-
-    let state = serde_json::to_string(st)?;
-    let path = h.path.clone();
-
-    let join_handle = tokio::spawn(async move {
-        if let Some(mut stdin) = child.stdin.take() {
-            match stdin.write_all(state.as_bytes()).await {
-                Ok(_) => {}
-                Err(e) => {
-                    info!(logger, "write to child stdin failed: {:?}", e);
-                }
-            }
-        }
-
-        // read something from stdout and stderr for debug
-        if let Some(stdout) = child.stdout.as_mut() {
-            let mut out = String::new();
-            match stdout.read_to_string(&mut out).await {
-                Ok(_) => {
-                    info!(logger, "child stdout: {}", out.as_str());
-                }
-                Err(e) => {
-                    info!(logger, "read from child stdout failed: {:?}", e);
-                }
-            }
-        }
-
-        let mut err = String::new();
-        if let Some(stderr) = child.stderr.as_mut() {
-            match stderr.read_to_string(&mut err).await {
-                Ok(_) => {
-                    info!(logger, "child stderr: {}", err.as_str());
-                }
-                Err(e) => {
-                    info!(logger, "read from child stderr failed: {:?}", e);
-                }
-            }
-        }
-
-        match child.wait().await {
-            Ok(exit) => {
-                let code = exit
-                    .code()
-                    .ok_or_else(|| anyhow!("hook exit status has no status code"))?;
-
-                if code != 0 {
-                    error!(
-                        logger,
-                        "hook {} exit status is {}, error message is {}", &path, code, err
-                    );
-                    return Err(anyhow!(nix::Error::UnknownErrno));
-                }
-
-                debug!(logger, "hook {} exit status is 0", &path);
-                Ok(())
-            }
-            Err(e) => Err(anyhow!(
-                "wait child error: {} {}",
-                e,
-                e.raw_os_error().unwrap()
-            )),
-        }
-    });
-
-    match tokio::time::timeout(Duration::new(timeout, 0), join_handle).await {
-        Ok(r) => r.unwrap(),
-        Err(_) => Err(anyhow!(nix::Error::ETIMEDOUT)),
-    }
-}
-
-// valid environment variables according to https://doc.rust-lang.org/std/env/fn.set_var.html#panics
-fn valid_env(e: &str) -> Option<(&str, &str)> {
-    // wherther key or value will contain NULL char.
-    if e.as_bytes().contains(&b'\0') {
-        return None;
-    }
-
-    let v: Vec<&str> = e.splitn(2, '=').collect();
-
-    // key can't hold an `equal` sign, but value can
-    if v.len() != 2 {
-        return None;
-    }
-
-    let (key, value) = (v[0].trim(), v[1].trim());
-
-    // key can't be empty
-    if key.is_empty() {
-        return None;
-    }
-
-    Some((key, value))
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1664,7 +1595,6 @@ mod tests {
     use std::os::unix::io::AsRawFd;
     use tempfile::tempdir;
     use test_utils::skip_if_not_root;
-    use tokio::process::Command;
 
     macro_rules! sl {
         () => {
@@ -1672,113 +1602,6 @@ mod tests {
         };
     }
 
-    async fn which(cmd: &str) -> String {
-        let output: std::process::Output = Command::new("which")
-            .arg(cmd)
-            .output()
-            .await
-            .expect("which command failed to run");
-
-        match String::from_utf8(output.stdout) {
-            Ok(v) => v.trim_end_matches('\n').to_string(),
-            Err(e) => panic!("Invalid UTF-8 sequence: {}", e),
-        }
-    }
-
-    #[tokio::test]
-    async fn test_execute_hook() {
-        let temp_file = "/tmp/test_execute_hook";
-
-        let touch = which("touch").await;
-
-        defer!(fs::remove_file(temp_file).unwrap(););
-        let invalid_str = vec![97, b'\0', 98];
-        let invalid_string = std::str::from_utf8(&invalid_str).unwrap();
-        let invalid_env = format!("{}=value", invalid_string);
-
-        execute_hook(
-            &slog_scope::logger(),
-            &Hook {
-                path: touch,
-                args: vec!["touch".to_string(), temp_file.to_string()],
-                env: vec![invalid_env],
-                timeout: Some(10),
-            },
-            &OCIState {
-                version: "1.2.3".to_string(),
-                id: "321".to_string(),
-                status: ContainerState::Running,
-                pid: 2,
-                bundle: "".to_string(),
-                annotations: Default::default(),
-            },
-        )
-        .await
-        .unwrap();
-
-        assert_eq!(Path::new(&temp_file).exists(), true);
-    }
-
-    #[tokio::test]
-    async fn test_execute_hook_with_error() {
-        let ls = which("ls").await;
-
-        let res = execute_hook(
-            &slog_scope::logger(),
-            &Hook {
-                path: ls,
-                args: vec!["ls".to_string(), "/tmp/not-exist".to_string()],
-                env: vec![],
-                timeout: None,
-            },
-            &OCIState {
-                version: "1.2.3".to_string(),
-                id: "321".to_string(),
-                status: ContainerState::Running,
-                pid: 2,
-                bundle: "".to_string(),
-                annotations: Default::default(),
-            },
-        )
-        .await;
-
-        let expected_err = nix::Error::UnknownErrno;
-        assert_eq!(
-            res.unwrap_err().downcast::<nix::Error>().unwrap(),
-            expected_err
-        );
-    }
-
-    #[tokio::test]
-    async fn test_execute_hook_with_timeout() {
-        let sleep = which("sleep").await;
-
-        let res = execute_hook(
-            &slog_scope::logger(),
-            &Hook {
-                path: sleep,
-                args: vec!["sleep".to_string(), "2".to_string()],
-                env: vec![],
-                timeout: Some(1),
-            },
-            &OCIState {
-                version: "1.2.3".to_string(),
-                id: "321".to_string(),
-                status: ContainerState::Running,
-                pid: 2,
-                bundle: "".to_string(),
-                annotations: Default::default(),
-            },
-        )
-        .await;
-
-        let expected_err = nix::Error::ETIMEDOUT;
-        assert_eq!(
-            res.unwrap_err().downcast::<nix::Error>().unwrap(),
-            expected_err
-        );
-    }
-
     #[test]
     fn test_status_transtition() {
         let mut status = ContainerStatus::new();
@@ -1931,20 +1754,12 @@ mod tests {
         assert!(format!("{:?}", ret).contains("failed to pause container"))
     }
 
-    #[test]
-    fn test_linuxcontainer_pause_cgroupmgr_is_none() {
-        let ret = new_linux_container_and_then(|mut c: LinuxContainer| {
-            c.cgroup_manager = None;
-            c.pause().map_err(|e| anyhow!(e))
-        });
-
-        assert!(ret.is_err(), "Expecting error, Got {:?}", ret);
-    }
-
     #[test]
     fn test_linuxcontainer_pause() {
         let ret = new_linux_container_and_then(|mut c: LinuxContainer| {
-            c.cgroup_manager = FsManager::new("").ok();
+            c.cgroup_manager = Box::new(FsManager::new("").map_err(|e| {
+                anyhow!(format!("fail to create cgroup manager with path: {:}", e))
+            })?);
             c.pause().map_err(|e| anyhow!(e))
         });
 
@@ -1963,21 +1778,12 @@ mod tests {
         assert!(format!("{:?}", ret).contains("not paused"))
     }
 
-    #[test]
-    fn test_linuxcontainer_resume_cgroupmgr_is_none() {
-        let ret = new_linux_container_and_then(|mut c: LinuxContainer| {
-            c.status.transition(ContainerState::Paused);
-            c.cgroup_manager = None;
-            c.resume().map_err(|e| anyhow!(e))
-        });
-
-        assert!(ret.is_err(), "Expecting error, Got {:?}", ret);
-    }
-
     #[test]
     fn test_linuxcontainer_resume() {
         let ret = new_linux_container_and_then(|mut c: LinuxContainer| {
-            c.cgroup_manager = FsManager::new("").ok();
+            c.cgroup_manager = Box::new(FsManager::new("").map_err(|e| {
+                anyhow!(format!("fail to create cgroup manager with path: {:}", e))
+            })?);
             // Change status to paused, this way we can resume it
             c.status.transition(ContainerState::Paused);
             c.resume().map_err(|e| anyhow!(e))
@@ -2110,49 +1916,4 @@ mod tests {
         let ret = do_init_child(std::io::stdin().as_raw_fd());
         assert!(ret.is_err(), "Expecting Err, Got {:?}", ret);
     }
-
-    #[test]
-    fn test_valid_env() {
-        let env = valid_env("a=b=c");
-        assert_eq!(Some(("a", "b=c")), env);
-
-        let env = valid_env("a=b");
-        assert_eq!(Some(("a", "b")), env);
-        let env = valid_env("a =b");
-        assert_eq!(Some(("a", "b")), env);
-
-        let env = valid_env(" a =b");
-        assert_eq!(Some(("a", "b")), env);
-
-        let env = valid_env("a= b");
-        assert_eq!(Some(("a", "b")), env);
-
-        let env = valid_env("a=b ");
-        assert_eq!(Some(("a", "b")), env);
-        let env = valid_env("a=b c ");
-        assert_eq!(Some(("a", "b c")), env);
-
-        let env = valid_env("=b");
-        assert_eq!(None, env);
-
-        let env = valid_env("a=");
-        assert_eq!(Some(("a", "")), env);
-
-        let env = valid_env("a==");
-        assert_eq!(Some(("a", "=")), env);
-
-        let env = valid_env("a");
-        assert_eq!(None, env);
-
-        let invalid_str = vec![97, b'\0', 98];
-        let invalid_string = std::str::from_utf8(&invalid_str).unwrap();
-
-        let invalid_env = format!("{}=value", invalid_string);
-        let env = valid_env(&invalid_env);
-        assert_eq!(None, env);
-
-        let invalid_env = format!("key={}", invalid_string);
-        let env = valid_env(&invalid_env);
-        assert_eq!(None, env);
-    }
 }

src/agent/rustjail/src/lib.rs

@@ -38,6 +38,7 @@ pub mod pipestream;
 pub mod process;
 #[cfg(feature = "seccomp")]
 pub mod seccomp;
+pub mod selinux;
 pub mod specconv;
 pub mod sync;
 pub mod sync_with_async;
@@ -81,11 +82,11 @@ pub fn process_grpc_to_oci(p: &grpc::Process) -> oci::Process {
         let cap = p.Capabilities.as_ref().unwrap();
 
         Some(oci::LinuxCapabilities {
-            bounding: cap.Bounding.clone().into_vec(),
-            effective: cap.Effective.clone().into_vec(),
-            inheritable: cap.Inheritable.clone().into_vec(),
-            permitted: cap.Permitted.clone().into_vec(),
-            ambient: cap.Ambient.clone().into_vec(),
+            bounding: cap.Bounding.clone(),
+            effective: cap.Effective.clone(),
+            inheritable: cap.Inheritable.clone(),
+            permitted: cap.Permitted.clone(),
+            ambient: cap.Ambient.clone(),
         })
     } else {
         None
@@ -107,8 +108,8 @@ pub fn process_grpc_to_oci(p: &grpc::Process) -> oci::Process {
         terminal: p.Terminal,
         console_size,
         user,
-        args: p.Args.clone().into_vec(),
-        env: p.Env.clone().into_vec(),
+        args: p.Args.clone(),
+        env: p.Env.clone(),
         cwd: p.Cwd.clone(),
         capabilities,
         rlimits,
@@ -129,9 +130,9 @@ fn root_grpc_to_oci(root: &grpc::Root) -> oci::Root {
 fn mount_grpc_to_oci(m: &grpc::Mount) -> oci::Mount {
     oci::Mount {
         destination: m.destination.clone(),
-        r#type: m.field_type.clone(),
+        r#type: m.type_.clone(),
         source: m.source.clone(),
-        options: m.options.clone().into_vec(),
+        options: m.options.clone(),
     }
 }
 
@@ -142,8 +143,8 @@ fn hook_grpc_to_oci(h: &[grpcHook]) -> Vec<oci::Hook> {
     for e in h.iter() {
         r.push(oci::Hook {
             path: e.Path.clone(),
-            args: e.Args.clone().into_vec(),
-            env: e.Env.clone().into_vec(),
+            args: e.Args.clone(),
+            env: e.Env.clone(),
             timeout: Some(e.Timeout as i32),
         });
     }
@@ -152,13 +153,17 @@ fn hook_grpc_to_oci(h: &[grpcHook]) -> Vec<oci::Hook> {
 
 fn hooks_grpc_to_oci(h: &grpc::Hooks) -> oci::Hooks {
     let prestart = hook_grpc_to_oci(h.Prestart.as_ref());
-
+    let create_runtime = hook_grpc_to_oci(h.CreateRuntime.as_ref());
+    let create_container = hook_grpc_to_oci(h.CreateContainer.as_ref());
+    let start_container = hook_grpc_to_oci(h.StartContainer.as_ref());
     let poststart = hook_grpc_to_oci(h.Poststart.as_ref());
-
     let poststop = hook_grpc_to_oci(h.Poststop.as_ref());
 
     oci::Hooks {
         prestart,
+        create_runtime,
+        create_container,
+        start_container,
         poststart,
         poststop,
     }
@@ -354,7 +359,7 @@ fn seccomp_grpc_to_oci(sec: &grpc::LinuxSeccomp) -> oci::LinuxSeccomp {
             let mut args = Vec::new();
 
             let errno_ret: u32 = if sys.has_errnoret() {
-                sys.get_errnoret()
+                sys.errnoret()
             } else {
                 libc::EPERM as u32
             };
@@ -369,7 +374,7 @@ fn seccomp_grpc_to_oci(sec: &grpc::LinuxSeccomp) -> oci::LinuxSeccomp {
             }
 
             r.push(oci::LinuxSyscall {
-                names: sys.Names.clone().into_vec(),
+                names: sys.Names.clone(),
                 action: sys.Action.clone(),
                 errno_ret,
                 args,
@@ -380,8 +385,8 @@ fn seccomp_grpc_to_oci(sec: &grpc::LinuxSeccomp) -> oci::LinuxSeccomp {
 
     oci::LinuxSeccomp {
         default_action: sec.DefaultAction.clone(),
-        architectures: sec.Architectures.clone().into_vec(),
-        flags: sec.Flags.clone().into_vec(),
+        architectures: sec.Architectures.clone(),
+        flags: sec.Flags.clone(),
         syscalls,
     }
 }
@@ -451,8 +456,8 @@ fn linux_grpc_to_oci(l: &grpc::Linux) -> oci::Linux {
         devices,
         seccomp,
         rootfs_propagation: l.RootfsPropagation.clone(),
-        masked_paths: l.MaskedPaths.clone().into_vec(),
-        readonly_paths: l.ReadonlyPaths.clone().into_vec(),
+        masked_paths: l.MaskedPaths.clone(),
+        readonly_paths: l.ReadonlyPaths.clone(),
         mount_label: l.MountLabel.clone(),
         intel_rdt,
     }
@@ -553,35 +558,30 @@ mod tests {
                 // All fields specified
                 grpcproc: grpc::Process {
                     Terminal: true,
-                    ConsoleSize: protobuf::SingularPtrField::<grpc::Box>::some(grpc::Box {
+                    ConsoleSize: protobuf::MessageField::<grpc::Box>::some(grpc::Box {
                         Height: 123,
                         Width: 456,
                         ..Default::default()
                     }),
-                    User: protobuf::SingularPtrField::<grpc::User>::some(grpc::User {
+                    User: protobuf::MessageField::<grpc::User>::some(grpc::User {
                         UID: 1234,
                         GID: 5678,
                         AdditionalGids: Vec::from([910, 1112]),
                         Username: String::from("username"),
                         ..Default::default()
                     }),
-                    Args: protobuf::RepeatedField::from(Vec::from([
-                        String::from("arg1"),
-                        String::from("arg2"),
-                    ])),
-                    Env: protobuf::RepeatedField::from(Vec::from([String::from("env")])),
+                    Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                    Env: Vec::from([String::from("env")]),
                     Cwd: String::from("cwd"),
-                    Capabilities: protobuf::SingularPtrField::some(grpc::LinuxCapabilities {
-                        Bounding: protobuf::RepeatedField::from(Vec::from([String::from("bnd")])),
-                        Effective: protobuf::RepeatedField::from(Vec::from([String::from("eff")])),
-                        Inheritable: protobuf::RepeatedField::from(Vec::from([String::from(
-                            "inher",
-                        )])),
-                        Permitted: protobuf::RepeatedField::from(Vec::from([String::from("perm")])),
-                        Ambient: protobuf::RepeatedField::from(Vec::from([String::from("amb")])),
+                    Capabilities: protobuf::MessageField::some(grpc::LinuxCapabilities {
+                        Bounding: Vec::from([String::from("bnd")]),
+                        Effective: Vec::from([String::from("eff")]),
+                        Inheritable: Vec::from([String::from("inher")]),
+                        Permitted: Vec::from([String::from("perm")]),
+                        Ambient: Vec::from([String::from("amb")]),
                         ..Default::default()
                     }),
-                    Rlimits: protobuf::RepeatedField::from(Vec::from([
+                    Rlimits: Vec::from([
                         grpc::POSIXRlimit {
                             Type: String::from("r#type"),
                             Hard: 123,
@@ -594,7 +594,7 @@ mod tests {
                             Soft: 1011,
                             ..Default::default()
                         },
-                    ])),
+                    ]),
                     NoNewPrivileges: true,
                     ApparmorProfile: String::from("apparmor profile"),
                     OOMScoreAdj: 123456,
@@ -644,7 +644,7 @@ mod tests {
             TestData {
                 // None ConsoleSize
                 grpcproc: grpc::Process {
-                    ConsoleSize: protobuf::SingularPtrField::<grpc::Box>::none(),
+                    ConsoleSize: protobuf::MessageField::<grpc::Box>::none(),
                     OOMScoreAdj: 0,
                     ..Default::default()
                 },
@@ -657,7 +657,7 @@ mod tests {
             TestData {
                 // None User
                 grpcproc: grpc::Process {
-                    User: protobuf::SingularPtrField::<grpc::User>::none(),
+                    User: protobuf::MessageField::<grpc::User>::none(),
                     OOMScoreAdj: 0,
                     ..Default::default()
                 },
@@ -675,7 +675,7 @@ mod tests {
             TestData {
                 // None Capabilities
                 grpcproc: grpc::Process {
-                    Capabilities: protobuf::SingularPtrField::none(),
+                    Capabilities: protobuf::MessageField::none(),
                     OOMScoreAdj: 0,
                     ..Default::default()
                 },
@@ -776,60 +776,57 @@ mod tests {
             TestData {
                 // All specified
                 grpchooks: grpc::Hooks {
-                    Prestart: protobuf::RepeatedField::from(Vec::from([
+                    Prestart: Vec::from([
                         grpc::Hook {
                             Path: String::from("prestartpath"),
-                            Args: protobuf::RepeatedField::from(Vec::from([
-                                String::from("arg1"),
-                                String::from("arg2"),
-                            ])),
-                            Env: protobuf::RepeatedField::from(Vec::from([
-                                String::from("env1"),
-                                String::from("env2"),
-                            ])),
+                            Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                            Env: Vec::from([String::from("env1"), String::from("env2")]),
                             Timeout: 10,
                             ..Default::default()
                         },
                         grpc::Hook {
                             Path: String::from("prestartpath2"),
-                            Args: protobuf::RepeatedField::from(Vec::from([
-                                String::from("arg3"),
-                                String::from("arg4"),
-                            ])),
-                            Env: protobuf::RepeatedField::from(Vec::from([
-                                String::from("env3"),
-                                String::from("env4"),
-                            ])),
+                            Args: Vec::from([String::from("arg3"), String::from("arg4")]),
+                            Env: Vec::from([String::from("env3"), String::from("env4")]),
                             Timeout: 25,
                             ..Default::default()
                         },
-                    ])),
-                    Poststart: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                    ]),
+                    Poststart: Vec::from([grpc::Hook {
                         Path: String::from("poststartpath"),
-                        Args: protobuf::RepeatedField::from(Vec::from([
-                            String::from("arg1"),
-                            String::from("arg2"),
-                        ])),
-                        Env: protobuf::RepeatedField::from(Vec::from([
-                            String::from("env1"),
-                            String::from("env2"),
-                        ])),
+                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        Env: Vec::from([String::from("env1"), String::from("env2")]),
                         Timeout: 10,
                         ..Default::default()
-                    }])),
-                    Poststop: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                    }]),
+                    Poststop: Vec::from([grpc::Hook {
                         Path: String::from("poststoppath"),
-                        Args: protobuf::RepeatedField::from(Vec::from([
-                            String::from("arg1"),
-                            String::from("arg2"),
-                        ])),
-                        Env: protobuf::RepeatedField::from(Vec::from([
-                            String::from("env1"),
-                            String::from("env2"),
-                        ])),
+                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        Env: Vec::from([String::from("env1"), String::from("env2")]),
                         Timeout: 10,
                         ..Default::default()
-                    }])),
+                    }]),
+                    CreateRuntime: Vec::from([grpc::Hook {
+                        Path: String::from("createruntimepath"),
+                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        Env: Vec::from([String::from("env1"), String::from("env2")]),
+                        Timeout: 10,
+                        ..Default::default()
+                    }]),
+                    CreateContainer: Vec::from([grpc::Hook {
+                        Path: String::from("createcontainerpath"),
+                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        Env: Vec::from([String::from("env1"), String::from("env2")]),
+                        Timeout: 10,
+                        ..Default::default()
+                    }]),
+                    StartContainer: Vec::from([grpc::Hook {
+                        Path: String::from("startcontainerpath"),
+                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        Env: Vec::from([String::from("env1"), String::from("env2")]),
+                        Timeout: 10,
+                        ..Default::default()
+                    }]),
                     ..Default::default()
                 },
                 result: oci::Hooks {
@@ -859,38 +856,65 @@ mod tests {
                         env: Vec::from([String::from("env1"), String::from("env2")]),
                         timeout: Some(10),
                     }]),
+                    create_runtime: Vec::from([oci::Hook {
+                        path: String::from("createruntimepath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
+                    create_container: Vec::from([oci::Hook {
+                        path: String::from("createcontainerpath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
+                    start_container: Vec::from([oci::Hook {
+                        path: String::from("startcontainerpath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
                 },
             },
             TestData {
                 // Prestart empty
                 grpchooks: grpc::Hooks {
-                    Prestart: protobuf::RepeatedField::from(Vec::from([])),
-                    Poststart: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                    Prestart: Vec::from([]),
+                    Poststart: Vec::from([grpc::Hook {
                         Path: String::from("poststartpath"),
-                        Args: protobuf::RepeatedField::from(Vec::from([
-                            String::from("arg1"),
-                            String::from("arg2"),
-                        ])),
-                        Env: protobuf::RepeatedField::from(Vec::from([
-                            String::from("env1"),
-                            String::from("env2"),
-                        ])),
+                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        Env: Vec::from([String::from("env1"), String::from("env2")]),
                         Timeout: 10,
                         ..Default::default()
-                    }])),
-                    Poststop: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                    }]),
+                    Poststop: Vec::from([grpc::Hook {
                         Path: String::from("poststoppath"),
-                        Args: protobuf::RepeatedField::from(Vec::from([
-                            String::from("arg1"),
-                            String::from("arg2"),
-                        ])),
-                        Env: protobuf::RepeatedField::from(Vec::from([
-                            String::from("env1"),
-                            String::from("env2"),
-                        ])),
+                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        Env: Vec::from([String::from("env1"), String::from("env2")]),
                         Timeout: 10,
                         ..Default::default()
-                    }])),
+                    }]),
+                    CreateRuntime: Vec::from([grpc::Hook {
+                        Path: String::from("createruntimepath"),
+                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        Env: Vec::from([String::from("env1"), String::from("env2")]),
+                        Timeout: 10,
+                        ..Default::default()
+                    }]),
+                    CreateContainer: Vec::from([grpc::Hook {
+                        Path: String::from("createcontainerpath"),
+                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        Env: Vec::from([String::from("env1"), String::from("env2")]),
+                        Timeout: 10,
+                        ..Default::default()
+                    }]),
+                    StartContainer: Vec::from([grpc::Hook {
+                        Path: String::from("startcontainerpath"),
+                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        Env: Vec::from([String::from("env1"), String::from("env2")]),
+                        Timeout: 10,
+                        ..Default::default()
+                    }]),
                     ..Default::default()
                 },
                 result: oci::Hooks {
@@ -907,6 +931,24 @@ mod tests {
                         env: Vec::from([String::from("env1"), String::from("env2")]),
                         timeout: Some(10),
                     }]),
+                    create_runtime: Vec::from([oci::Hook {
+                        path: String::from("createruntimepath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
+                    create_container: Vec::from([oci::Hook {
+                        path: String::from("createcontainerpath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
+                    start_container: Vec::from([oci::Hook {
+                        path: String::from("startcontainerpath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
                 },
             },
         ];
@@ -944,11 +986,8 @@ mod tests {
                 grpcmount: grpc::Mount {
                     destination: String::from("destination"),
                     source: String::from("source"),
-                    field_type: String::from("fieldtype"),
-                    options: protobuf::RepeatedField::from(Vec::from([
-                        String::from("option1"),
-                        String::from("option2"),
-                    ])),
+                    type_: String::from("fieldtype"),
+                    options: Vec::from([String::from("option1"), String::from("option2")]),
                     ..Default::default()
                 },
                 result: oci::Mount {
@@ -962,8 +1001,8 @@ mod tests {
                 grpcmount: grpc::Mount {
                     destination: String::from("destination"),
                     source: String::from("source"),
-                    field_type: String::from("fieldtype"),
-                    options: protobuf::RepeatedField::from(Vec::new()),
+                    type_: String::from("fieldtype"),
+                    options: Vec::new(),
                     ..Default::default()
                 },
                 result: oci::Mount {
@@ -977,8 +1016,8 @@ mod tests {
                 grpcmount: grpc::Mount {
                     destination: String::new(),
                     source: String::from("source"),
-                    field_type: String::from("fieldtype"),
-                    options: protobuf::RepeatedField::from(Vec::from([String::from("option1")])),
+                    type_: String::from("fieldtype"),
+                    options: Vec::from([String::from("option1")]),
                     ..Default::default()
                 },
                 result: oci::Mount {
@@ -992,8 +1031,8 @@ mod tests {
                 grpcmount: grpc::Mount {
                     destination: String::from("destination"),
                     source: String::from("source"),
-                    field_type: String::new(),
-                    options: protobuf::RepeatedField::from(Vec::from([String::from("option1")])),
+                    type_: String::new(),
+                    options: Vec::from([String::from("option1")]),
                     ..Default::default()
                 },
                 result: oci::Mount {
@@ -1053,27 +1092,15 @@ mod tests {
                 grpchook: &[
                     grpc::Hook {
                         Path: String::from("path"),
-                        Args: protobuf::RepeatedField::from(Vec::from([
-                            String::from("arg1"),
-                            String::from("arg2"),
-                        ])),
-                        Env: protobuf::RepeatedField::from(Vec::from([
-                            String::from("env1"),
-                            String::from("env2"),
-                        ])),
+                        Args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        Env: Vec::from([String::from("env1"), String::from("env2")]),
                         Timeout: 10,
                         ..Default::default()
                     },
                     grpc::Hook {
                         Path: String::from("path2"),
-                        Args: protobuf::RepeatedField::from(Vec::from([
-                            String::from("arg3"),
-                            String::from("arg4"),
-                        ])),
-                        Env: protobuf::RepeatedField::from(Vec::from([
-                            String::from("env3"),
-                            String::from("env4"),
-                        ])),
+                        Args: Vec::from([String::from("arg3"), String::from("arg4")]),
+                        Env: Vec::from([String::from("env3"), String::from("env4")]),
                         Timeout: 20,
                         ..Default::default()
                     },

src/agent/rustjail/src/mount.rs

@@ -25,6 +25,7 @@ use std::fs::File;
 use std::io::{BufRead, BufReader};
 
 use crate::container::DEFAULT_DEVICES;
+use crate::selinux;
 use crate::sync::write_count;
 use std::string::ToString;
 
@@ -34,7 +35,7 @@ use crate::log_child;
 // struct is populated from the content in the /proc/<pid>/mountinfo file.
 #[derive(std::fmt::Debug, PartialEq)]
 pub struct Info {
-    mount_point: String,
+    pub mount_point: String,
     optional: String,
     fstype: String,
 }
@@ -181,6 +182,8 @@ pub fn init_rootfs(
         None => flags |= MsFlags::MS_SLAVE,
     }
 
+    let label = &linux.mount_label;
+
     let root = spec
         .root
         .as_ref()
@@ -244,7 +247,7 @@ pub fn init_rootfs(
                 }
             }
 
-            mount_from(cfd_log, m, rootfs, flags, &data, "")?;
+            mount_from(cfd_log, m, rootfs, flags, &data, label)?;
             // bind mount won't change mount options, we need remount to make mount options
             // effective.
             // first check that we have non-default options required before attempting a
@@ -524,7 +527,6 @@ pub fn pivot_rootfs<P: ?Sized + NixPath + std::fmt::Debug>(path: &P) -> Result<(
 
 fn rootfs_parent_mount_private(path: &str) -> Result<()> {
     let mount_infos = parse_mount_table(MOUNTINFO_PATH)?;
-
     let mut max_len = 0;
     let mut mount_point = String::from("");
     let mut options = String::from("");
@@ -551,7 +553,7 @@ fn rootfs_parent_mount_private(path: &str) -> Result<()> {
 
 // Parse /proc/self/mountinfo because comparing Dev and ino does not work from
 // bind mounts
-fn parse_mount_table(mountinfo_path: &str) -> Result<Vec<Info>> {
+pub fn parse_mount_table(mountinfo_path: &str) -> Result<Vec<Info>> {
     let file = File::open(mountinfo_path)?;
     let reader = BufReader::new(file);
     let mut infos = Vec::new();
@@ -767,9 +769,9 @@ fn mount_from(
     rootfs: &str,
     flags: MsFlags,
     data: &str,
-    _label: &str,
+    label: &str,
 ) -> Result<()> {
-    let d = String::from(data);
+    let mut d = String::from(data);
     let dest = secure_join(rootfs, &m.destination);
 
     let src = if m.r#type.as_str() == "bind" {
@@ -780,7 +782,7 @@ fn mount_from(
             Path::new(&dest).parent().unwrap()
         };
 
-        fs::create_dir_all(&dir).map_err(|e| {
+        fs::create_dir_all(dir).map_err(|e| {
             log_child!(
                 cfd_log,
                 "create dir {}: {}",
@@ -822,6 +824,37 @@ fn mount_from(
         e
     })?;
 
+    // Set the SELinux context for the mounts
+    let mut use_xattr = false;
+    if !label.is_empty() {
+        if selinux::is_enabled()? {
+            let device = Path::new(&m.source)
+                .file_name()
+                .ok_or_else(|| anyhow!("invalid device source path: {}", &m.source))?
+                .to_str()
+                .ok_or_else(|| anyhow!("failed to convert device source path: {}", &m.source))?;
+
+            match device {
+                // SELinux does not support labeling of /proc or /sys
+                "proc" | "sysfs" => (),
+                // SELinux does not support mount labeling against /dev/mqueue,
+                // so we use setxattr instead
+                "mqueue" => {
+                    use_xattr = true;
+                }
+                _ => {
+                    log_child!(cfd_log, "add SELinux mount label to {}", dest.as_str());
+                    selinux::add_mount_label(&mut d, label);
+                }
+            }
+        } else {
+            log_child!(
+                cfd_log,
+                "SELinux label for the mount is provided but SELinux is not enabled on the running kernel"
+            );
+        }
+    }
+
     mount(
         Some(src.as_str()),
         dest.as_str(),
@@ -834,6 +867,10 @@ fn mount_from(
         e
     })?;
 
+    if !label.is_empty() && selinux::is_enabled()? && use_xattr {
+        xattr::set(dest.as_str(), "security.selinux", label.as_bytes())?;
+    }
+
     if flags.contains(MsFlags::MS_BIND)
         && flags.intersects(
             !(MsFlags::MS_REC

src/agent/rustjail/src/seccomp.rs

@@ -63,7 +63,7 @@ pub fn get_unknown_syscalls(scmp: &LinuxSeccomp) -> Option<Vec<String>> {
 // init_seccomp creates a seccomp filter and loads it for the current process
 // including all the child processes.
 pub fn init_seccomp(scmp: &LinuxSeccomp) -> Result<()> {
-    let def_action = ScmpAction::from_str(scmp.default_action.as_str(), Some(libc::EPERM as i32))?;
+    let def_action = ScmpAction::from_str(scmp.default_action.as_str(), Some(libc::EPERM))?;
 
     // Create a new filter context
     let mut filter = ScmpFilterContext::new_filter(def_action)?;

src/agent/rustjail/src/selinux.rs

@@ -0,0 +1,80 @@
+// Copyright 2022 Sony Group Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use anyhow::{Context, Result};
+use nix::unistd::gettid;
+use std::fs::{self, OpenOptions};
+use std::io::prelude::*;
+use std::path::Path;
+
+pub fn is_enabled() -> Result<bool> {
+    let buf = fs::read_to_string("/proc/mounts")?;
+    let enabled = buf.contains("selinuxfs");
+
+    Ok(enabled)
+}
+
+pub fn add_mount_label(data: &mut String, label: &str) {
+    if data.is_empty() {
+        let context = format!("context=\"{}\"", label);
+        data.push_str(&context);
+    } else {
+        let context = format!(",context=\"{}\"", label);
+        data.push_str(&context);
+    }
+}
+
+pub fn set_exec_label(label: &str) -> Result<()> {
+    let mut attr_path = Path::new("/proc/thread-self/attr/exec").to_path_buf();
+    if !attr_path.exists() {
+        // Fall back to the old convention
+        attr_path = Path::new("/proc/self/task")
+            .join(gettid().to_string())
+            .join("attr/exec")
+    }
+
+    let mut file = OpenOptions::new()
+        .write(true)
+        .truncate(true)
+        .open(attr_path)?;
+    file.write_all(label.as_bytes())
+        .with_context(|| "failed to apply SELinux label")?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const TEST_LABEL: &str = "system_u:system_r:unconfined_t:s0";
+
+    #[test]
+    fn test_is_enabled() {
+        let ret = is_enabled();
+        assert!(ret.is_ok(), "Expecting Ok, Got {:?}", ret);
+    }
+
+    #[test]
+    fn test_add_mount_label() {
+        let mut data = String::new();
+        add_mount_label(&mut data, TEST_LABEL);
+        assert_eq!(data, format!("context=\"{}\"", TEST_LABEL));
+
+        let mut data = String::from("defaults");
+        add_mount_label(&mut data, TEST_LABEL);
+        assert_eq!(data, format!("defaults,context=\"{}\"", TEST_LABEL));
+    }
+
+    #[test]
+    fn test_set_exec_label() {
+        let ret = set_exec_label(TEST_LABEL);
+        if is_enabled().unwrap() {
+            assert!(ret.is_ok(), "Expecting Ok, Got {:?}", ret);
+        } else {
+            assert!(ret.is_err(), "Expecting error, Got {:?}", ret);
+        }
+    }
+}

src/agent/rustjail/src/validator.rs

@@ -6,6 +6,7 @@
 use crate::container::Config;
 use anyhow::{anyhow, Context, Result};
 use oci::{Linux, LinuxIdMapping, LinuxNamespace, Spec};
+use regex::Regex;
 use std::collections::HashMap;
 use std::path::{Component, PathBuf};
 
@@ -86,6 +87,23 @@ fn hostname(oci: &Spec) -> Result<()> {
 
 fn security(oci: &Spec) -> Result<()> {
     let linux = get_linux(oci)?;
+    let label_pattern = r".*_u:.*_r:.*_t:s[0-9]|1[0-5].*";
+    let label_regex = Regex::new(label_pattern)?;
+
+    if let Some(ref process) = oci.process {
+        if !process.selinux_label.is_empty() && !label_regex.is_match(&process.selinux_label) {
+            return Err(anyhow!(
+                "SELinux label for the process is invalid format: {}",
+                &process.selinux_label
+            ));
+        }
+    }
+    if !linux.mount_label.is_empty() && !label_regex.is_match(&linux.mount_label) {
+        return Err(anyhow!(
+            "SELinux label for the mount is invalid format: {}",
+            &linux.mount_label
+        ));
+    }
 
     if linux.masked_paths.is_empty() && linux.readonly_paths.is_empty() {
         return Ok(());
@@ -95,8 +113,6 @@ fn security(oci: &Spec) -> Result<()> {
         return Err(anyhow!("Linux namespace does not contain mount"));
     }
 
-    // don't care about selinux at present
-
     Ok(())
 }
 
@@ -285,7 +301,7 @@ pub fn validate(conf: &Config) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use oci::Mount;
+    use oci::{Mount, Process};
 
     #[test]
     fn test_namespace() {
@@ -388,6 +404,29 @@ mod tests {
         ];
         spec.linux = Some(linux);
         security(&spec).unwrap();
+
+        // SELinux
+        let valid_label = "system_u:system_r:container_t:s0:c123,c456";
+        let mut process = Process::default();
+        process.selinux_label = valid_label.to_string();
+        spec.process = Some(process);
+        security(&spec).unwrap();
+
+        let mut linux = Linux::default();
+        linux.mount_label = valid_label.to_string();
+        spec.linux = Some(linux);
+        security(&spec).unwrap();
+
+        let invalid_label = "system_u:system_r:container_t";
+        let mut process = Process::default();
+        process.selinux_label = invalid_label.to_string();
+        spec.process = Some(process);
+        security(&spec).unwrap_err();
+
+        let mut linux = Linux::default();
+        linux.mount_label = invalid_label.to_string();
+        spec.linux = Some(linux);
+        security(&spec).unwrap_err();
     }
 
     #[test]

src/agent/src/ap.rs

@@ -0,0 +1,79 @@
+// Copyright (c) IBM Corp. 2023
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+use std::fmt;
+use std::str::FromStr;
+
+use anyhow::{anyhow, Context};
+
+// IBM Adjunct Processor (AP) is used for cryptographic operations
+// by IBM Crypto Express hardware security modules on IBM zSystem & LinuxONE (s390x).
+// In Linux, virtual cryptographic devices are called AP queues.
+// The name of an AP queue respects a format <xx>.<xxxx> in hexadecimal notation [1, p.467]:
+//   - <xx> is an adapter ID
+//   - <xxxx> is an adapter domain ID
+// [1] https://www.ibm.com/docs/en/linuxonibm/pdf/lku5dd05.pdf
+
+#[derive(Debug)]
+pub struct Address {
+    pub adapter_id: u8,
+    pub adapter_domain: u16,
+}
+
+impl Address {
+    pub fn new(adapter_id: u8, adapter_domain: u16) -> Address {
+        Address {
+            adapter_id,
+            adapter_domain,
+        }
+    }
+}
+
+impl FromStr for Address {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> anyhow::Result<Self> {
+        let split: Vec<&str> = s.split('.').collect();
+        if split.len() != 2 {
+            return Err(anyhow!(
+                "Wrong AP bus format. It needs to be in the form <xx>.<xxxx> (e.g. 0a.003f), got {:?}",
+                s
+            ));
+        }
+
+        let adapter_id = u8::from_str_radix(split[0], 16).context(format!(
+            "Wrong AP bus format. AP ID needs to be in the form <xx> (e.g. 0a), got {:?}",
+            split[0]
+        ))?;
+        let adapter_domain = u16::from_str_radix(split[1], 16).context(format!(
+            "Wrong AP bus format. AP domain needs to be in the form <xxxx> (e.g. 003f), got {:?}",
+            split[1]
+        ))?;
+
+        Ok(Address::new(adapter_id, adapter_domain))
+    }
+}
+
+impl fmt::Display for Address {
+    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+        write!(f, "{:02x}.{:04x}", self.adapter_id, self.adapter_domain)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_from_str() {
+        let device = Address::from_str("a.1").unwrap();
+        assert_eq!(format!("{}", device), "0a.0001");
+
+        assert!(Address::from_str("").is_err());
+        assert!(Address::from_str(".").is_err());
+        assert!(Address::from_str("0.0.0").is_err());
+        assert!(Address::from_str("0g.0000").is_err());
+        assert!(Address::from_str("0a.10000").is_err());
+    }
+}

src/agent/src/config.rs

@@ -200,7 +200,7 @@ impl AgentConfig {
         let config_position = args.iter().position(|a| a == "--config" || a == "-c");
         if let Some(config_position) = config_position {
             if let Some(config_file) = args.get(config_position + 1) {
-                return AgentConfig::from_config_file(config_file);
+                return AgentConfig::from_config_file(config_file).context("AgentConfig from args");
             } else {
                 panic!("The config argument wasn't formed properly: {:?}", args);
             }
@@ -216,7 +216,8 @@ impl AgentConfig {
             // or if it can't be parsed properly.
             if param.starts_with(format!("{}=", CONFIG_FILE).as_str()) {
                 let config_file = get_string_value(param)?;
-                return AgentConfig::from_config_file(&config_file);
+                return AgentConfig::from_config_file(&config_file)
+                    .context("AgentConfig from kernel cmdline");
             }
 
             // parse cmdline flags
@@ -304,7 +305,8 @@ impl AgentConfig {
 
     #[instrument]
     pub fn from_config_file(file: &str) -> Result<AgentConfig> {
-        let config = fs::read_to_string(file)?;
+        let config = fs::read_to_string(file)
+            .with_context(|| format!("Failed to read config file {}", file))?;
         AgentConfig::from_str(&config)
     }
 

src/agent/src/device.rs

@@ -16,13 +16,12 @@ use std::str::FromStr;
 use std::sync::Arc;
 use tokio::sync::Mutex;
 
-#[cfg(target_arch = "s390x")]
-use crate::ccw;
 use crate::linux_abi::*;
 use crate::pci;
 use crate::sandbox::Sandbox;
 use crate::uevent::{wait_for_uevent, Uevent, UeventMatcher};
 use anyhow::{anyhow, Context, Result};
+use cfg_if::cfg_if;
 use oci::{LinuxDeviceCgroup, LinuxResources, Spec};
 use protocols::agent::Device;
 use tracing::instrument;
@@ -35,7 +34,7 @@ macro_rules! sl {
 }
 
 const VM_ROOTFS: &str = "/";
-
+const BLOCK: &str = "block";
 pub const DRIVER_9P_TYPE: &str = "9p";
 pub const DRIVER_VIRTIOFS_TYPE: &str = "virtio-fs";
 pub const DRIVER_BLK_TYPE: &str = "blk";
@@ -46,14 +45,22 @@ pub const DRIVER_NVDIMM_TYPE: &str = "nvdimm";
 pub const DRIVER_EPHEMERAL_TYPE: &str = "ephemeral";
 pub const DRIVER_LOCAL_TYPE: &str = "local";
 pub const DRIVER_WATCHABLE_BIND_TYPE: &str = "watchable-bind";
-// VFIO device to be bound to a guest kernel driver
-pub const DRIVER_VFIO_GK_TYPE: &str = "vfio-gk";
-// VFIO device to be bound to vfio-pci and made available inside the
+// VFIO PCI device to be bound to a guest kernel driver
+pub const DRIVER_VFIO_PCI_GK_TYPE: &str = "vfio-pci-gk";
+// VFIO PCI device to be bound to vfio-pci and made available inside the
 // container as a VFIO device node
-pub const DRIVER_VFIO_TYPE: &str = "vfio";
+pub const DRIVER_VFIO_PCI_TYPE: &str = "vfio-pci";
+pub const DRIVER_VFIO_AP_TYPE: &str = "vfio-ap";
 pub const DRIVER_OVERLAYFS_TYPE: &str = "overlayfs";
 pub const FS_TYPE_HUGETLB: &str = "hugetlbfs";
 
+cfg_if! {
+    if #[cfg(target_arch = "s390x")] {
+        use crate::ap;
+        use crate::ccw;
+    }
+}
+
 #[instrument]
 pub fn online_device(path: &str) -> Result<()> {
     fs::write(path, "1")?;
@@ -197,7 +204,7 @@ impl ScsiBlockMatcher {
 
 impl UeventMatcher for ScsiBlockMatcher {
     fn is_match(&self, uev: &Uevent) -> bool {
-        uev.subsystem == "block" && uev.devpath.contains(&self.search) && !uev.devname.is_empty()
+        uev.subsystem == BLOCK && uev.devpath.contains(&self.search) && !uev.devname.is_empty()
     }
 }
 
@@ -231,7 +238,7 @@ impl VirtioBlkPciMatcher {
 
 impl UeventMatcher for VirtioBlkPciMatcher {
     fn is_match(&self, uev: &Uevent) -> bool {
-        uev.subsystem == "block" && self.rex.is_match(&uev.devpath) && !uev.devname.is_empty()
+        uev.subsystem == BLOCK && self.rex.is_match(&uev.devpath) && !uev.devname.is_empty()
     }
 }
 
@@ -280,7 +287,7 @@ pub async fn get_virtio_blk_ccw_device_name(
     sandbox: &Arc<Mutex<Sandbox>>,
     device: &ccw::Device,
 ) -> Result<String> {
-    let matcher = VirtioBlkCCWMatcher::new(&create_ccw_root_bus_path(), device);
+    let matcher = VirtioBlkCCWMatcher::new(CCW_ROOT_BUS_PATH, device);
     let uev = wait_for_uevent(sandbox, matcher).await?;
     let devname = uev.devname;
     return match Path::new(SYSTEM_DEV_PATH).join(&devname).to_str() {
@@ -304,7 +311,7 @@ impl PmemBlockMatcher {
 
 impl UeventMatcher for PmemBlockMatcher {
     fn is_match(&self, uev: &Uevent) -> bool {
-        uev.subsystem == "block"
+        uev.subsystem == BLOCK
             && uev.devpath.starts_with(ACPI_DEV_PATH)
             && uev.devpath.ends_with(&self.suffix)
             && !uev.devname.is_empty()
@@ -401,6 +408,81 @@ async fn get_vfio_device_name(sandbox: &Arc<Mutex<Sandbox>>, grp: IommuGroup) ->
     Ok(format!("{}/{}", SYSTEM_DEV_PATH, &uev.devname))
 }
 
+#[cfg(target_arch = "s390x")]
+#[derive(Debug)]
+struct ApMatcher {
+    syspath: String,
+}
+
+#[cfg(target_arch = "s390x")]
+impl ApMatcher {
+    fn new(address: ap::Address) -> ApMatcher {
+        ApMatcher {
+            syspath: format!(
+                "{}/card{:02x}/{}",
+                AP_ROOT_BUS_PATH, address.adapter_id, address
+            ),
+        }
+    }
+}
+
+#[cfg(target_arch = "s390x")]
+impl UeventMatcher for ApMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.action == "add" && uev.devpath == self.syspath
+    }
+}
+
+#[cfg(target_arch = "s390x")]
+#[instrument]
+async fn wait_for_ap_device(sandbox: &Arc<Mutex<Sandbox>>, address: ap::Address) -> Result<()> {
+    let matcher = ApMatcher::new(address);
+    wait_for_uevent(sandbox, matcher).await?;
+    Ok(())
+}
+
+#[derive(Debug)]
+struct MmioBlockMatcher {
+    suffix: String,
+}
+
+impl MmioBlockMatcher {
+    fn new(devname: &str) -> MmioBlockMatcher {
+        MmioBlockMatcher {
+            suffix: format!(r"/block/{}", devname),
+        }
+    }
+}
+
+impl UeventMatcher for MmioBlockMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.subsystem == BLOCK && uev.devpath.ends_with(&self.suffix) && !uev.devname.is_empty()
+    }
+}
+
+#[instrument]
+pub async fn get_virtio_mmio_device_name(
+    sandbox: &Arc<Mutex<Sandbox>>,
+    devpath: &str,
+) -> Result<()> {
+    let devname = devpath
+        .strip_prefix("/dev/")
+        .ok_or_else(|| anyhow!("Storage source '{}' must start with /dev/", devpath))?;
+
+    let matcher = MmioBlockMatcher::new(devname);
+    let uev = wait_for_uevent(sandbox, matcher)
+        .await
+        .context("failed to wait for uevent")?;
+    if uev.devname != devname {
+        return Err(anyhow!(
+            "Unexpected device name {} for mmio device (expected {})",
+            uev.devname,
+            devname
+        ));
+    }
+    Ok(())
+}
+
 /// Scan SCSI bus for the given SCSI address(SCSI-Id and LUN)
 #[instrument]
 fn scan_scsi_bus(scsi_addr: &str) -> Result<()> {
@@ -414,7 +496,7 @@ fn scan_scsi_bus(scsi_addr: &str) -> Result<()> {
 
     // Scan scsi host passing in the channel, SCSI id and LUN.
     // Channel is always 0 because we have only one SCSI controller.
-    let scan_data = format!("0 {} {}", tokens[0], tokens[1]);
+    let scan_data = &format!("0 {} {}", tokens[0], tokens[1]);
 
     for entry in fs::read_dir(SYSFS_SCSI_HOST_PATH)? {
         let host = entry?.file_name();
@@ -428,7 +510,7 @@ fn scan_scsi_bus(scsi_addr: &str) -> Result<()> {
 
         let scan_path = PathBuf::from(&format!("{}/{}/{}", SYSFS_SCSI_HOST_PATH, host_str, "scan"));
 
-        fs::write(scan_path, &scan_data)?;
+        fs::write(scan_path, scan_data)?;
     }
 
     Ok(())
@@ -636,12 +718,18 @@ pub fn update_env_pci(
 #[instrument]
 async fn virtiommio_blk_device_handler(
     device: &Device,
-    _sandbox: &Arc<Mutex<Sandbox>>,
+    sandbox: &Arc<Mutex<Sandbox>>,
 ) -> Result<SpecUpdate> {
     if device.vm_path.is_empty() {
         return Err(anyhow!("Invalid path for virtio mmio blk device"));
     }
 
+    if !Path::new(&device.vm_path).exists() {
+        get_virtio_mmio_device_name(sandbox, &device.vm_path.to_string())
+            .await
+            .context("failed to get mmio device name")?;
+    }
+
     Ok(DevNumUpdate::from_vm_path(&device.vm_path)?.into())
 }
 
@@ -699,7 +787,7 @@ async fn virtio_nvdimm_device_handler(
     Ok(DevNumUpdate::from_vm_path(&device.vm_path)?.into())
 }
 
-fn split_vfio_option(opt: &str) -> Option<(&str, &str)> {
+fn split_vfio_pci_option(opt: &str) -> Option<(&str, &str)> {
     let mut tokens = opt.split('=');
     let hostbdf = tokens.next()?;
     let path = tokens.next()?;
@@ -714,14 +802,18 @@ fn split_vfio_option(opt: &str) -> Option<(&str, &str)> {
 // Each option should have the form "DDDD:BB:DD.F=<pcipath>"
 //     DDDD:BB:DD.F is the device's PCI address in the host
 //     <pcipath> is a PCI path to the device in the guest (see pci.rs)
-async fn vfio_device_handler(device: &Device, sandbox: &Arc<Mutex<Sandbox>>) -> Result<SpecUpdate> {
-    let vfio_in_guest = device.field_type != DRIVER_VFIO_GK_TYPE;
+#[instrument]
+async fn vfio_pci_device_handler(
+    device: &Device,
+    sandbox: &Arc<Mutex<Sandbox>>,
+) -> Result<SpecUpdate> {
+    let vfio_in_guest = device.type_ != DRIVER_VFIO_PCI_GK_TYPE;
     let mut pci_fixups = Vec::<(pci::Address, pci::Address)>::new();
     let mut group = None;
 
     for opt in device.options.iter() {
-        let (host, pcipath) =
-            split_vfio_option(opt).ok_or_else(|| anyhow!("Malformed VFIO option {:?}", opt))?;
+        let (host, pcipath) = split_vfio_pci_option(opt)
+            .ok_or_else(|| anyhow!("Malformed VFIO PCI option {:?}", opt))?;
         let host =
             pci::Address::from_str(host).context("Bad host PCI address in VFIO option {:?}")?;
         let pcipath = pci::Path::from_str(pcipath)?;
@@ -763,6 +855,28 @@ async fn vfio_device_handler(device: &Device, sandbox: &Arc<Mutex<Sandbox>>) ->
     })
 }
 
+// The VFIO AP (Adjunct Processor) device handler takes all the APQNs provided as device options
+// and awaits them. It sets the minimum AP rescan time of 5 seconds and temporarily adds that
+// amount to the hotplug timeout.
+#[cfg(target_arch = "s390x")]
+#[instrument]
+async fn vfio_ap_device_handler(
+    device: &Device,
+    sandbox: &Arc<Mutex<Sandbox>>,
+) -> Result<SpecUpdate> {
+    // Force AP bus rescan
+    fs::write(AP_SCANS_PATH, "1")?;
+    for apqn in device.options.iter() {
+        wait_for_ap_device(sandbox, ap::Address::from_str(apqn)?).await?;
+    }
+    Ok(Default::default())
+}
+
+#[cfg(not(target_arch = "s390x"))]
+async fn vfio_ap_device_handler(_: &Device, _: &Arc<Mutex<Sandbox>>) -> Result<SpecUpdate> {
+    Err(anyhow!("AP is only supported on s390x"))
+}
+
 #[instrument]
 pub async fn add_devices(
     devices: &[Device],
@@ -808,9 +922,9 @@ pub async fn add_devices(
 async fn add_device(device: &Device, sandbox: &Arc<Mutex<Sandbox>>) -> Result<SpecUpdate> {
     // log before validation to help with debugging gRPC protocol version differences.
     info!(sl!(), "device-id: {}, device-type: {}, device-vm-path: {}, device-container-path: {}, device-options: {:?}",
-          device.id, device.field_type, device.vm_path, device.container_path, device.options);
+          device.id, device.type_, device.vm_path, device.container_path, device.options);
 
-    if device.field_type.is_empty() {
+    if device.type_.is_empty() {
         return Err(anyhow!("invalid type for device {:?}", device));
     }
 
@@ -822,14 +936,17 @@ async fn add_device(device: &Device, sandbox: &Arc<Mutex<Sandbox>>) -> Result<Sp
         return Err(anyhow!("invalid container path for device {:?}", device));
     }
 
-    match device.field_type.as_str() {
+    match device.type_.as_str() {
         DRIVER_BLK_TYPE => virtio_blk_device_handler(device, sandbox).await,
         DRIVER_BLK_CCW_TYPE => virtio_blk_ccw_device_handler(device, sandbox).await,
         DRIVER_MMIO_BLK_TYPE => virtiommio_blk_device_handler(device, sandbox).await,
         DRIVER_NVDIMM_TYPE => virtio_nvdimm_device_handler(device, sandbox).await,
         DRIVER_SCSI_TYPE => virtio_scsi_device_handler(device, sandbox).await,
-        DRIVER_VFIO_GK_TYPE | DRIVER_VFIO_TYPE => vfio_device_handler(device, sandbox).await,
-        _ => Err(anyhow!("Unknown device type {}", device.field_type)),
+        DRIVER_VFIO_PCI_GK_TYPE | DRIVER_VFIO_PCI_TYPE => {
+            vfio_pci_device_handler(device, sandbox).await
+        }
+        DRIVER_VFIO_AP_TYPE => vfio_ap_device_handler(device, sandbox).await,
+        _ => Err(anyhow!("Unknown device type {}", device.type_)),
     }
 }
 
@@ -1325,7 +1442,7 @@ mod tests {
 
         let mut uev = crate::uevent::Uevent::default();
         uev.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
-        uev.subsystem = "block".to_string();
+        uev.subsystem = BLOCK.to_string();
         uev.devpath = devpath.clone();
         uev.devname = devname.to_string();
 
@@ -1359,7 +1476,7 @@ mod tests {
         let mut uev_a = crate::uevent::Uevent::default();
         let relpath_a = "/0000:00:0a.0";
         uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
-        uev_a.subsystem = "block".to_string();
+        uev_a.subsystem = BLOCK.to_string();
         uev_a.devname = devname.to_string();
         uev_a.devpath = format!("{}{}/virtio4/block/{}", root_bus, relpath_a, devname);
         let matcher_a = VirtioBlkPciMatcher::new(relpath_a);
@@ -1378,7 +1495,7 @@ mod tests {
     #[cfg(target_arch = "s390x")]
     #[tokio::test]
     async fn test_virtio_blk_ccw_matcher() {
-        let root_bus = create_ccw_root_bus_path();
+        let root_bus = CCW_ROOT_BUS_PATH;
         let subsystem = "block";
         let devname = "vda";
         let relpath = "0.0.0002";
@@ -1443,7 +1560,7 @@ mod tests {
         let mut uev_a = crate::uevent::Uevent::default();
         let addr_a = "0:0";
         uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
-        uev_a.subsystem = "block".to_string();
+        uev_a.subsystem = BLOCK.to_string();
         uev_a.devname = devname.to_string();
         uev_a.devpath = format!(
             "{}/0000:00:00.0/virtio0/host0/target0:0:0/0:0:{}/block/sda",
@@ -1486,14 +1603,41 @@ mod tests {
         assert!(!matcher_a.is_match(&uev_b));
     }
 
+    #[tokio::test]
+    async fn test_mmio_block_matcher() {
+        let devname_a = "vda";
+        let devname_b = "vdb";
+        let mut uev_a = crate::uevent::Uevent::default();
+        uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
+        uev_a.subsystem = BLOCK.to_string();
+        uev_a.devname = devname_a.to_string();
+        uev_a.devpath = format!(
+            "/sys/devices/virtio-mmio-cmdline/virtio-mmio.0/virtio0/block/{}",
+            devname_a
+        );
+        let matcher_a = MmioBlockMatcher::new(devname_a);
+
+        let mut uev_b = uev_a.clone();
+        uev_b.devpath = format!(
+            "/sys/devices/virtio-mmio-cmdline/virtio-mmio.4/virtio4/block/{}",
+            devname_b
+        );
+        let matcher_b = MmioBlockMatcher::new(devname_b);
+
+        assert!(matcher_a.is_match(&uev_a));
+        assert!(matcher_b.is_match(&uev_b));
+        assert!(!matcher_b.is_match(&uev_a));
+        assert!(!matcher_a.is_match(&uev_b));
+    }
+
     #[test]
-    fn test_split_vfio_option() {
+    fn test_split_vfio_pci_option() {
         assert_eq!(
-            split_vfio_option("0000:01:00.0=02/01"),
+            split_vfio_pci_option("0000:01:00.0=02/01"),
             Some(("0000:01:00.0", "02/01"))
         );
-        assert_eq!(split_vfio_option("0000:01:00.0=02/01=rubbish"), None);
-        assert_eq!(split_vfio_option("0000:01:00.0"), None);
+        assert_eq!(split_vfio_pci_option("0000:01:00.0=02/01=rubbish"), None);
+        assert_eq!(split_vfio_pci_option("0000:01:00.0"), None);
     }
 
     #[test]
@@ -1531,7 +1675,7 @@ mod tests {
         pci_driver_override(syspci, dev0, "drv_b").unwrap();
         assert_eq!(fs::read_to_string(&dev0override).unwrap(), "drv_b");
         assert_eq!(fs::read_to_string(&probepath).unwrap(), dev0.to_string());
-        assert_eq!(fs::read_to_string(&drvaunbind).unwrap(), dev0.to_string());
+        assert_eq!(fs::read_to_string(drvaunbind).unwrap(), dev0.to_string());
     }
 
     #[test]
@@ -1543,7 +1687,7 @@ mod tests {
         let dev0 = pci::Address::new(0, 0, pci::SlotFn::new(0, 0).unwrap());
         let dev0path = syspci.join("devices").join(dev0.to_string());
 
-        fs::create_dir_all(&dev0path).unwrap();
+        fs::create_dir_all(dev0path).unwrap();
 
         // Test dev0
         assert!(pci_iommu_group(&syspci, dev0).unwrap().is_none());
@@ -1554,7 +1698,7 @@ mod tests {
         let dev1group = dev1path.join("iommu_group");
 
         fs::create_dir_all(&dev1path).unwrap();
-        std::os::unix::fs::symlink("../../../kernel/iommu_groups/12", &dev1group).unwrap();
+        std::os::unix::fs::symlink("../../../kernel/iommu_groups/12", dev1group).unwrap();
 
         // Test dev1
         assert_eq!(
@@ -1567,9 +1711,40 @@ mod tests {
         let dev2path = syspci.join("devices").join(dev2.to_string());
         let dev2group = dev2path.join("iommu_group");
 
-        fs::create_dir_all(&dev2group).unwrap();
+        fs::create_dir_all(dev2group).unwrap();
 
         // Test dev2
         assert!(pci_iommu_group(&syspci, dev2).is_err());
     }
+
+    #[cfg(target_arch = "s390x")]
+    #[tokio::test]
+    async fn test_vfio_ap_matcher() {
+        let subsystem = "ap";
+        let card = "0a";
+        let relpath = format!("{}.0001", card);
+
+        let mut uev = Uevent::default();
+        uev.action = U_EVENT_ACTION_ADD.to_string();
+        uev.subsystem = subsystem.to_string();
+        uev.devpath = format!("{}/card{}/{}", AP_ROOT_BUS_PATH, card, relpath);
+
+        let ap_address = ap::Address::from_str(&relpath).unwrap();
+        let matcher = ApMatcher::new(ap_address);
+
+        assert!(matcher.is_match(&uev));
+
+        let mut uev_remove = uev.clone();
+        uev_remove.action = U_EVENT_ACTION_REMOVE.to_string();
+        assert!(!matcher.is_match(&uev_remove));
+
+        let mut uev_other_device = uev.clone();
+        uev_other_device.devpath = format!(
+            "{}/card{}/{}",
+            AP_ROOT_BUS_PATH,
+            card,
+            format!("{}.0002", card)
+        );
+        assert!(!matcher.is_match(&uev_other_device));
+    }
 }

src/agent/src/linux_abi.rs

@@ -3,6 +3,8 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+use cfg_if::cfg_if;
+
 /// Linux ABI related constants.
 
 #[cfg(target_arch = "aarch64")]
@@ -64,10 +66,14 @@ pub fn create_pci_root_bus_path() -> String {
     ret
 }
 
-#[cfg(target_arch = "s390x")]
-pub fn create_ccw_root_bus_path() -> String {
-    String::from("/devices/css0")
+cfg_if! {
+    if #[cfg(target_arch = "s390x")] {
+        pub const CCW_ROOT_BUS_PATH: &str = "/devices/css0";
+        pub const AP_ROOT_BUS_PATH: &str = "/devices/ap";
+        pub const AP_SCANS_PATH: &str = "/sys/bus/ap/scans";
+    }
 }
+
 // From https://www.kernel.org/doc/Documentation/acpi/namespace.txt
 // The Linux kernel's core ACPI subsystem creates struct acpi_device
 // objects for ACPI namespace objects representing devices, power resources

src/agent/src/main.rs

@@ -20,6 +20,7 @@ extern crate scopeguard;
 extern crate slog;
 
 use anyhow::{anyhow, Context, Result};
+use cfg_if::cfg_if;
 use clap::{AppSettings, Parser};
 use nix::fcntl::OFlag;
 use nix::sys::socket::{self, AddressFamily, SockFlag, SockType, VsockAddr};
@@ -34,8 +35,6 @@ use std::process::exit;
 use std::sync::Arc;
 use tracing::{instrument, span};
 
-#[cfg(target_arch = "s390x")]
-mod ccw;
 mod config;
 mod console;
 mod device;
@@ -74,6 +73,13 @@ use tokio::{
 mod rpc;
 mod tracer;
 
+cfg_if! {
+    if #[cfg(target_arch = "s390x")] {
+        mod ap;
+        mod ccw;
+    }
+}
+
 const NAME: &str = "kata-agent";
 
 lazy_static! {
@@ -339,7 +345,7 @@ async fn start_sandbox(
     sandbox.lock().await.sender = Some(tx);
 
     // vsock:///dev/vsock, port
-    let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str())?;
+    let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str(), init_mode)?;
     server.start().await?;
 
     rx.await?;
@@ -436,9 +442,8 @@ mod tests {
             let msg = format!("test[{}]: {:?}", i, d);
             let (rfd, wfd) = unistd::pipe2(OFlag::O_CLOEXEC).unwrap();
             defer!({
-                // rfd is closed by the use of PipeStream in the crate_logger_task function,
-                // but we will attempt to close in case of a failure
-                let _ = unistd::close(rfd);
+                // XXX: Never try to close rfd, because it will be closed by PipeStream in
+                // create_logger_task() and it's not safe to close the same fd twice time.
                 unistd::close(wfd).unwrap();
             });
 

src/agent/src/metrics.rs

@@ -5,10 +5,11 @@
 
 extern crate procfs;
 
-use prometheus::{Encoder, Gauge, GaugeVec, IntCounter, TextEncoder};
+use prometheus::{Encoder, Gauge, GaugeVec, IntCounter, Opts, Registry, TextEncoder};
 
-use anyhow::Result;
+use anyhow::{anyhow, Result};
 use slog::warn;
+use std::sync::Mutex;
 use tracing::instrument;
 
 const NAMESPACE_KATA_AGENT: &str = "kata_agent";
@@ -23,55 +24,70 @@ macro_rules! sl {
 
 lazy_static! {
 
-    static ref     AGENT_SCRAPE_COUNT: IntCounter =
-    prometheus::register_int_counter!(format!("{}_{}",NAMESPACE_KATA_AGENT,"scrape_count"), "Metrics scrape count").unwrap();
+    static ref REGISTERED: Mutex<bool> = Mutex::new(false);
 
-    static ref     AGENT_THREADS: Gauge =
-    prometheus::register_gauge!(format!("{}_{}",NAMESPACE_KATA_AGENT,"threads"), "Agent process threads").unwrap();
+    // custom registry
+    static ref REGISTRY: Registry = Registry::new();
 
-    static ref     AGENT_TOTAL_TIME: Gauge =
-    prometheus::register_gauge!(format!("{}_{}",NAMESPACE_KATA_AGENT,"total_time"), "Agent process total time").unwrap();
+    static ref AGENT_SCRAPE_COUNT: IntCounter =
+    IntCounter::new(format!("{}_{}",NAMESPACE_KATA_AGENT,"scrape_count"), "Metrics scrape count").unwrap();
 
-    static ref     AGENT_TOTAL_VM: Gauge =
-    prometheus::register_gauge!(format!("{}_{}",NAMESPACE_KATA_AGENT,"total_vm"), "Agent process total VM size").unwrap();
+    // agent metrics
+    static ref AGENT_THREADS: Gauge =
+    Gauge::new(format!("{}_{}",NAMESPACE_KATA_AGENT,"threads"), "Agent process threads").unwrap();
 
-    static ref     AGENT_TOTAL_RSS: Gauge =
-    prometheus::register_gauge!(format!("{}_{}",NAMESPACE_KATA_AGENT,"total_rss"), "Agent process total RSS size").unwrap();
+    static ref AGENT_TOTAL_TIME: Gauge =
+    Gauge::new(format!("{}_{}",NAMESPACE_KATA_AGENT,"total_time"), "Agent process total time").unwrap();
 
-    static ref     AGENT_PROC_STATUS: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_AGENT,"proc_status"), "Agent process status.", &["item"]).unwrap();
+    static ref AGENT_TOTAL_VM: Gauge =
+    Gauge::new(format!("{}_{}",NAMESPACE_KATA_AGENT,"total_vm"), "Agent process total VM size").unwrap() ;
 
-    static ref     AGENT_IO_STAT: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_AGENT,"io_stat"), "Agent process IO statistics.", &["item"]).unwrap();
+    static ref AGENT_TOTAL_RSS: Gauge =
+    Gauge::new(format!("{}_{}",NAMESPACE_KATA_AGENT,"total_rss"), "Agent process total RSS size").unwrap();
 
-    static ref     AGENT_PROC_STAT: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_AGENT,"proc_stat"), "Agent process statistics.", &["item"]).unwrap();
+    static ref AGENT_PROC_STATUS: GaugeVec =
+    GaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_AGENT,"proc_status"), "Agent process status."), &["item"]).unwrap();
+
+    static ref AGENT_IO_STAT: GaugeVec =
+    GaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_AGENT,"io_stat"), "Agent process IO statistics."), &["item"]).unwrap();
+
+    static ref AGENT_PROC_STAT: GaugeVec =
+    GaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_AGENT,"proc_stat"), "Agent process statistics."), &["item"]).unwrap();
 
     // guest os metrics
-    static ref     GUEST_LOAD: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"load") , "Guest system load.", &["item"]).unwrap();
+    static ref GUEST_LOAD: GaugeVec =
+    GaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_GUEST,"load"), "Guest system load."), &["item"]).unwrap();
 
-    static ref     GUEST_TASKS: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"tasks") , "Guest system load.", &["item"]).unwrap();
+    static ref GUEST_TASKS: GaugeVec =
+    GaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_GUEST,"tasks"), "Guest system load."), &["item"]).unwrap();
 
-    static ref     GUEST_CPU_TIME: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"cpu_time") , "Guest CPU statistics.", &["cpu","item"]).unwrap();
+    static ref GUEST_CPU_TIME: GaugeVec =
+    GaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_GUEST,"cpu_time"), "Guest CPU statistics."), &["cpu","item"]).unwrap();
 
-    static ref     GUEST_VM_STAT: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"vm_stat") , "Guest virtual memory statistics.", &["item"]).unwrap();
+    static ref GUEST_VM_STAT: GaugeVec =
+    GaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_GUEST,"vm_stat"), "Guest virtual memory statistics."), &["item"]).unwrap();
 
-    static ref     GUEST_NETDEV_STAT: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"netdev_stat") , "Guest net devices statistics.", &["interface","item"]).unwrap();
+    static ref GUEST_NETDEV_STAT: GaugeVec =
+    GaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_GUEST,"netdev_stat"), "Guest net devices statistics."), &["interface","item"]).unwrap();
 
-    static ref     GUEST_DISKSTAT: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"diskstat") , "Disks statistics in system.", &["disk","item"]).unwrap();
+    static ref GUEST_DISKSTAT: GaugeVec =
+    GaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_GUEST,"diskstat"), "Disks statistics in system."), &["disk","item"]).unwrap();
 
-    static ref     GUEST_MEMINFO: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"meminfo") , "Statistics about memory usage in the system.", &["item"]).unwrap();
+    static ref GUEST_MEMINFO: GaugeVec =
+    GaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_GUEST,"meminfo"), "Statistics about memory usage in the system."), &["item"]).unwrap();
 }
 
 #[instrument]
 pub fn get_metrics(_: &protocols::agent::GetMetricsRequest) -> Result<String> {
+    let mut registered = REGISTERED
+        .lock()
+        .map_err(|e| anyhow!("failed to check agent metrics register status {:?}", e))?;
+
+    if !(*registered) {
+        register_metrics()?;
+        *registered = true;
+    }
+
     AGENT_SCRAPE_COUNT.inc();
 
     // update agent process metrics
@@ -81,7 +97,7 @@ pub fn get_metrics(_: &protocols::agent::GetMetricsRequest) -> Result<String> {
     update_guest_metrics();
 
     // gather all metrics and return as a String
-    let metric_families = prometheus::gather();
+    let metric_families = REGISTRY.gather();
 
     let mut buffer = Vec::new();
     let encoder = TextEncoder::new();
@@ -90,6 +106,31 @@ pub fn get_metrics(_: &protocols::agent::GetMetricsRequest) -> Result<String> {
     Ok(String::from_utf8(buffer)?)
 }
 
+#[instrument]
+fn register_metrics() -> Result<()> {
+    REGISTRY.register(Box::new(AGENT_SCRAPE_COUNT.clone()))?;
+
+    // agent metrics
+    REGISTRY.register(Box::new(AGENT_THREADS.clone()))?;
+    REGISTRY.register(Box::new(AGENT_TOTAL_TIME.clone()))?;
+    REGISTRY.register(Box::new(AGENT_TOTAL_VM.clone()))?;
+    REGISTRY.register(Box::new(AGENT_TOTAL_RSS.clone()))?;
+    REGISTRY.register(Box::new(AGENT_PROC_STATUS.clone()))?;
+    REGISTRY.register(Box::new(AGENT_IO_STAT.clone()))?;
+    REGISTRY.register(Box::new(AGENT_PROC_STAT.clone()))?;
+
+    // guest metrics
+    REGISTRY.register(Box::new(GUEST_LOAD.clone()))?;
+    REGISTRY.register(Box::new(GUEST_TASKS.clone()))?;
+    REGISTRY.register(Box::new(GUEST_CPU_TIME.clone()))?;
+    REGISTRY.register(Box::new(GUEST_VM_STAT.clone()))?;
+    REGISTRY.register(Box::new(GUEST_NETDEV_STAT.clone()))?;
+    REGISTRY.register(Box::new(GUEST_DISKSTAT.clone()))?;
+    REGISTRY.register(Box::new(GUEST_MEMINFO.clone()))?;
+
+    Ok(())
+}
+
 #[instrument]
 fn update_agent_metrics() -> Result<()> {
     let me = procfs::process::Process::myself();

src/agent/src/mount.rs

@@ -21,10 +21,11 @@ use nix::unistd::{Gid, Uid};
 use regex::Regex;
 
 use crate::device::{
-    get_scsi_device_name, get_virtio_blk_pci_device_name, online_device, wait_for_pmem_device,
-    DRIVER_9P_TYPE, DRIVER_BLK_CCW_TYPE, DRIVER_BLK_TYPE, DRIVER_EPHEMERAL_TYPE, DRIVER_LOCAL_TYPE,
-    DRIVER_MMIO_BLK_TYPE, DRIVER_NVDIMM_TYPE, DRIVER_OVERLAYFS_TYPE, DRIVER_SCSI_TYPE,
-    DRIVER_VIRTIOFS_TYPE, DRIVER_WATCHABLE_BIND_TYPE, FS_TYPE_HUGETLB,
+    get_scsi_device_name, get_virtio_blk_pci_device_name, get_virtio_mmio_device_name,
+    online_device, wait_for_pmem_device, DRIVER_9P_TYPE, DRIVER_BLK_CCW_TYPE, DRIVER_BLK_TYPE,
+    DRIVER_EPHEMERAL_TYPE, DRIVER_LOCAL_TYPE, DRIVER_MMIO_BLK_TYPE, DRIVER_NVDIMM_TYPE,
+    DRIVER_OVERLAYFS_TYPE, DRIVER_SCSI_TYPE, DRIVER_VIRTIOFS_TYPE, DRIVER_WATCHABLE_BIND_TYPE,
+    FS_TYPE_HUGETLB,
 };
 use crate::linux_abi::*;
 use crate::pci;
@@ -211,10 +212,10 @@ async fn ephemeral_storage_handler(
     // By now we only support one option field: "fsGroup" which
     // isn't an valid mount option, thus we should remove it when
     // do mount.
-    if storage.options.len() > 0 {
+    if !storage.options.is_empty() {
         // ephemeral_storage didn't support mount options except fsGroup.
         let mut new_storage = storage.clone();
-        new_storage.options = protobuf::RepeatedField::default();
+        new_storage.options = Default::default();
         common_storage_handler(logger, &new_storage)?;
 
         let opts_vec: Vec<String> = storage.options.to_vec();
@@ -240,6 +241,70 @@ async fn ephemeral_storage_handler(
     Ok("".to_string())
 }
 
+// update_ephemeral_mounts takes a list of ephemeral mounts and remounts them
+// with mount options passed by the caller
+#[instrument]
+pub async fn update_ephemeral_mounts(
+    logger: Logger,
+    storages: Vec<Storage>,
+    sandbox: Arc<Mutex<Sandbox>>,
+) -> Result<()> {
+    for (_, storage) in storages.iter().enumerate() {
+        let handler_name = storage.driver.clone();
+        let logger = logger.new(o!(
+	    "msg" => "updating tmpfs storage",
+            "subsystem" => "storage",
+            "storage-type" => handler_name.to_owned()));
+
+        match handler_name.as_str() {
+            DRIVER_EPHEMERAL_TYPE => {
+                fs::create_dir_all(Path::new(&storage.mount_point))?;
+
+                if storage.options.is_empty() {
+                    continue;
+                } else {
+                    // assume that fsGid has already been set
+                    let mut opts = Vec::<&str>::new();
+                    for (_, opt) in storage.options.iter().enumerate() {
+                        if opt.starts_with(FS_GID) {
+                            continue;
+                        }
+                        opts.push(opt)
+                    }
+                    let mount_path = Path::new(&storage.mount_point);
+                    let src_path = Path::new(&storage.source);
+
+                    let (flags, options) = parse_mount_flags_and_options(opts);
+
+                    info!(logger, "mounting storage";
+                    "mount-source" => src_path.display(),
+                    "mount-destination" => mount_path.display(),
+                    "mount-fstype"  => storage.fstype.as_str(),
+                    "mount-options" => options.as_str(),
+                    );
+
+                    return baremount(
+                        src_path,
+                        mount_path,
+                        storage.fstype.as_str(),
+                        flags,
+                        options.as_str(),
+                        &logger,
+                    );
+                }
+            }
+            _ => {
+                return Err(anyhow!(
+                    "Unsupported storage type for syncing mounts {}. Only ephemeral storage update is supported",
+                    storage.driver.to_owned()
+                ));
+            }
+        };
+    }
+
+    Ok(())
+}
+
 #[instrument]
 async fn overlayfs_storage_handler(
     logger: &Logger,
@@ -409,8 +474,14 @@ async fn virtiommio_blk_storage_handler(
     storage: &Storage,
     sandbox: Arc<Mutex<Sandbox>>,
 ) -> Result<String> {
+    let storage = storage.clone();
+    if !Path::new(&storage.source).exists() {
+        get_virtio_mmio_device_name(&sandbox, &storage.source)
+            .await
+            .context("failed to get mmio device name")?;
+    }
     //The source path is VmPath
-    common_storage_handler(logger, storage)
+    common_storage_handler(logger, &storage)
 }
 
 // virtiofs_storage_handler handles the storage for virtio-fs.
@@ -590,7 +661,7 @@ pub fn set_ownership(logger: &Logger, storage: &Storage) -> Result<()> {
     if storage.fs_group.is_none() {
         return Ok(());
     }
-    let fs_group = storage.get_fs_group();
+    let fs_group = storage.fs_group();
 
     let mut read_only = false;
     let opts_vec: Vec<String> = storage.options.to_vec();
@@ -607,7 +678,7 @@ pub fn set_ownership(logger: &Logger, storage: &Storage) -> Result<()> {
         err
     })?;
 
-    if fs_group.group_change_policy == FSGroupChangePolicy::OnRootMismatch
+    if fs_group.group_change_policy == FSGroupChangePolicy::OnRootMismatch.into()
         && metadata.gid() == fs_group.group_id
     {
         let mut mask = if read_only { RO_MASK } else { RW_MASK };
@@ -648,7 +719,7 @@ pub fn recursive_ownership_change(
 ) -> Result<()> {
     let mut mask = if read_only { RO_MASK } else { RW_MASK };
     if path.is_dir() {
-        for entry in fs::read_dir(&path)? {
+        for entry in fs::read_dir(path)? {
             recursive_ownership_change(entry?.path().as_path(), uid, gid, read_only)?;
         }
         mask |= EXEC_MASK;
@@ -779,16 +850,20 @@ pub async fn add_storages(
             }
         };
 
-        // Todo need to rollback the mounted storage if err met.
-
-        if res.is_err() {
-            error!(
-                logger,
-                "add_storages failed, storage: {:?}, error: {:?} ", storage, res
-            );
-        }
-
-        let mount_point = res?;
+        let mount_point = match res {
+            Err(e) => {
+                error!(
+                    logger,
+                    "add_storages failed, storage: {:?}, error: {:?} ", storage, e
+                );
+                let mut sb = sandbox.lock().await;
+                sb.unset_sandbox_storage(&storage.mount_point)
+                    .map_err(|e| warn!(logger, "fail to unset sandbox storage {:?}", e))
+                    .ok();
+                return Err(e);
+            }
+            Ok(m) => m,
+        };
 
         if !mount_point.is_empty() {
             mount_list.push(mount_point);
@@ -890,7 +965,7 @@ pub fn get_cgroup_mounts(
         }]);
     }
 
-    let file = File::open(&cg_path)?;
+    let file = File::open(cg_path)?;
     let reader = BufReader::new(file);
 
     let mut has_device_cgroup = false;
@@ -1026,7 +1101,6 @@ fn parse_options(option_list: Vec<String>) -> HashMap<String, String> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use protobuf::RepeatedField;
     use protocols::agent::FSGroup;
     use std::fs::File;
     use std::fs::OpenOptions;
@@ -1773,7 +1847,7 @@ mod tests {
             let tempdir = tempdir().unwrap();
 
             let src = if d.mask_src {
-                tempdir.path().join(&d.src)
+                tempdir.path().join(d.src)
             } else {
                 Path::new(d.src).to_path_buf()
             };
@@ -1947,9 +2021,8 @@ mod tests {
                 mount_path: "rw_mount",
                 fs_group: Some(FSGroup {
                     group_id: 3000,
-                    group_change_policy: FSGroupChangePolicy::Always,
-                    unknown_fields: Default::default(),
-                    cached_size: Default::default(),
+                    group_change_policy: FSGroupChangePolicy::Always.into(),
+                    ..Default::default()
                 }),
                 read_only: false,
                 expected_group_id: 3000,
@@ -1959,9 +2032,8 @@ mod tests {
                 mount_path: "ro_mount",
                 fs_group: Some(FSGroup {
                     group_id: 3000,
-                    group_change_policy: FSGroupChangePolicy::OnRootMismatch,
-                    unknown_fields: Default::default(),
-                    cached_size: Default::default(),
+                    group_change_policy: FSGroupChangePolicy::OnRootMismatch.into(),
+                    ..Default::default()
                 }),
                 read_only: true,
                 expected_group_id: 3000,
@@ -1981,10 +2053,7 @@ mod tests {
             let directory_mode = mount_dir.as_path().metadata().unwrap().permissions().mode();
             let mut storage_data = Storage::new();
             if d.read_only {
-                storage_data.set_options(RepeatedField::from_slice(&[
-                    "foo".to_string(),
-                    "ro".to_string(),
-                ]));
+                storage_data.set_options(vec!["foo".to_string(), "ro".to_string()]);
             }
             if let Some(fs_group) = d.fs_group.clone() {
                 storage_data.set_fs_group(fs_group);

src/agent/src/namespace.rs

@@ -78,6 +78,7 @@ impl Namespace {
     // setup creates persistent namespace without switching to it.
     // Note, pid namespaces cannot be persisted.
     #[instrument]
+    #[allow(clippy::question_mark)]
     pub async fn setup(mut self) -> Result<Self> {
         fs::create_dir_all(&self.persistent_ns_dir)?;
 
@@ -88,7 +89,7 @@ impl Namespace {
         }
         let logger = self.logger.clone();
 
-        let new_ns_path = ns_path.join(&ns_type.get());
+        let new_ns_path = ns_path.join(ns_type.get());
 
         File::create(new_ns_path.as_path())?;
 
@@ -102,7 +103,7 @@ impl Namespace {
                 let source = Path::new(&origin_ns_path);
                 let destination = new_ns_path.as_path();
 
-                File::open(&source)?;
+                File::open(source)?;
 
                 // Create a new netns on the current thread.
                 let cf = ns_type.get_flags();