Merge pull request #8267 from gkurz/3.2.0-branch-bump

# Kata Containers 3.2.0

.github/cargo-deny-composite-action/cargo-deny-generator.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Copyright (c) 2022 Red Hat
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+script_dir=$(dirname "$(readlink -f "$0")")
+parent_dir=$(realpath "${script_dir}/../..")
+cidir="${parent_dir}/ci"
+source "${cidir}/lib.sh"
+
+cargo_deny_file="${script_dir}/action.yaml"
+
+cat cargo-deny-skeleton.yaml.in > "${cargo_deny_file}"
+
+changed_files_status=$(run_get_pr_changed_file_details)
+changed_files_status=$(echo "$changed_files_status" | grep "Cargo\.toml$" || true)
+changed_files=$(echo "$changed_files_status" | awk '{print $NF}' || true)
+
+if [ -z "$changed_files" ]; then
+  cat >> "${cargo_deny_file}" << EOF
+    - run: echo "No Cargo.toml files to check"
+      shell: bash
+EOF
+fi
+
+for path in $changed_files
+do
+    cat >> "${cargo_deny_file}" << EOF
+
+    - name: ${path}
+      continue-on-error: true
+      shell: bash
+      run: |
+        pushd $(dirname ${path})
+        cargo deny check
+        popd
+EOF
+done

.github/cargo-deny-composite-action/cargo-deny-skeleton.yaml.in

@@ -0,0 +1,30 @@
+#
+# Copyright (c) 2022 Red Hat
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+name: 'Cargo Crates Check'
+description: 'Checks every Cargo.toml file using cargo-deny'
+
+env:
+  CARGO_TERM_COLOR: always
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install Rust
+      uses: actions-rs/toolchain@v1
+      with:
+        profile: minimal
+        toolchain: nightly 
+        override: true
+
+    - name: Cache
+      uses: Swatinem/rust-cache@v2
+
+    - name: Install Cargo deny
+      shell: bash
+      run: |
+        which cargo
+        cargo install --locked cargo-deny || true

.github/workflows/PR-wip-checks.yaml

@@ -9,6 +9,10 @@ on:
       - labeled
       - unlabeled
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   pr_wip_check:
     runs-on: ubuntu-latest

.github/workflows/add-backport-label.yaml

@@ -0,0 +1,104 @@
+name: Add backport label
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - edited
+      - labeled
+      - unlabeled
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check-issues:
+    if: ${{ github.event.label.name != 'auto-backport' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code to allow hub to communicate with the project
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        uses: actions/checkout@v4
+
+      - name: Install hub extension script
+        run: |
+          pushd $(mktemp -d) &>/dev/null
+          git clone --single-branch --depth 1 "https://github.com/kata-containers/.github" && cd .github/scripts
+          sudo install hub-util.sh /usr/local/bin
+          popd &>/dev/null
+
+      - name: Determine whether to add label 
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CONTAINS_AUTO_BACKPORT: ${{ contains(github.event.pull_request.labels.*.name, 'auto-backport') }}
+        id: add_label
+        run: |
+          pr=${{ github.event.pull_request.number }}
+          linked_issue_urls=$(hub-util.sh \
+            list-issues-for-pr "$pr" |\
+            grep -v "^\#"  |\
+            cut -d';' -f3 || true)
+          [ -z "$linked_issue_urls" ] && {
+            echo "::error::No linked issues for PR $pr"
+            exit 1
+          }
+          has_bug=false
+          for issue_url in $(echo "$linked_issue_urls")
+          do
+            issue=$(echo "$issue_url"| awk -F\/ '{print $NF}' || true)
+            [ -z "$issue" ] && {
+              echo "::error::Cannot determine issue number from $issue_url for PR $pr"
+              exit 1
+            }
+            labels=$(hub-util.sh list-labels-for-issue "$issue")
+            
+            label_names=$(echo $labels | jq -r '.[].name' || true)
+            if [[ "$label_names" =~ "bug" ]]; then
+              has_bug=true
+              break
+            fi
+          done
+
+          has_backport_needed_label=${{ contains(github.event.pull_request.labels.*.name, 'needs-backport') }}
+          has_no_backport_needed_label=${{ contains(github.event.pull_request.labels.*.name, 'no-backport-needed') }}
+
+          echo "add_backport_label=false" >> $GITHUB_OUTPUT
+          if [ $has_backport_needed_label  = true ] || [ $has_bug  = true ]; then
+            if [[ $has_no_backport_needed_label = false ]]; then
+              echo "add_backport_label=true" >> $GITHUB_OUTPUT
+            fi
+          fi
+
+          # Do not spam comment, only if auto-backport label is going to be newly added.
+          echo "auto_backport_added=$CONTAINS_AUTO_BACKPORT" >> $GITHUB_OUTPUT
+
+      - name: Add comment
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && steps.add_label.outputs.add_backport_label == 'true' && steps.add_label.outputs.auto_backport_added == 'false' }}
+        uses: actions/github-script@v6
+        with:
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: 'This issue has been marked for auto-backporting. Add label(s) backport-to-BRANCHNAME to backport to them'
+            })
+
+      # Allow label to be removed by adding no-backport-needed label
+      - name: Remove auto-backport label
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && steps.add_label.outputs.add_backport_label == 'false' }}
+        uses: andymckay/labeler@e6c4322d0397f3240f0e7e30a33b5c5df2d39e90
+        with:
+          remove-labels: "auto-backport"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Add auto-backport label
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && steps.add_label.outputs.add_backport_label == 'true' }}
+        uses: andymckay/labeler@e6c4322d0397f3240f0e7e30a33b5c5df2d39e90
+        with:
+          add-labels: "auto-backport"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/add-issues-to-project.yaml

@@ -11,6 +11,10 @@ on:
       - opened
       - reopened
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   add-new-issues-to-backlog:
     runs-on: ubuntu-latest
@@ -35,7 +39,7 @@ jobs:
           popd &>/dev/null
 
       - name: Checkout code to allow hub to communicate with the project
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Add issue to issue backlog
         env:

.github/workflows/add-pr-sizing-label.yaml

@@ -0,0 +1,53 @@
+# Copyright (c) 2022 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+name: Add PR sizing label
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - reopened
+      - synchronize
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  add-pr-size-label:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
+
+      - name: Install PR sizing label script
+        run: |
+          # Clone into a temporary directory to avoid overwriting
+          # any existing github directory.
+          pushd $(mktemp -d) &>/dev/null
+          git clone --single-branch --depth 1 "https://github.com/kata-containers/.github" && cd .github/scripts
+          sudo install pr-add-size-label.sh /usr/local/bin
+          popd &>/dev/null
+
+      - name: Add PR sizing label
+        env:
+          GITHUB_TOKEN: ${{ secrets.KATA_GITHUB_ACTIONS_PR_SIZE_TOKEN }}
+        run: |
+          pr=${{ github.event.number }}
+          # Removing man-db, workflow kept failing, fixes: #4480
+          sudo apt -y remove --purge man-db
+          sudo apt -y install diffstat patchutils
+
+          pr-add-size-label.sh -p "$pr"

.github/workflows/auto-backport.yaml

@@ -0,0 +1,33 @@
+on:
+  pull_request_target:
+    types: ["labeled", "closed"]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  backport:
+    name: Backport PR
+    runs-on: ubuntu-latest
+    if: |
+      github.event.pull_request.merged == true
+      && contains(github.event.pull_request.labels.*.name, 'auto-backport')
+      && (
+        (github.event.action == 'labeled' && github.event.label.name == 'auto-backport')
+        || (github.event.action == 'closed')
+      )
+    steps:
+      - name: Backport Action
+        uses: sqren/backport-github-action@v8.9.2
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          auto_backport_label_prefix: backport-to-
+
+      - name: Info log
+        if: ${{ success() }}
+        run: cat /home/runner/.backport/backport.info.log
+        
+      - name: Debug log
+        if: ${{ failure() }}
+        run: cat /home/runner/.backport/backport.debug.log

.github/workflows/basic-ci-amd64.yaml

@@ -0,0 +1,200 @@
+name: CI | Basic amd64 tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-cri-containerd:
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # the all the tests are not flaky, otherwise we'll fail
+      # all the tests due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        containerd_version: ['lts', 'active']
+        vmm: ['clh', 'qemu']
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/cri-containerd/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/cri-containerd/gha-run.sh install-kata kata-artifacts
+
+      - name: Run cri-containerd tests
+        run: bash tests/integration/cri-containerd/gha-run.sh run
+
+  run-containerd-stability:
+    strategy:
+      fail-fast: false
+      matrix:
+        containerd_version: ['lts', 'active']
+        vmm: ['clh', 'qemu']
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/stability/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/stability/gha-run.sh install-kata kata-artifacts
+
+      - name: Run containerd-stability tests
+        run: bash tests/stability/gha-run.sh run
+
+  run-nydus:
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # the all the tests are not flaky, otherwise we'll fail
+      # all the tests due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        containerd_version: ['lts', 'active']
+        vmm: ['clh', 'qemu', 'dragonball']
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/nydus/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/nydus/gha-run.sh install-kata kata-artifacts
+
+      - name: Run nydus tests
+        run: bash tests/integration/nydus/gha-run.sh run
+
+  run-runk:
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINERD_VERSION: lts
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/runk/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/runk/gha-run.sh install-kata kata-artifacts
+
+      - name: Run tracing tests
+        run: bash tests/integration/runk/gha-run.sh run
+
+  run-vfio:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm: ['clh', 'qemu']
+    runs-on: garm-ubuntu-2304
+    env:
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/functional/vfio/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Run vfio tests
+        timeout-minutes: 15
+        run: bash tests/functional/vfio/gha-run.sh run

.github/workflows/build-kata-static-tarball-amd64.yaml

@@ -0,0 +1,137 @@
+name: CI | Build kata-static tarball for amd64
+on:
+  workflow_call:
+    inputs:
+      stage:
+        required: false
+        type: string
+        default: test
+      tarball-suffix:
+        required: false
+        type: string
+      push-to-registry:
+        required: false
+        type: string
+        default: no
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  build-asset:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        asset:
+          - agent
+          - agent-ctl
+          - cloud-hypervisor
+          - cloud-hypervisor-glibc
+          - firecracker
+          - kata-ctl
+          - kernel
+          - kernel-sev
+          - kernel-dragonball-experimental
+          - kernel-tdx-experimental
+          - kernel-nvidia-gpu
+          - kernel-nvidia-gpu-snp
+          - kernel-nvidia-gpu-tdx-experimental
+          - log-parser-rs
+          - nydus
+          - ovmf
+          - ovmf-sev
+          - qemu
+          - qemu-snp-experimental
+          - qemu-tdx-experimental
+          - rootfs-image
+          - rootfs-image-tdx
+          - rootfs-initrd
+          - rootfs-initrd-mariner
+          - rootfs-initrd-sev
+          - runk
+          - shim-v2
+          - tdvf
+          - trace-forwarder
+          - virtiofsd
+        stage:
+          - ${{ inputs.stage }}
+        exclude:
+          - asset: agent
+            stage: release
+          - asset: cloud-hypervisor-glibc
+            stage: release
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-amd64${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: build-asset
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts versions.yaml
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error

.github/workflows/build-kata-static-tarball-arm64.yaml

@@ -0,0 +1,120 @@
+name: CI | Build kata-static tarball for arm64
+on:
+  workflow_call:
+    inputs:
+      stage:
+        required: false
+        type: string
+        default: test
+      tarball-suffix:
+        required: false
+        type: string
+      push-to-registry:
+        required: false
+        type: string
+        default: no
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  build-asset:
+    runs-on: arm64-builder
+    strategy:
+      matrix:
+        asset:
+          - cloud-hypervisor
+          - firecracker
+          - kernel
+          - kernel-dragonball-experimental
+          - nydus
+          - qemu
+          - rootfs-image
+          - rootfs-initrd
+          - shim-v2
+          - virtiofsd
+        stage:
+          - ${{ inputs.stage }}
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-arm64${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: arm64-builder
+    needs: build-asset
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts-arm64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts versions.yaml
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball-arm64${{ inputs.tarball-suffix }}
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error

.github/workflows/build-kata-static-tarball-s390x.yaml

@@ -0,0 +1,117 @@
+name: CI | Build kata-static tarball for s390x
+on:
+  workflow_call:
+    inputs:
+      stage:
+        required: false
+        type: string
+        default: test
+      tarball-suffix:
+        required: false
+        type: string
+      push-to-registry:
+        required: false
+        type: string
+        default: no
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  build-asset:
+    runs-on: s390x
+    strategy:
+      matrix:
+        asset:
+          - kernel
+          - qemu
+          - rootfs-image
+          - rootfs-initrd
+          - shim-v2
+          - virtiofsd
+        stage:
+          - ${{ inputs.stage }}
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+          sudo chown -R $(id -u):$(id -g) "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-s390x${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: s390x
+    needs: build-asset
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts-s390x${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts versions.yaml
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball-s390x${{ inputs.tarball-suffix }}
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error

.github/workflows/cargo-deny-runner.yaml

@@ -0,0 +1,31 @@
+name: Cargo Crates Check Runner
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+    paths-ignore: [ '**.md', '**.png', '**.jpg', '**.jpeg', '**.svg', '/docs/**' ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  cargo-deny-runner:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Code
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        uses: actions/checkout@v4
+      - name: Generate Action
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        run: bash cargo-deny-generator.sh
+        working-directory: ./.github/cargo-deny-composite-action/
+        env:
+          GOPATH: ${{ runner.workspace }}/kata-containers
+      - name: Run Action
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        uses: ./.github/cargo-deny-composite-action

.github/workflows/ci-nightly.yaml

@@ -0,0 +1,19 @@
+name: Kata Containers Nightly CI
+on:
+  schedule:
+    - cron: '0 0 * * *'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  kata-containers-ci-on-push:
+    uses: ./.github/workflows/ci.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      pr-number: "nightly"
+      tag: ${{ github.sha }}-nightly
+      target-branch: ${{ github.ref_name }}
+    secrets: inherit

.github/workflows/ci-on-push.yaml

@@ -0,0 +1,32 @@
+name: Kata Containers CI
+on:
+  pull_request_target:
+    branches:
+      - 'main'
+      - 'stable-*'
+    types:
+      # Adding 'labeled' to the list of activity types that trigger this event
+      # (default: opened, synchronize, reopened) so that we can run this
+      # workflow when the 'ok-to-test' label is added.
+      # Reference: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target
+      - opened
+      - synchronize
+      - reopened
+      - labeled
+    paths-ignore:
+      - 'docs/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  kata-containers-ci-on-push:
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'ok-to-test') }}
+    uses: ./.github/workflows/ci.yaml
+    with:
+      commit-hash: ${{ github.event.pull_request.head.sha }}
+      pr-number: ${{ github.event.pull_request.number }}
+      tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
+      target-branch: ${{ github.event.pull_request.base.ref }}
+    secrets: inherit

.github/workflows/ci.yaml

@@ -0,0 +1,185 @@
+name: Run the Kata Containers CI
+on:
+  workflow_call:
+    inputs:
+      commit-hash:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  build-kata-static-tarball-amd64:
+    uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+
+  publish-kata-deploy-payload-amd64:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/publish-kata-deploy-payload-amd64.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
+
+  build-and-publish-tee-confidential-unencrypted-image:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to Kata Containers ghcr.io
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker build and push
+        uses: docker/build-push-action@v4
+        with:
+          tags: ghcr.io/kata-containers/test-images:unencrypted-${{ inputs.pr-number }}
+          push: true
+          context: tests/integration/kubernetes/runtimeclass_workloads/confidential/unencrypted/
+          platforms: linux/amd64, linux/s390x
+          file: tests/integration/kubernetes/runtimeclass_workloads/confidential/unencrypted/Dockerfile
+
+  run-docker-tests-on-garm:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/run-docker-tests-on-garm.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+
+  run-nerdctl-tests-on-garm:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/run-nerdctl-tests-on-garm.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+
+  run-kata-deploy-tests-on-aks:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-kata-deploy-tests-on-aks.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
+
+  run-kata-deploy-tests-on-garm:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-kata-deploy-tests-on-garm.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
+
+  run-kata-monitor-tests:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/run-kata-monitor-tests.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+
+  run-k8s-tests-on-aks:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-k8s-tests-on-aks.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
+
+  run-k8s-tests-on-garm:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-k8s-tests-on-garm.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
+
+  run-k8s-tests-with-crio-on-garm:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-k8s-tests-with-crio-on-garm.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
+
+  run-kata-coco-tests:
+    needs: [publish-kata-deploy-payload-amd64, build-and-publish-tee-confidential-unencrypted-image]
+    uses: ./.github/workflows/run-kata-coco-tests.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+
+  run-metrics-tests:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/run-metrics.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+
+  run-basic-amd64-tests:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/basic-ci-amd64.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}

.github/workflows/commit-message-check.yaml

@@ -6,11 +6,15 @@ on:
       - reopened
       - synchronize
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   error_msg: |+
     See the document below for help on formatting commits for the project.
 
-    https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md#patch-format
+    https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md#patch-format
 
 jobs:
   commit-message-check:
@@ -62,8 +66,12 @@ jobs:
         #   to be specified at the start of the regex as the action is passed
         #   the entire commit message.
         #
+        # - This check will pass if the commit message only contains a subject
+        #   line, as other body message properties are enforced elsewhere.
+        #
         # - Body lines *can* be longer than the maximum if they start
-        #   with a non-alphabetic character.
+        #   with a non-alphabetic character or if there is no whitespace in
+        #   the line.
         #
         #   This allows stack traces, log files snippets, emails, long URLs,
         #   etc to be specified. Some of these naturally "work" as they start
@@ -74,8 +82,8 @@ jobs:
         #
         # - A SoB comment can be any length (as it is unreasonable to penalise
         #   people with long names/email addresses :)
-        pattern: '^.+(\n([a-zA-Z].{0,149}|[^a-zA-Z\n].*|Signed-off-by:.*|))+$'
-        error: 'Body line too long (max 72)'
+        pattern: '(^[^\n]+$|^.+(\n([a-zA-Z].{0,150}|[^a-zA-Z\n].*|[^\s\n]*|Signed-off-by:.*|))+$)'
+        error: 'Body line too long (max 150)'
         post_error: ${{ env.error_msg }}
 
     - name: Check Fixes

.github/workflows/darwin-tests.yaml

@@ -5,21 +5,22 @@ on:
       - edited
       - reopened
       - synchronize
+    paths-ignore: [ '**.md', '**.png', '**.jpg', '**.jpeg', '**.svg', '/docs/**' ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 name: Darwin tests
 jobs:
   test:
-    strategy:
-      matrix:
-        go-version: [1.16.x, 1.17.x]
-        os: [macos-latest]
-    runs-on: ${{ matrix.os }}
+    runs-on: macos-latest
     steps:
     - name: Install Go
       uses: actions/setup-go@v2
       with:
-        go-version: ${{ matrix.go-version }}
+        go-version: 1.19.3
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
     - name: Build utils
       run: ./ci/darwin-test.sh

.github/workflows/docs-url-alive-check.yaml

@@ -0,0 +1,37 @@
+on:
+  schedule:
+    - cron:  '0 23 * * 0'
+
+name: Docs URL Alive Check
+jobs:
+  test:
+    runs-on: ubuntu-20.04
+    # don't run this action on forks
+    if: github.repository_owner == 'kata-containers'
+    env:
+      target_branch: ${{ github.base_ref }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.19.3
+      env:
+        GOPATH: ${{ runner.workspace }}/kata-containers
+    - name: Set env
+      run: |
+        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
+        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        path: ./src/github.com/${{ github.repository }}
+    - name: Setup
+      run: |
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
+      env:
+        GOPATH: ${{ runner.workspace }}/kata-containers
+    # docs url alive check
+    - name: Docs URL Alive Check
+      run: |
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && make docs-url-alive-check

.github/workflows/kata-deploy-push.yaml

@@ -1,83 +0,0 @@
-name: kata deploy build
-
-on: 
-  pull_request:
-    types:
-      - opened
-      - edited
-      - reopened
-      - synchronize
-    paths:
-      - tools/**
-      - versions.yaml
-
-jobs:
-  build-asset:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        asset:
-          - kernel
-          - shim-v2
-          - qemu
-          - cloud-hypervisor
-          - firecracker
-          - rootfs-image
-          - rootfs-initrd
-    steps:
-      - uses: actions/checkout@v2
-      - name: Install docker
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          curl -fsSL https://test.docker.com -o test-docker.sh
-          sh test-docker.sh
-
-      - name: Build ${{ matrix.asset }}
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r --preserve=all "${build_dir}" "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-
-      - name: store-artifact ${{ matrix.asset }}
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: ubuntu-latest
-    needs: build-asset
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifacts
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-artifacts
-          path: build
-      - name: merge-artifacts
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          make merge-builds
-      - name: store-artifacts
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-static-tarball
-          path: kata-static.tar.xz
-
-  make-kata-tarball:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: make kata-tarball
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          make kata-tarball
-          sudo make install-tarball

.github/workflows/kata-deploy-test.yaml

@@ -1,147 +0,0 @@
-on:
-  issue_comment:
-    types: [created, edited]
-
-name: test-kata-deploy
-
-jobs:
-  check-comment-and-membership:
-    runs-on: ubuntu-latest
-    if: |
-      github.event.issue.pull_request
-      && github.event_name == 'issue_comment'
-      && github.event.action == 'created'
-      && startsWith(github.event.comment.body, '/test_kata_deploy')
-    steps:
-      - name: Check membership
-        uses: kata-containers/is-organization-member@1.0.1
-        id: is_organization_member
-        with:
-          organization: kata-containers
-          username: ${{ github.event.comment.user.login }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Fail if not member
-        run: |
-          result=${{ steps.is_organization_member.outputs.result }}
-          if [ $result == false ]; then
-              user=${{ github.event.comment.user.login }}
-              echo Either ${user} is not part of the kata-containers organization
-              echo or ${user} has its Organization Visibility set to Private at
-              echo https://github.com/orgs/kata-containers/people?query=${user}
-              echo 
-              echo Ensure you change your Organization Visibility to Public and
-              echo trigger the test again.
-              exit 1
-          fi
-
-  build-asset:
-    runs-on: ubuntu-latest
-    needs: check-comment-and-membership
-    strategy:
-      matrix:
-        asset:
-          - cloud-hypervisor
-          - firecracker
-          - kernel
-          - qemu
-          - rootfs-image
-          - rootfs-initrd
-          - shim-v2
-    steps:
-      - name: get-PR-ref
-        id: get-PR-ref
-        run: |
-            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
-            echo "reference for PR: " ${ref}
-            echo "##[set-output name=pr-ref;]${ref}"
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
-
-      - name: Install docker
-        run: |
-          curl -fsSL https://test.docker.com -o test-docker.sh
-          sh test-docker.sh
-
-      - name: Build ${{ matrix.asset }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: ubuntu-latest
-    needs: build-asset
-    steps:
-      - name: get-PR-ref
-        id: get-PR-ref
-        run: |
-            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
-            echo "reference for PR: " ${ref}
-            echo "##[set-output name=pr-ref;]${ref}"
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
-      - name: get-artifacts
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-static-tarball
-          path: kata-static.tar.xz
-
-  kata-deploy:
-    needs: create-kata-tarball
-    runs-on: ubuntu-latest
-    steps:
-      - name: get-PR-ref
-        id: get-PR-ref
-        run: |
-            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
-            echo "reference for PR: " ${ref}
-            echo "##[set-output name=pr-ref;]${ref}"
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-static-tarball
-      - name: build-and-push-kata-deploy-ci
-        id: build-and-push-kata-deploy-ci
-        run: |
-          PR_SHA=$(git log --format=format:%H -n1)
-          mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
-          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t quay.io/kata-containers/kata-deploy-ci:$PR_SHA $GITHUB_WORKSPACE/tools/packaging/kata-deploy
-          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
-          docker push quay.io/kata-containers/kata-deploy-ci:$PR_SHA
-          mkdir -p packaging/kata-deploy
-          ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
-          echo "::set-output name=PKG_SHA::${PR_SHA}"
-      - name: test-kata-deploy-ci-in-aks
-        uses: ./packaging/kata-deploy/action
-        with:
-          packaging-sha: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
-        env:
-          PKG_SHA: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
-          AZ_APPID: ${{ secrets.AZ_APPID }}
-          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
-          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
-          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}

.github/workflows/kata-runtime-classes-sync.yaml

@@ -0,0 +1,36 @@
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  kata-deploy-runtime-classes-check:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+    - name: Ensure the split out runtime classes match the all-in-one file
+      run: |
+        pushd tools/packaging/kata-deploy/runtimeclasses/
+        echo "::group::Combine runtime classes"
+        for runtimeClass in `find . -type f \( -name "*.yaml" -and -not -name "kata-runtimeClasses.yaml" \) | sort`; do
+            echo "Adding ${runtimeClass} to the resultingRuntimeClasses.yaml"
+            cat ${runtimeClass} >> resultingRuntimeClasses.yaml;
+        done
+        echo "::endgroup::"
+        echo "::group::Displaying the content of resultingRuntimeClasses.yaml"
+        cat resultingRuntimeClasses.yaml
+        echo "::endgroup::"
+        echo ""
+        echo "::group::Displaying the content of kata-runtimeClasses.yaml"
+        cat kata-runtimeClasses.yaml
+        echo "::endgroup::"
+        echo ""
+        diff resultingRuntimeClasses.yaml kata-runtimeClasses.yaml

.github/workflows/move-issues-to-in-progress.yaml

@@ -38,7 +38,17 @@ jobs:
 
       - name: Checkout code to allow hub to communicate with the project
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
 
       - name: Move issue to "In progress"
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}

.github/workflows/payload-after-push.yaml

@@ -0,0 +1,91 @@
+name: CI | Publish Kata Containers payload
+on:
+  push:
+    branches:
+      - main
+      - stable-*
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-assets-amd64:
+    uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      push-to-registry: yes
+      target-branch: ${{ github.ref_name }}
+    secrets: inherit
+
+  build-assets-arm64:
+    uses: ./.github/workflows/build-kata-static-tarball-arm64.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      push-to-registry: yes
+      target-branch: ${{ github.ref_name }}
+    secrets: inherit
+
+  build-assets-s390x:
+    uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      push-to-registry: yes
+      target-branch: ${{ github.ref_name }}
+    secrets: inherit
+
+  publish-kata-deploy-payload-amd64:
+    needs: build-assets-amd64
+    uses: ./.github/workflows/publish-kata-deploy-payload-amd64.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      registry: quay.io
+      repo: kata-containers/kata-deploy-ci
+      tag: kata-containers-amd64
+      target-branch: ${{ github.ref_name }}
+    secrets: inherit
+
+  publish-kata-deploy-payload-arm64:
+    needs: build-assets-arm64
+    uses: ./.github/workflows/publish-kata-deploy-payload-arm64.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      registry: quay.io
+      repo: kata-containers/kata-deploy-ci
+      tag: kata-containers-arm64
+      target-branch: ${{ github.ref_name }}
+    secrets: inherit
+
+  publish-kata-deploy-payload-s390x:
+    needs: build-assets-s390x
+    uses: ./.github/workflows/publish-kata-deploy-payload-s390x.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      registry: quay.io
+      repo: kata-containers/kata-deploy-ci
+      tag: kata-containers-s390x
+      target-branch: ${{ github.ref_name }}
+    secrets: inherit
+
+  publish-manifest:
+    runs-on: ubuntu-latest
+    needs: [publish-kata-deploy-payload-amd64, publish-kata-deploy-payload-arm64, publish-kata-deploy-payload-s390x]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Push multi-arch manifest
+        run: |
+          docker manifest create quay.io/kata-containers/kata-deploy-ci:kata-containers-latest \
+          --amend quay.io/kata-containers/kata-deploy-ci:kata-containers-amd64 \
+          --amend quay.io/kata-containers/kata-deploy-ci:kata-containers-arm64 \
+          --amend quay.io/kata-containers/kata-deploy-ci:kata-containers-s390x
+          docker manifest push quay.io/kata-containers/kata-deploy-ci:kata-containers-latest

.github/workflows/publish-kata-deploy-payload-amd64.yaml

@@ -0,0 +1,66 @@
+name: CI | Publish kata-deploy payload for amd64
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  kata-payload:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.registry == 'quay.io' }}
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Login to Kata Containers ghcr.io
+        if: ${{ inputs.registry == 'ghcr.io' }}
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz \
+          ${{ inputs.registry }}/${{ inputs.repo }} ${{ inputs.tag }}

.github/workflows/publish-kata-deploy-payload-arm64.yaml

@@ -0,0 +1,71 @@
+name: CI | Publish kata-deploy payload for arm64
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  kata-payload:
+    runs-on: arm64-builder
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-arm64${{ inputs.tarball-suffix }}
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.registry == 'quay.io' }}
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Login to Kata Containers ghcr.io
+        if: ${{ inputs.registry == 'ghcr.io' }}
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz \
+          ${{ inputs.registry }}/${{ inputs.repo }} ${{ inputs.tag }}
+

.github/workflows/publish-kata-deploy-payload-s390x.yaml

@@ -0,0 +1,70 @@
+name: CI | Publish kata-deploy payload for s390x
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  kata-payload:
+    runs-on: s390x
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-s390x${{ inputs.tarball-suffix }}
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.registry == 'quay.io' }}
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Login to Kata Containers ghcr.io
+        if: ${{ inputs.registry == 'ghcr.io' }}
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz \
+          ${{ inputs.registry }}/${{ inputs.repo }} ${{ inputs.tag }}

.github/workflows/release-amd64.yaml

@@ -0,0 +1,53 @@
+name: Publish Kata release artifacts for amd64
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-kata-static-tarball-amd64:
+    uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
+    with:
+      stage: release
+
+  kata-deploy:
+    needs: build-kata-static-tarball-amd64
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to Kata Containers docker.io
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64
+
+      - name: build-and-push-kata-deploy-ci-amd64
+        id: build-and-push-kata-deploy-ci-amd64
+        run: |
+          # We need to do such trick here as the format of the $GITHUB_REF 
+          # is "refs/tags/<tag>"
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tags=($tag)
+          tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
+          for tag in ${tags[@]}; do
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
+          done

.github/workflows/release-arm64.yaml

@@ -0,0 +1,53 @@
+name: Publish Kata release artifacts for arm64
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-kata-static-tarball-arm64:
+    uses: ./.github/workflows/build-kata-static-tarball-arm64.yaml
+    with:
+      stage: release
+
+  kata-deploy:
+    needs: build-kata-static-tarball-arm64
+    runs-on: arm64-builder
+    steps:
+      - name: Login to Kata Containers docker.io
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-arm64
+
+      - name: build-and-push-kata-deploy-ci-arm64
+        id: build-and-push-kata-deploy-ci-arm64
+        run: |
+          # We need to do such trick here as the format of the $GITHUB_REF 
+          # is "refs/tags/<tag>"
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tags=($tag)
+          tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
+          for tag in ${tags[@]}; do
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
+          done

.github/workflows/release-s390x.yaml

@@ -0,0 +1,53 @@
+name: Publish Kata release artifacts for s390x
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-kata-static-tarball-s390x:
+    uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml
+    with:
+      stage: release
+
+  kata-deploy:
+    needs: build-kata-static-tarball-s390x
+    runs-on: s390x
+    steps:
+      - name: Login to Kata Containers docker.io
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+
+      - name: build-and-push-kata-deploy-ci-s390x
+        id: build-and-push-kata-deploy-ci-s390x
+        run: |
+          # We need to do such trick here as the format of the $GITHUB_REF 
+          # is "refs/tags/<tag>"
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tags=($tag)
+          tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
+          for tag in ${tags[@]}; do
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
+          done

.github/workflows/release.yaml

@@ -1,160 +1,156 @@
-name: Publish Kata 2.x release artifacts
+name: Publish Kata release artifacts
 on:
   push:
     tags:
-     - '2.*'
+      - '[0-9]+.[0-9]+.[0-9]+*'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 jobs:
-  build-asset:
+  build-and-push-assets-amd64:
+    uses: ./.github/workflows/release-amd64.yaml
+    with:
+      target-arch: amd64
+    secrets: inherit
+
+  build-and-push-assets-arm64:
+    uses: ./.github/workflows/release-arm64.yaml
+    with:
+      target-arch: arm64
+    secrets: inherit
+
+  build-and-push-assets-s390x:
+    uses: ./.github/workflows/release-s390x.yaml
+    with:
+      target-arch: s390x
+    secrets: inherit
+
+  publish-multi-arch-images:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        asset:
-          - cloud-hypervisor
-          - firecracker
-          - kernel
-          - qemu
-          - rootfs-image
-          - rootfs-initrd
-          - shim-v2
+    needs: [build-and-push-assets-amd64, build-and-push-assets-arm64, build-and-push-assets-s390x]
     steps:
-      - uses: actions/checkout@v2
-      - name: Install docker
-        run: |
-          curl -fsSL https://test.docker.com -o test-docker.sh
-          sh test-docker.sh
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
-      - name: Build ${{ matrix.asset }}
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh --build="${KATA_ASSET}"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+      - name: Login to Kata Containers docker.io
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
 
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v2
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
         with:
-          name: kata-artifacts
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          if-no-files-found: error
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-  create-kata-tarball:
-    runs-on: ubuntu-latest
-    needs: build-asset
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifacts
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-static-tarball
-          path: kata-static.tar.xz
-
-  kata-deploy:
-    needs: create-kata-tarball
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-static-tarball
-      - name: build-and-push-kata-deploy-ci
-        id: build-and-push-kata-deploy-ci
-        run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          pushd $GITHUB_WORKSPACE
-          git checkout $tag
-          pkg_sha=$(git rev-parse HEAD)
-          popd
-          mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
-          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:$pkg_sha -t quay.io/kata-containers/kata-deploy-ci:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
-          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
-          docker push katadocker/kata-deploy-ci:$pkg_sha
-          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
-          docker push quay.io/kata-containers/kata-deploy-ci:$pkg_sha
-          mkdir -p packaging/kata-deploy
-          ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
-          echo "::set-output name=PKG_SHA::${pkg_sha}"
-      - name: test-kata-deploy-ci-in-aks
-        uses: ./packaging/kata-deploy/action
-        with:
-          packaging-sha: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
-        env:
-          PKG_SHA: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
-          AZ_APPID: ${{ secrets.AZ_APPID }}
-          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
-          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
-          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
-      - name: push-tarball
+      - name: Push multi-arch manifest
         run: |
           # tag the container image we created and push to DockerHub
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
           tags=($tag)
           tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
-          for tag in ${tags[@]}; do \
-            docker tag katadocker/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} katadocker/kata-deploy:${tag} && \
-            docker tag quay.io/kata-containers/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} quay.io/kata-containers/kata-deploy:${tag} && \
-            docker push katadocker/kata-deploy:${tag} && \
-            docker push quay.io/kata-containers/kata-deploy:${tag}; \
+          # push to quay.io and docker.io
+          for tag in ${tags[@]}; do
+            docker manifest create quay.io/kata-containers/kata-deploy:${tag} \
+              --amend quay.io/kata-containers/kata-deploy:${tag}-amd64 \
+              --amend quay.io/kata-containers/kata-deploy:${tag}-arm64 \
+              --amend quay.io/kata-containers/kata-deploy:${tag}-s390x
+
+            docker manifest create docker.io/katadocker/kata-deploy:${tag} \
+              --amend docker.io/katadocker/kata-deploy:${tag}-amd64 \
+              --amend docker.io/katadocker/kata-deploy:${tag}-arm64 \
+              --amend docker.io/katadocker/kata-deploy:${tag}-s390x
+
+            docker manifest push quay.io/kata-containers/kata-deploy:${tag}
+            docker manifest push docker.io/katadocker/kata-deploy:${tag}
           done
 
-  upload-static-tarball:
-    needs: kata-deploy
+  upload-multi-arch-static-tarball:
+    needs: publish-multi-arch-images
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: download-artifacts
-        uses: actions/download-artifact@v2
+      - uses: actions/checkout@v4
+
+      - name: download-artifacts-amd64
+        uses: actions/download-artifact@v3
         with:
-          name: kata-static-tarball
-      - name: install hub
-        run: |
-          HUB_VER=$(curl -s "https://api.github.com/repos/github/hub/releases/latest" | jq -r .tag_name | sed 's/^v//')
-          wget -q -O- https://github.com/github/hub/releases/download/v$HUB_VER/hub-linux-amd64-$HUB_VER.tgz | \
-          tar xz --strip-components=2 --wildcards '*/bin/hub' && sudo mv hub /usr/local/bin/hub
-      - name: push static tarball to github
+          name: kata-static-tarball-amd64
+      - name: push amd64 static tarball to github
         run: |
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          tarball="kata-static-$tag-x86_64.tar.xz"
+          tarball="kata-static-$tag-amd64.tar.xz"
           mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
           pushd $GITHUB_WORKSPACE
           echo "uploading asset '${tarball}' for tag: ${tag}"
-          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}"
+          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} gh release upload "${tag}" "${tarball}"
+          popd
+
+      - name: download-artifacts-arm64
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-arm64
+      - name: push arm64 static tarball to github
+        run: |
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tarball="kata-static-$tag-arm64.tar.xz"
+          mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
+          pushd $GITHUB_WORKSPACE
+          echo "uploading asset '${tarball}' for tag: ${tag}"
+          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} gh release upload "${tag}" "${tarball}"
+          popd
+
+      - name: download-artifacts-s390x
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+      - name: push s390x static tarball to github
+        run: |
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tarball="kata-static-$tag-s390x.tar.xz"
+          mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
+          pushd $GITHUB_WORKSPACE
+          echo "uploading asset '${tarball}' for tag: ${tag}"
+          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} gh release upload "${tag}" "${tarball}"
+          popd
+
+  upload-versions-yaml:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: upload versions.yaml
+        env:
+          GITHUB_TOKEN: ${{ secrets.GIT_UPLOAD_TOKEN }}
+        run: |
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          pushd $GITHUB_WORKSPACE
+          versions_file="kata-containers-$tag-versions.yaml"
+          cp versions.yaml ${versions_file}
+          gh release upload "${tag}" "${versions_file}"
           popd
 
   upload-cargo-vendored-tarball:
-    needs: upload-static-tarball
+    needs: upload-multi-arch-static-tarball
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: generate-and-upload-tarball
         run: |
-          pushd $GITHUB_WORKSPACE/src/agent
-          cargo vendor >> .cargo/config
-          popd
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
           tarball="kata-containers-$tag-vendor.tar.gz"
           pushd $GITHUB_WORKSPACE
-          tar -cvzf "${tarball}" src/agent/.cargo/config src/agent/vendor
-          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}" 
+          bash -c "tools/packaging/release/generate_vendor.sh ${tarball}"
+          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} gh release upload "${tag}" "${tarball}"
           popd
 
   upload-libseccomp-tarball:
     needs: upload-cargo-vendored-tarball
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: download-and-upload-tarball
         env:
           GITHUB_TOKEN: ${{ secrets.GIT_UPLOAD_TOKEN }}
@@ -174,6 +170,6 @@ jobs:
           # "-m" option should be empty to re-use the existing release title
           # without opening a text editor.
           # For the details, check https://hub.github.com/hub-release.1.html.
-          hub release edit -m "" -a "${tarball}" "${tag}"
-          hub release edit -m "" -a "${asc}" "${tag}"
+          gh release upload "${tag}" "${tarball}"
+          gh release upload "${tag}" "${asc}"
           popd

.github/workflows/require-pr-porting-labels.yaml

@@ -15,6 +15,10 @@ on:
     branches:
       - main
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   check-pr-porting-labels:
     runs-on: ubuntu-latest
@@ -32,7 +36,17 @@ jobs:
 
       - name: Checkout code to allow hub to communicate with the project
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
 
       - name: Install porting checker script
         run: |

.github/workflows/run-docker-tests-on-garm.yaml

@@ -0,0 +1,56 @@
+name: CI | Run docker integration tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-docker-tests:
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # all the tests are not flaky, otherwise we'll fail them
+      # all due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        vmm:
+          - clh
+          - qemu
+    runs-on: garm-ubuntu-2304-smaller
+    env:
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/docker/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/docker/gha-run.sh install-kata kata-artifacts
+
+      - name: Run docker smoke test
+        timeout-minutes: 5
+        run: bash tests/integration/docker/gha-run.sh run

.github/workflows/run-k8s-tests-on-aks.yaml

@@ -0,0 +1,98 @@
+name: CI | Run kubernetes tests on AKS
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-k8s-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        host_os:
+          - ubuntu
+        vmm:
+          - clh
+          - dragonball
+          - qemu
+        instance-type:
+          - small
+          - normal
+        include:
+          - host_os: cbl-mariner
+            vmm: clh
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HOST_OS: ${{ matrix.host_os }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: "vanilla"
+      USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Download Azure CLI
+        run: bash tests/integration/kubernetes/gha-run.sh install-azure-cli
+
+      - name: Log into the Azure account
+        run: bash tests/integration/kubernetes/gha-run.sh login-azure
+        env:
+          AZ_APPID: ${{ secrets.AZ_APPID }}
+          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
+          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
+
+      - name: Create AKS cluster
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh create-cluster
+
+      - name: Install `bats`
+        run: bash tests/integration/kubernetes/gha-run.sh install-bats
+
+      - name: Install `kubectl`
+        run: bash tests/integration/kubernetes/gha-run.sh install-kubectl
+
+      - name: Download credentials for the Kubernetes CLI to use them
+        run: bash tests/integration/kubernetes/gha-run.sh get-cluster-credentials
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-aks
+      
+      - name: Run tests
+        timeout-minutes: 60
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Delete AKS cluster
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh delete-cluster

.github/workflows/run-k8s-tests-on-garm.yaml

@@ -0,0 +1,88 @@
+name: CI | Run kubernetes tests on GARM
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-k8s-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - clh #cloud-hypervisor
+          - fc #firecracker
+          - qemu
+        snapshotter:
+          - devmapper
+        k8s:
+          - k3s
+        instance:
+          - garm-ubuntu-2004
+          - garm-ubuntu-2004-smaller
+        include:
+          - instance: garm-ubuntu-2004
+            instance-type: normal
+          - instance: garm-ubuntu-2004-smaller
+            instance-type: small
+    runs-on: ${{ matrix.instance }}
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: ${{ matrix.k8s }}
+      SNAPSHOTTER: ${{ matrix.snapshotter }}
+      USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy ${{ matrix.k8s }}
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s
+
+      - name: Configure the ${{ matrix.snapshotter }} snapshotter
+        run: bash tests/integration/kubernetes/gha-run.sh configure-snapshotter
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-garm
+
+      - name: Install `bats`
+        run: bash tests/integration/kubernetes/gha-run.sh install-bats
+  
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+  
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-garm

.github/workflows/run-k8s-tests-with-crio-on-garm.yaml

@@ -0,0 +1,86 @@
+name: CI | Run kubernetes tests, using CRI-O, on GARM
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-k8s-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu
+        k8s:
+          - k0s
+        instance:
+          - garm-ubuntu-2004
+          - garm-ubuntu-2004-smaller
+        include:
+          - instance: garm-ubuntu-2004
+            instance-type: normal
+          - instance: garm-ubuntu-2004-smaller
+            instance-type: small
+          - k8s: k0s
+            k8s-extra-params: '--cri-socket remote:unix:///var/run/crio/crio.sock --kubelet-extra-args --cgroup-driver="systemd"'
+    runs-on: ${{ matrix.instance }}
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: ${{ matrix.k8s }}
+      KUBERNETES_EXTRA_PARAMS: ${{ matrix.k8s-extra-params }}
+      USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Configure CRI-O
+        run: bash tests/integration/kubernetes/gha-run.sh setup-crio
+
+      - name: Deploy ${{ matrix.k8s }}
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-garm
+
+      - name: Install `bats`
+        run: bash tests/integration/kubernetes/gha-run.sh install-bats
+  
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+  
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-garm

.github/workflows/run-kata-coco-tests.yaml

@@ -0,0 +1,176 @@
+name: CI | Run kata coco tests
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-kata-deploy-tests-on-tdx:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-tdx
+    runs-on: tdx
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: "k3s"
+      USING_NFD: "true"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Run tests
+        run: bash tests/functional/kata-deploy/gha-run.sh run-tests
+
+  run-k8s-tests-on-tdx:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-tdx
+    runs-on: tdx
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: "k3s"
+      USING_NFD: "true"
+      K8S_TEST_HOST_TYPE: "baremetal"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-tdx
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-tdx
+
+  run-k8s-tests-on-sev:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-sev
+    runs-on: sev
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBECONFIG: /home/kata/.kube/config
+      KUBERNETES: "vanilla"
+      USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: "baremetal"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-sev
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-sev
+
+  run-k8s-tests-sev-snp:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-snp
+    runs-on: sev-snp
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBECONFIG: /home/kata/.kube/config
+      KUBERNETES: "vanilla"
+      USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: "baremetal"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-snp
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-snp

.github/workflows/run-kata-deploy-tests-on-aks.yaml

@@ -0,0 +1,89 @@
+name: CI | Run kata-deploy tests on AKS
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-kata-deploy-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        host_os:
+          - ubuntu
+        vmm:
+          - clh
+          - dragonball
+          - qemu
+        include:
+          - host_os: cbl-mariner
+            vmm: clh
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HOST_OS: ${{ matrix.host_os }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: "vanilla"
+      USING_NFD: "false"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Download Azure CLI
+        run: bash tests/functional/kata-deploy/gha-run.sh install-azure-cli
+
+      - name: Log into the Azure account
+        run: bash tests/functional/kata-deploy/gha-run.sh login-azure
+        env:
+          AZ_APPID: ${{ secrets.AZ_APPID }}
+          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
+          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
+
+      - name: Create AKS cluster
+        timeout-minutes: 10
+        run: bash tests/functional/kata-deploy/gha-run.sh create-cluster
+
+      - name: Install `bats`
+        run: bash tests/functional/kata-deploy/gha-run.sh install-bats
+
+      - name: Install `kubectl`
+        run: bash tests/functional/kata-deploy/gha-run.sh install-kubectl
+
+      - name: Download credentials for the Kubernetes CLI to use them
+        run: bash tests/functional/kata-deploy/gha-run.sh get-cluster-credentials
+
+      - name: Run tests
+        run: bash tests/functional/kata-deploy/gha-run.sh run-tests
+      
+      - name: Delete AKS cluster
+        if: always()
+        run: bash tests/functional/kata-deploy/gha-run.sh delete-cluster

.github/workflows/run-kata-deploy-tests-on-garm.yaml

@@ -0,0 +1,65 @@
+name: CI | Run kata-deploy tests on GARM
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-kata-deploy-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - clh
+          - qemu
+        k8s:
+          - k0s
+          - k3s
+          - rke2
+    runs-on: garm-ubuntu-2004-smaller
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: ${{ matrix.k8s }}
+      USING_NFD: "false"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy ${{ matrix.k8s }}
+        run:  bash tests/functional/kata-deploy/gha-run.sh deploy-k8s
+
+      - name: Install `bats`
+        run: bash tests/functional/kata-deploy/gha-run.sh install-bats
+
+      - name: Run tests
+        run: bash tests/functional/kata-deploy/gha-run.sh run-tests

.github/workflows/run-kata-monitor-tests.yaml

@@ -0,0 +1,59 @@
+name: CI | Run kata-monitor tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-monitor:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu
+        container_engine:
+          - crio
+          - containerd
+        include:
+          - container_engine: containerd
+            containerd_version: lts
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINER_ENGINE: ${{ matrix.container_engine }}
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/functional/kata-monitor/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/functional/kata-monitor/gha-run.sh install-kata kata-artifacts
+
+      - name: Run kata-monitor tests
+        run: bash tests/functional/kata-monitor/gha-run.sh run

.github/workflows/run-metrics.yaml

@@ -0,0 +1,94 @@
+name: CI | Run test metrics
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  setup-kata:
+    name: Kata Setup
+    runs-on: metrics
+    env:
+      GOPATH: ${{ github.workspace }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/metrics/gha-run.sh install-kata kata-artifacts
+
+  run-metrics:
+    needs: setup-kata
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # the all the tests are not flaky, otherwise we'll fail
+      # all the tests due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        vmm: ['clh', 'qemu']
+      max-parallel: 1
+    runs-on: metrics
+    env:
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - name: enabling the hypervisor
+        run: bash tests/metrics/gha-run.sh enabling-hypervisor
+
+      - name: run launch times test
+        run: bash tests/metrics/gha-run.sh run-test-launchtimes
+
+      - name: run memory foot print test
+        run:  bash tests/metrics/gha-run.sh run-test-memory-usage
+
+      - name: run memory usage inside container test
+        run:  bash tests/metrics/gha-run.sh run-test-memory-usage-inside-container
+
+      - name: run blogbench test
+        run:  bash tests/metrics/gha-run.sh run-test-blogbench
+
+      - name: run tensorflow test
+        run:  bash tests/metrics/gha-run.sh run-test-tensorflow
+
+      - name: run fio test
+        run:  bash tests/metrics/gha-run.sh run-test-fio
+
+      - name: run iperf test
+        run:  bash tests/metrics/gha-run.sh run-test-iperf
+
+      - name: run latency test
+        run:  bash tests/metrics/gha-run.sh run-test-latency
+
+      - name: make metrics tarball ${{ matrix.vmm }}
+        run: bash tests/metrics/gha-run.sh make-tarball-results
+
+      - name: archive metrics results ${{ matrix.vmm }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: metrics-artifacts-${{ matrix.vmm }}
+          path: results-${{ matrix.vmm }}.tar.gz
+          retention-days: 1
+          if-no-files-found: error

.github/workflows/run-nerdctl-tests-on-garm.yaml

@@ -0,0 +1,57 @@
+name: CI | Run nerdctl integration tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-nerdctl-tests:
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # all the tests are not flaky, otherwise we'll fail them
+      # all due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        vmm:
+          - clh
+          - dragonball
+          - qemu
+    runs-on: garm-ubuntu-2304-smaller
+    env:
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/nerdctl/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/nerdctl/gha-run.sh install-kata kata-artifacts
+
+      - name: Run nerdctl smoke test
+        timeout-minutes: 5
+        run: bash tests/integration/nerdctl/gha-run.sh run

.github/workflows/run-runk-tests.yaml

@@ -0,0 +1,46 @@
+name: CI | Run runk tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-runk:
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINERD_VERSION: lts
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/runk/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/runk/gha-run.sh install-kata kata-artifacts
+
+      - name: Run tracing tests
+        run: bash tests/integration/runk/gha-run.sh run

.github/workflows/snap-release.yaml

@@ -1,39 +0,0 @@
-name: Release Kata 2.x in snapcraft store
-on:
-  push:
-    tags:
-      - '2.*'
-jobs:
-  release-snap:
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Check out Git repository
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Install Snapcraft
-        uses: samuelmeuli/action-snapcraft@v1
-        with:
-          snapcraft_token: ${{ secrets.snapcraft_token }}
-
-      - name: Build snap
-        run: |
-          sudo apt-get install -y git git-extras
-          kata_url="https://github.com/kata-containers/kata-containers"
-          latest_version=$(git ls-remote --tags ${kata_url}  | egrep -o "refs.*" | egrep -v "\-alpha|\-rc|{}" | egrep -o "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+" | sort -V -r | head -1)
-          current_version="$(echo ${GITHUB_REF} | cut -d/ -f3)"
-          # Check semantic versioning format (x.y.z) and if the current tag is the latest tag
-          if echo "${current_version}" | grep -q "^[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+$" && echo -e "$latest_version\n$current_version" | sort -C -V; then
-            # Current version is the latest version, build it
-            snapcraft -d snap --destructive-mode
-          fi
-
-      - name: Upload snap
-        run: |
-          snap_version="$(echo ${GITHUB_REF} | cut -d/ -f3)"
-          snap_file="kata-containers_${snap_version}_amd64.snap"
-          # Upload the snap if it exists
-          if [ -f ${snap_file} ]; then
-            snapcraft upload --release=stable ${snap_file}
-          fi

.github/workflows/snap.yaml

@@ -1,27 +0,0 @@
-name: snap CI
-on:
-  pull_request:
-    types:
-      - opened
-      - synchronize
-      - reopened
-      - edited
-
-jobs:
-  test:
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Check out
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Install Snapcraft
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: samuelmeuli/action-snapcraft@v1
-
-      - name: Build snap
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          snapcraft -d snap --destructive-mode

.github/workflows/static-checks.yaml

@@ -6,91 +6,189 @@ on:
       - reopened
       - synchronize
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 name: Static checks
 jobs:
-  test:
-    strategy:
-      matrix:
-        go-version: [1.16.x, 1.17.x]
-        os: [ubuntu-20.04]
-    runs-on: ${{ matrix.os }}
-    env:
-      TRAVIS: "true"
-      TRAVIS_BRANCH: ${{ github.base_ref }}
-      TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }}
-      TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }}
-      RUST_BACKTRACE: "1"
-      target_branch: ${{ github.base_ref }}
+  check-kernel-config-version:
+    runs-on: ubuntu-latest
     steps:
-    - name: Install Go
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/setup-go@v2
-      with:
-        go-version: ${{ matrix.go-version }}
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Setup GOPATH
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}"
-        echo "TRAVIS: ${TRAVIS}"
-    - name: Set env
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
-    - name: Checkout code
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-        path: ./src/github.com/${{ github.repository }}
-    - name: Setup travis references
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}"
-        target_branch=${TRAVIS_BRANCH}
-    - name: Setup
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Installing rust
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh
-        PATH=$PATH:"$HOME/.cargo/bin"
-        rustup target add x86_64-unknown-linux-musl
-        rustup component add rustfmt clippy
-    - name: Setup seccomp
-      run: |
-        libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
-        gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
-        echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
-        echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
-        echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
-    # Check whether the vendored code is up-to-date & working as the first thing
-    - name: Check vendored code
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make vendor
-    - name: Static Checks
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make static-checks
-    - name: Run Compiler Checks
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make check
-    - name: Run Unit Tests
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make test
-    - name: Run Unit Tests As Root User
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && sudo -E PATH="$PATH" make test
+      - name: Checkout the code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Ensure the kernel config version has been updated
+        run: |
+          kernel_dir="tools/packaging/kernel/"
+          kernel_version_file="${kernel_dir}kata_config_version"
+          modified_files=$(git diff --name-only origin/$GITHUB_BASE_REF..HEAD)
+          if git diff --name-only origin/$GITHUB_BASE_REF..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then
+            echo "Kernel directory has changed, checking if $kernel_version_file has been updated"
+            if echo "$modified_files" | grep -v "README.md" | grep "${kernel_dir}" >>"/dev/null"; then
+              echo "$modified_files" | grep "$kernel_version_file" >>/dev/null || ( echo "Please bump version in $kernel_version_file" && exit 1)
+            else
+              echo "Readme file changed, no need for kernel config version update."
+            fi
+            echo "Check passed"
+          fi
+
+  build-checks:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - agent
+          - dragonball
+          - runtime
+          - runtime-rs
+          - agent-ctl
+          - kata-ctl
+          - log-parser-rs
+          - runk
+          - trace-forwarder
+        command:
+          - "make vendor"
+          - "make check"
+          - "make test"
+          - "sudo -E PATH=\"$PATH\" make test"
+        include:
+          - component: agent
+            component-path: src/agent
+          - component: dragonball
+            component-path: src/dragonball
+          - component: runtime
+            component-path: src/runtime
+          - component: runtime-rs
+            component-path: src/runtime-rs
+          - component: agent-ctl
+            component-path: src/tools/agent-ctl
+          - component: kata-ctl
+            component-path: src/tools/kata-ctl
+          - component: log-parser-rs
+            component-path: src/tools/log-parser-rs
+          - component: runk
+            component-path: src/tools/runk
+          - component: trace-forwarder
+            component-path: src/tools/trace-forwarder
+          - install-libseccomp: no
+          - component: agent
+            install-libseccomp: yes
+          - component: runk
+            install-libseccomp: yes
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install yq
+        run: |
+          ./ci/install_yq.sh
+        env:
+          INSTALL_IN_GOPATH: false
+      - name: Install golang
+        if: ${{ matrix.component == 'runtime' }}
+        run: |
+          ./tests/install_go.sh -f -p
+          echo "/usr/local/go/bin" >> $GITHUB_PATH
+      - name: Install rust
+        if: ${{ matrix.component != 'runtime' }}
+        run: |
+          ./tests/install_rust.sh
+          echo "${HOME}/.cargo/bin" >> $GITHUB_PATH
+      - name: Install musl-tools
+        if: ${{ matrix.component != 'runtime' }}
+        run: sudo apt-get -y install musl-tools
+      - name: Install libseccomp
+        if: ${{ matrix.command != 'make vendor'  &&  matrix.command != 'make check' &&  matrix.install-libseccomp == 'yes' }}
+        run: |
+          libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
+          gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
+          ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
+          echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
+          echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
+          echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
+      - name: Setup XDG_RUNTIME_DIR for the `runtime` tests
+        if: ${{ matrix.command != 'make vendor' && matrix.command != 'make check' && matrix.component == 'runtime' }}
+        run: |
+          XDG_RUNTIME_DIR=$(mktemp -d /tmp/kata-tests-$USER.XXX | tee >(xargs chmod 0700))
+          echo "XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR}" >> $GITHUB_ENV
+      - name: Running `${{ matrix.command }}` for ${{ matrix.component }}
+        run: |
+          cd ${{ matrix.component-path }}
+          ${{ matrix.command }}
+        env:
+          RUST_BACKTRACE: "1"
+
+  build-checks-depending-on-kvm:
+    runs-on: garm-ubuntu-2004-smaller
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - runtime-rs
+        include:
+          - component: runtime-rs
+            command: "sudo -E env PATH=$PATH LIBC=gnu SUPPORT_VIRTUALIZATION=true make test"
+          - component: runtime-rs
+            component-path: src/dragonball
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install system deps
+        run: |
+          sudo apt-get install -y build-essential musl-tools
+      - name: Install yq
+        run: |
+          sudo -E ./ci/install_yq.sh
+        env:
+          INSTALL_IN_GOPATH: false
+      - name: Install rust
+        run: |
+          export PATH="$PATH:/usr/local/bin"
+          ./tests/install_rust.sh
+      - name: Running `${{ matrix.command }}` for ${{ matrix.component }}
+        run: |
+          export PATH="$PATH:${HOME}/.cargo/bin"
+          cd ${{ matrix.component-path }}
+          ${{ matrix.command }}
+        env:
+          RUST_BACKTRACE: "1"
+
+  static-checks:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        cmd:
+          - "make static-checks"
+    env:
+      GOPATH: ${{ github.workspace }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          path: ./src/github.com/${{ github.repository }}
+      - name: Install yq
+        run: |
+          cd ${GOPATH}/src/github.com/${{ github.repository }}
+          ./ci/install_yq.sh
+        env:
+          INSTALL_IN_GOPATH: false
+      - name: Install golang
+        run: |
+          cd ${GOPATH}/src/github.com/${{ github.repository }}
+          ./tests/install_go.sh -f -p
+          echo "/usr/local/go/bin" >> $GITHUB_PATH
+      - name: Install system dependencies
+        run: |
+          sudo apt-get -y install moreutils hunspell pandoc
+      - name: Run check
+        run: |
+          export PATH=${PATH}:${GOPATH}/bin
+          cd ${GOPATH}/src/github.com/${{ github.repository }} && ${{ matrix.cmd }}

.gitignore

@@ -4,9 +4,14 @@
 **/*.rej
 **/target
 **/.vscode
+**/.idea
+**/.fleet
+**/*.swp
+**/*.swo
 pkg/logging/Cargo.lock
 src/agent/src/version.rs
 src/agent/kata-agent.service
 src/agent/protocols/src/*.rs
 !src/agent/protocols/src/lib.rs
-
+build
+src/tools/log-parser/kata-log-parser

Makefile

@@ -6,24 +6,30 @@
 # List of available components
 COMPONENTS =
 
+COMPONENTS += libs
 COMPONENTS += agent
+COMPONENTS += dragonball
 COMPONENTS += runtime
+COMPONENTS += runtime-rs
 
 # List of available tools
 TOOLS =
 
 TOOLS += agent-ctl
+TOOLS += kata-ctl
+TOOLS += log-parser
+TOOLS += log-parser-rs
+TOOLS += runk
 TOOLS += trace-forwarder
 
-STANDARD_TARGETS = build check clean install test vendor
+STANDARD_TARGETS = build check clean install static-checks-build test vendor
+
+# Variables for the build-and-publish-kata-debug target
+KATA_DEBUG_REGISTRY ?= ""
+KATA_DEBUG_TAG ?= ""
 
 default: all
 
-all: logging-crate-tests build
-
-logging-crate-tests:
-	make -C src/libs/logging
-
 include utils.mk
 include ./tools/packaging/kata-deploy/local-build/Makefile
 
@@ -36,13 +42,19 @@ generate-protocols:
 	make -C src/agent generate-protocols
 
 # Some static checks rely on generated source files of components.
-static-checks: build
+static-checks: static-checks-build
 	bash ci/static-checks.sh
 
+docs-url-alive-check:
+	bash ci/docs-url-alive-check.sh
+
+build-and-publish-kata-debug:
+	bash tools/packaging/kata-debug/kata-debug-build-and-upload-payload.sh ${KATA_DEBUG_REGISTRY} ${KATA_DEBUG_TAG} 
+
 .PHONY: \
 	all \
-	binary-tarball \
+	kata-tarball \
+	install-tarball \
 	default \
-	install-binary-tarball \
-	logging-crate-tests \
-	static-checks
+	static-checks \
+	docs-url-alive-check

README.md

@@ -1,4 +1,6 @@
-<img src="https://www.openstack.org/assets/kata/kata-vertical-on-white.png" width="150">
+<img src="https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/kata/SVG/kata-1.svg" width="900">
+
+[![CI | Publish Kata Containers payload](https://github.com/kata-containers/kata-containers/actions/workflows/payload-after-push.yaml/badge.svg)](https://github.com/kata-containers/kata-containers/actions/workflows/payload-after-push.yaml) [![Kata Containers Nightly CI](https://github.com/kata-containers/kata-containers/actions/workflows/ci-nightly.yaml/badge.svg)](https://github.com/kata-containers/kata-containers/actions/workflows/ci-nightly.yaml)
 
 # Kata Containers
 
@@ -71,6 +73,7 @@ See the [official documentation](docs) including:
 - [Developer guide](docs/Developer-Guide.md)
 - [Design documents](docs/design)
   - [Architecture overview](docs/design/architecture)
+  - [Architecture 3.0 overview](docs/design/architecture_3.0/)
 
 ## Configuration
 
@@ -116,7 +119,9 @@ The table below lists the core parts of the project:
 | Component | Type | Description |
 |-|-|-|
 | [runtime](src/runtime) | core | Main component run by a container manager and providing a containerd shimv2 runtime implementation. |
+| [runtime-rs](src/runtime-rs) | core | The Rust version runtime. |
 | [agent](src/agent) | core | Management process running inside the virtual machine / POD that sets up the container environment. |
+| [`dragonball`](src/dragonball) | core | An optional built-in VMM brings out-of-the-box Kata Containers experience with optimizations on container workloads |
 | [documentation](docs) | documentation | Documentation common to all components (such as design and install documentation). |
 | [tests](https://github.com/kata-containers/tests) | tests | Excludes unit tests which live with the main code. |
 
@@ -129,8 +134,12 @@ The table below lists the remaining parts of the project:
 | [packaging](tools/packaging) | infrastructure | Scripts and metadata for producing packaged binaries<br/>(components, hypervisors, kernel and rootfs). |
 | [kernel](https://www.kernel.org) | kernel | Linux kernel used by the hypervisor to boot the guest image. Patches are stored [here](tools/packaging/kernel). |
 | [osbuilder](tools/osbuilder) | infrastructure | Tool to create "mini O/S" rootfs and initrd images and kernel for the hypervisor. |
+| [kata-debug](tools/packaging/kata-debug/README.md) | infrastructure | Utility tool to gather Kata Containers debug information from Kubernetes clusters. |
 | [`agent-ctl`](src/tools/agent-ctl) | utility | Tool that provides low-level access for testing the agent. |
+| [`kata-ctl`](src/tools/kata-ctl) | utility | Tool that provides advanced commands and debug facilities. |
+| [`log-parser-rs`](src/tools/log-parser-rs) | utility | Tool that aid in analyzing logs from the kata runtime. |
 | [`trace-forwarder`](src/tools/trace-forwarder) | utility | Agent tracing helper. |
+| [`runk`](src/tools/runk) | utility | Standard OCI container runtime based on the agent. |
 | [`ci`](https://github.com/kata-containers/ci) | CI | Continuous Integration configuration files and scripts. |
 | [`katacontainers.io`](https://github.com/kata-containers/www.katacontainers.io) | Source for the [`katacontainers.io`](https://www.katacontainers.io) site. |
 
@@ -138,8 +147,10 @@ The table below lists the remaining parts of the project:
 
 Kata Containers is now
 [available natively for most distributions](docs/install/README.md#packaged-installation-methods).
-However, packaging scripts and metadata are still used to generate snap and GitHub releases. See
-the [components](#components) section for further details.
+
+## Metrics tests
+
+See the [metrics documentation](tests/metrics/README.md).
 
 ## Glossary of Terms
 

VERSION

@@ -1 +1 @@
-2.4.0-rc0
+3.2.0

ci/darwin-test.sh

@@ -11,10 +11,10 @@ runtimedir=$cidir/../src/runtime
 
 build_working_packages() {
 	# working packages:
-	device_api=$runtimedir/virtcontainers/device/api
-	device_config=$runtimedir/virtcontainers/device/config
-	device_drivers=$runtimedir/virtcontainers/device/drivers
-	device_manager=$runtimedir/virtcontainers/device/manager
+	device_api=$runtimedir/pkg/device/api
+	device_config=$runtimedir/pkg/device/config
+	device_drivers=$runtimedir/pkg/device/drivers
+	device_manager=$runtimedir/pkg/device/manager
 	rc_pkg_dir=$runtimedir/pkg/resourcecontrol/
 	utils_pkg_dir=$runtimedir/virtcontainers/utils
 

ci/docs-url-alive-check.sh

@@ -1,6 +1,6 @@
-#!/usr/bin/env bash
+#!/bin/bash
 #
-# Copyright (c) 2020 Intel Corporation
+# Copyright (c) 2021 Easystack Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 
@@ -9,4 +9,4 @@ set -e
 cidir=$(dirname "$0")
 source "${cidir}/lib.sh"
 
-run_go_test
+run_docs_url_alive_check

ci/install_libseccomp.sh

@@ -7,41 +7,41 @@
 
 set -o errexit
 
-cidir=$(dirname "$0")
-source "${cidir}/lib.sh"
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+script_name="$(basename "${BASH_SOURCE[0]}")"
 
-clone_tests_repo
-
-source "${tests_repo_dir}/.ci/lib.sh"
+source "${script_dir}/../tests/common.bash"
 
 # The following variables if set on the environment will change the behavior
 # of gperf and libseccomp configure scripts, that may lead this script to
 # fail. So let's ensure they are unset here.
 unset PREFIX DESTDIR
 
-arch=$(uname -m)
+arch=${ARCH:-$(uname -m)}
 workdir="$(mktemp -d --tmpdir build-libseccomp.XXXXX)"
 
 # Variables for libseccomp
-# Currently, specify the libseccomp version directly without using `versions.yaml`
-# because the current Snap workflow is incomplete.
-# After solving the issue, replace this code by using the `versions.yaml`.
-# libseccomp_version=$(get_version "externals.libseccomp.version")
-# libseccomp_url=$(get_version "externals.libseccomp.url")
-libseccomp_version="2.5.1"
-libseccomp_url="https://github.com/seccomp/libseccomp"
+libseccomp_version="${LIBSECCOMP_VERSION:-""}"
+if [ -z "${libseccomp_version}" ]; then
+    libseccomp_version=$(get_from_kata_deps "externals.libseccomp.version")
+fi
+libseccomp_url="${LIBSECCOMP_URL:-""}"
+if [ -z "${libseccomp_url}" ]; then
+    libseccomp_url=$(get_from_kata_deps "externals.libseccomp.url")
+fi
 libseccomp_tarball="libseccomp-${libseccomp_version}.tar.gz"
 libseccomp_tarball_url="${libseccomp_url}/releases/download/v${libseccomp_version}/${libseccomp_tarball}"
 cflags="-O2"
 
 # Variables for gperf
-# Currently, specify the gperf version directly without using `versions.yaml`
-# because the current Snap workflow is incomplete.
-# After solving the issue, replace this code by using the `versions.yaml`.
-# gperf_version=$(get_version "externals.gperf.version")
-# gperf_url=$(get_version "externals.gperf.url")
-gperf_version="3.1"
-gperf_url="https://ftp.gnu.org/gnu/gperf"
+gperf_version="${GPERF_VERSION:-""}"
+if [ -z "${gperf_version}" ]; then
+    gperf_version=$(get_from_kata_deps "externals.gperf.version")
+fi
+gperf_url="${GPERF_URL:-""}"
+if [ -z "${gperf_url}" ]; then
+    gperf_url=$(get_from_kata_deps "externals.gperf.url")
+fi
 gperf_tarball="gperf-${gperf_version}.tar.gz"
 gperf_tarball_url="${gperf_url}/${gperf_tarball}"
 
@@ -70,7 +70,8 @@ build_and_install_gperf() {
     curl -sLO "${gperf_tarball_url}"
     tar -xf "${gperf_tarball}"
     pushd "gperf-${gperf_version}"
-    ./configure --prefix="${gperf_install_dir}"
+    # Unset $CC for configure, we will always use native for gperf
+    CC= ./configure --prefix="${gperf_install_dir}"
     make
     make install
     export PATH=$PATH:"${gperf_install_dir}"/bin
@@ -84,7 +85,8 @@ build_and_install_libseccomp() {
     curl -sLO "${libseccomp_tarball_url}"
     tar -xf "${libseccomp_tarball}"
     pushd "libseccomp-${libseccomp_version}"
-    ./configure --prefix="${libseccomp_install_dir}" CFLAGS="${cflags}" --enable-static
+    [ "${arch}" == $(uname -m) ] && cc_name="" || cc_name="${arch}-linux-gnu-gcc"
+    CC=${cc_name} ./configure --prefix="${libseccomp_install_dir}" CFLAGS="${cflags}" --enable-static --host="${arch}"
     make
     make install
     popd

ci/install_musl.sh

@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-# Copyright (c) 2020 Ant Group
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-set -e
-
-install_aarch64_musl() {
-	local arch=$(uname -m)
-	if [ "${arch}" == "aarch64" ]; then
-		local musl_tar="${arch}-linux-musl-native.tgz"
-		local musl_dir="${arch}-linux-musl-native"
-		pushd /tmp
-		if curl -sLO --fail https://musl.cc/${musl_tar}; then
-			tar -zxf ${musl_tar}
-			mkdir -p /usr/local/musl/
-			cp -r ${musl_dir}/* /usr/local/musl/
-		fi
-		popd
-	fi
-}
-
-install_aarch64_musl

ci/install_yq.sh

@@ -43,6 +43,16 @@ function install_yq() {
 	"aarch64")
 		goarch=arm64
 		;;
+	"arm64")
+		# If we're on an apple silicon machine, just assign amd64. 
+		# The version of yq we use doesn't have a darwin arm build, 
+		# but Rosetta can come to the rescue here.
+		if [ $goos == "Darwin" ]; then 
+			goarch=amd64
+		else 
+			goarch=arm64
+		fi
+		;;
 	"ppc64le")
 		goarch=ppc64le
 		;;
@@ -64,7 +74,7 @@ function install_yq() {
 	fi
 
 	## NOTE: ${var,,} => gives lowercase value of var
-	local yq_url="https://${yq_pkg}/releases/download/${yq_version}/yq_${goos,,}_${goarch}"
+	local yq_url="https://${yq_pkg}/releases/download/${yq_version}/yq_${goos}_${goarch}"
 	curl -o "${yq_path}" -LSsf "${yq_url}"
 	[ $? -ne 0 ] && die "Download ${yq_url} failed"
 	chmod +x "${yq_path}"

ci/lib.sh

@@ -18,6 +18,13 @@ clone_tests_repo()
 {
 	if [ -d "$tests_repo_dir" ]; then
 		[ -n "${CI:-}" ] && return
+		# git config --global --add safe.directory will always append
+		# the target to .gitconfig without checking the existence of
+		# the target, so it's better to check it before adding the target repo.
+		local sd="$(git config --global --get safe.directory ${tests_repo_dir} || true)"
+		if [ -z "${sd}" ]; then
+			git config --global --add safe.directory ${tests_repo_dir}
+		fi
 		pushd "${tests_repo_dir}"
 		git checkout "${branch}"
 		git pull
@@ -39,8 +46,21 @@ run_static_checks()
 	bash "$tests_repo_dir/.ci/static-checks.sh" "$@"
 }
 
-run_go_test()
+run_docs_url_alive_check()
 {
 	clone_tests_repo
-	bash "$tests_repo_dir/.ci/go-test.sh"
+	# Make sure we have the targeting branch
+	git remote set-branches --add origin "${branch}"
+	git fetch -a
+	bash "$tests_repo_dir/.ci/static-checks.sh" --docs --all "github.com/kata-containers/kata-containers"
+}
+
+run_get_pr_changed_file_details()
+{
+	clone_tests_repo
+	# Make sure we have the targeting branch
+	git remote set-branches --add origin "${branch}"
+	git fetch -a
+	source "$tests_repo_dir/.ci/lib.sh"
+	get_pr_changed_file_details
 }

deny.toml

@@ -0,0 +1,33 @@
+targets = [
+    { triple = "x86_64-apple-darwin" },
+    { triple = "x86_64-unknown-linux-gnu" },
+    { triple = "x86_64-unknown-linux-musl" },
+]
+
+[advisories]
+vulnerability = "deny"
+unsound = "deny"
+unmaintained = "deny"
+ignore = ["RUSTSEC-2020-0071"]
+
+[bans]
+multiple-versions = "allow"
+deny = [
+    { name = "cmake" },
+    { name = "openssl-sys" },
+]
+
+[licenses]
+unlicensed = "deny"
+allow-osi-fsf-free = "neither"
+copyleft = "allow"
+# We want really high confidence when inferring licenses from text
+confidence-threshold = 0.93
+allow = ["0BSD", "Apache-2.0", "BSD-2-Clause", "BSD-3-Clause", "CC0-1.0", "ISC", "MIT", "MPL-2.0"]
+private = { ignore = true}
+
+exceptions = []
+
+[sources]
+unknown-registry = "allow"
+unknown-git = "allow"

docs/Developer-Guide.md

@@ -2,6 +2,8 @@
 
 This document is written **specifically for developers**: it is not intended for end users.
 
+If you want to contribute changes that you have made, please read the [community guidelines](https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md) for information about our processes.
+
 # Assumptions
 
 - You are working on a non-critical test or development system.
@@ -33,51 +35,41 @@ You need to install the following to build Kata Containers components:
 - `make`.
 - `gcc` (required for building the shim and runtime).
 
-# Build and install the Kata Containers runtime
+# Build and install Kata Containers
+## Build and install the Kata Containers runtime
 
-```
-$ go get -d -u github.com/kata-containers/kata-containers
-$ cd $GOPATH/src/github.com/kata-containers/kata-containers/src/runtime
-$ make && sudo -E PATH=$PATH make install
+```bash
+$ git clone https://github.com/kata-containers/kata-containers.git
+$ pushd kata-containers/src/runtime
+$ make && sudo -E "PATH=$PATH" make install
+$ sudo mkdir -p /etc/kata-containers/
+$ sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers
+$ popd
 ```
 
 The build will create the following:
 
 - runtime binary: `/usr/local/bin/kata-runtime` and `/usr/local/bin/containerd-shim-kata-v2`
-- configuration file: `/usr/share/defaults/kata-containers/configuration.toml`
-
-# Check hardware requirements
-
-You can check if your system is capable of creating a Kata Container by running the following:
-
-```
-$ sudo kata-runtime check
-```
-
-If your system is *not* able to run Kata Containers, the previous command will error out and explain why.
+- configuration file: `/usr/share/defaults/kata-containers/configuration.toml` and `/etc/kata-containers/configuration.toml`
 
 ## Configure to use initrd or rootfs image
 
 Kata containers can run with either an initrd image or a rootfs image.
 
-If you want to test with `initrd`, make sure you have `initrd = /usr/share/kata-containers/kata-containers-initrd.img`
-in your configuration file, commenting out the `image` line:
+If you want to test with `initrd`, make sure you have uncommented `initrd = /usr/share/kata-containers/kata-containers-initrd.img`
+in your configuration file, commenting out the `image` line in
+`/etc/kata-containers/configuration.toml`. For example:
 
-`/usr/share/defaults/kata-containers/configuration.toml` and comment out the `image` line with the following. For example:
-
-```
-$ sudo mkdir -p /etc/kata-containers/
-$ sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers
+```bash
 $ sudo sed -i 's/^\(image =.*\)/# \1/g' /etc/kata-containers/configuration.toml
+$ sudo sed -i 's/^# \(initrd =.*\)/\1/g' /etc/kata-containers/configuration.toml
 ```
 You can create the initrd image as shown in the [create an initrd image](#create-an-initrd-image---optional) section.
 
-If you want to test with a rootfs `image`, make sure you have `image = /usr/share/kata-containers/kata-containers.img`
+If you want to test with a rootfs `image`, make sure you have uncommented `image = /usr/share/kata-containers/kata-containers.img`
 in your configuration file, commenting out the `initrd` line. For example:
 
-```
-$ sudo mkdir -p /etc/kata-containers/
-$ sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers
+```bash
 $ sudo sed -i 's/^\(initrd =.*\)/# \1/g' /etc/kata-containers/configuration.toml
 ```
 The rootfs image is created as shown in the [create a rootfs image](#create-a-rootfs-image) section.
@@ -90,19 +82,38 @@ rootfs `image`(100MB+).
 
 Enable seccomp as follows:
 
-```
+```bash
 $ sudo sed -i '/^disable_guest_seccomp/ s/true/false/' /etc/kata-containers/configuration.toml
 ```
 
 This will pass container seccomp profiles to the kata agent.
 
+## Enable SELinux on the guest
+
+> **Note:**
+>
+> - To enable SELinux on the guest, SELinux MUST be also enabled on the host.
+> - You MUST create and build a rootfs image for SELinux in advance.
+>   See [Create a rootfs image](#create-a-rootfs-image) and [Build a rootfs image](#build-a-rootfs-image).
+> - SELinux on the guest is supported in only a rootfs image currently, so
+>   you cannot enable SELinux with the agent init (`AGENT_INIT=yes`) yet.
+
+Enable guest SELinux in Enforcing mode as follows:
+
+```
+$ sudo sed -i '/^disable_guest_selinux/ s/true/false/g' /etc/kata-containers/configuration.toml
+```
+
+The runtime automatically will set `selinux=1` to the kernel parameters and `xattr` option to
+`virtiofsd` when `disable_guest_selinux` is set to `false`.
+
+If you want to enable SELinux in Permissive mode, add `enforcing=0` to the kernel parameters.
+
 ## Enable full debug
 
 Enable full debug as follows:
 
-```
-$ sudo mkdir -p /etc/kata-containers/
-$ sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers
+```bash
 $ sudo sed -i -e 's/^# *\(enable_debug\).*=.*$/\1 = true/g' /etc/kata-containers/configuration.toml
 $ sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.log=debug initcall_debug"/g' /etc/kata-containers/configuration.toml
 ```
@@ -116,7 +127,7 @@ detailed below.
 The Kata logs appear in the `containerd` log files, along with logs from `containerd` itself.
 
 For more information about `containerd` debug, please see the
-[`containerd` documentation](https://github.com/containerd/containerd/blob/master/docs/getting-started.md).
+[`containerd` documentation](https://github.com/containerd/containerd/blob/main/docs/getting-started.md).
 
 #### Enabling full `containerd` debug
 
@@ -175,7 +186,7 @@ and offers possible workarounds and fixes.
 it stores. When messages are suppressed, it is noted in the logs. This can be checked
 for by looking for those notifications, such as:
 
-```sh
+```bash
 $ sudo journalctl --since today | fgrep Suppressed
 Jun 29 14:51:17 mymachine systemd-journald[346]: Suppressed 4150 messages from /system.slice/docker.service
 ```
@@ -200,7 +211,7 @@ RateLimitBurst=0
 
 Restart `systemd-journald` for the changes to take effect:
 
-```sh
+```bash
 $ sudo systemctl restart systemd-journald
 ```
 
@@ -214,39 +225,52 @@ $ sudo systemctl restart systemd-journald
 
 The agent is built with a statically linked `musl.` The default `libc` used is `musl`, but on `ppc64le` and `s390x`, `gnu` should be used. To configure this:
 
-```
-$ export ARCH=$(uname -m)
+```bash
+$ export ARCH="$(uname -m)"
 $ if [ "$ARCH" = "ppc64le" -o "$ARCH" = "s390x" ]; then export LIBC=gnu; else export LIBC=musl; fi
-$ [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
-$ rustup target add ${ARCH}-unknown-linux-${LIBC}
+$ [ "${ARCH}" == "ppc64le" ] && export ARCH=powerpc64le
+$ rustup target add "${ARCH}-unknown-linux-${LIBC}"
 ```
 
 To build the agent:
 
-```
-$ go get -d -u github.com/kata-containers/kata-containers
-$ cd $GOPATH/src/github.com/kata-containers/kata-containers/src/agent && make
-```
-
 The agent is built with seccomp capability by default.
 If you want to build the agent without the seccomp capability, you need to run `make` with `SECCOMP=no` as follows.
 
+```bash
+$ make -C kata-containers/src/agent SECCOMP=no
 ```
-$ make -C $GOPATH/src/github.com/kata-containers/kata-containers/src/agent SECCOMP=no
+
+For building the agent with seccomp support using `musl`, set the environment
+variables for the [`libseccomp` crate](https://github.com/libseccomp-rs/libseccomp-rs).
+
+```bash
+$ export LIBSECCOMP_LINK_TYPE=static
+$ export LIBSECCOMP_LIB_PATH="the path of the directory containing libseccomp.a"
+$ make -C kata-containers/src/agent
 ```
 
+If the compilation fails when the agent tries to link the `libseccomp` library statically
+against `musl`, you will need to build `libseccomp` manually with `-U_FORTIFY_SOURCE`.
+You can use [our script](https://github.com/kata-containers/kata-containers/blob/main/ci/install_libseccomp.sh)
+to install `libseccomp` for the agent.
+
+```bash
+$ mkdir -p ${seccomp_install_path} ${gperf_install_path}
+$ kata-containers/ci/install_libseccomp.sh ${seccomp_install_path} ${gperf_install_path}
+$ export LIBSECCOMP_LIB_PATH="${seccomp_install_path}/lib"
+```
+
+On `ppc64le` and `s390x`, `glibc` is used. You will need to install the `libseccomp` library
+provided by your distribution.
+
+> e.g. `libseccomp-dev` for Ubuntu, or `libseccomp-devel` for CentOS
+
 > **Note:**
 >
 > - If you enable seccomp in the main configuration file but build the agent without seccomp capability,
 >   the runtime exits conservatively with an error message.
 
-## Get the osbuilder
-
-```
-$ go get -d -u github.com/kata-containers/kata-containers
-$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder
-```
-
 ## Create a rootfs image
 ### Create a local rootfs
 
@@ -254,24 +278,32 @@ As a prerequisite, you need to install Docker. Otherwise, you will not be
 able to run the `rootfs.sh` script with `USE_DOCKER=true` as expected in
 the following example.
 
-```
-$ export ROOTFS_DIR=${GOPATH}/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder/rootfs
-$ sudo rm -rf ${ROOTFS_DIR}
-$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder
-$ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true ./rootfs.sh ${distro}'
+```bash
+$ export distro="ubuntu" # example
+$ export ROOTFS_DIR="$(realpath kata-containers/tools/osbuilder/rootfs-builder/rootfs)"
+$ sudo rm -rf "${ROOTFS_DIR}"
+$ pushd kata-containers/tools/osbuilder/rootfs-builder
+$ script -fec 'sudo -E USE_DOCKER=true ./rootfs.sh "${distro}"'
+$ popd
 ```
 
 You MUST choose a distribution (e.g., `ubuntu`) for `${distro}`.
 You can get a supported distributions list in the Kata Containers by running the following.
 
-```
-$ ./rootfs.sh -l
+```bash
+$ ./kata-containers/tools/osbuilder/rootfs-builder/rootfs.sh -l
 ```
 
 If you want to build the agent without seccomp capability, you need to run the `rootfs.sh` script with `SECCOMP=no` as follows.
 
+```bash
+$ script -fec 'sudo -E AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh "${distro}"'
 ```
-$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
+
+If you want to enable SELinux on the guest, you MUST choose `centos` and run the `rootfs.sh` script with `SELINUX=yes` as follows.
+
+```
+$ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true SELINUX=yes ./rootfs.sh centos'
 ```
 
 > **Note:**
@@ -287,18 +319,32 @@ $ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no
 >
 > - You should only do this step if you are testing with the latest version of the agent.
 
-```
-$ sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/usr/bin ../../../src/agent/target/x86_64-unknown-linux-musl/release/kata-agent
-$ sudo install -o root -g root -m 0440 ../../../src/agent/kata-agent.service ${ROOTFS_DIR}/usr/lib/systemd/system/
-$ sudo install -o root -g root -m 0440 ../../../src/agent/kata-containers.target ${ROOTFS_DIR}/usr/lib/systemd/system/
+```bash
+$ sudo install -o root -g root -m 0550 -t "${ROOTFS_DIR}/usr/bin" "${ROOTFS_DIR}/../../../../src/agent/target/x86_64-unknown-linux-musl/release/kata-agent"
+$ sudo install -o root -g root -m 0440 "${ROOTFS_DIR}/../../../../src/agent/kata-agent.service" "${ROOTFS_DIR}/usr/lib/systemd/system/"
+$ sudo install -o root -g root -m 0440 "${ROOTFS_DIR}/../../../../src/agent/kata-containers.target" "${ROOTFS_DIR}/usr/lib/systemd/system/"
 ```
 
 ### Build a rootfs image
 
+```bash
+$ pushd  kata-containers/tools/osbuilder/image-builder
+$ script -fec 'sudo -E USE_DOCKER=true ./image_builder.sh "${ROOTFS_DIR}"'
+$ popd
 ```
-$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/image-builder
-$ script -fec 'sudo -E USE_DOCKER=true ./image_builder.sh ${ROOTFS_DIR}'
+
+If you want to enable SELinux on the guest, you MUST run the `image_builder.sh` script with `SELINUX=yes`
+to label the guest image as follows.
+To label the image on the host, you need to make sure that SELinux is enabled (`selinuxfs` is mounted) on the host
+and the rootfs MUST be created by running the `rootfs.sh` with `SELINUX=yes`.
+
 ```
+$ script -fec 'sudo -E USE_DOCKER=true SELINUX=yes ./image_builder.sh ${ROOTFS_DIR}'
+```
+
+Currently, the `image_builder.sh` uses `chcon` as an interim solution in order to apply `container_runtime_exec_t`
+to the `kata-agent`. Hence, if you run `restorecon` to the guest image after running the `image_builder.sh`,
+the `kata-agent` needs to be labeled `container_runtime_exec_t` again by yourself.
 
 > **Notes:**
 >
@@ -309,25 +355,31 @@ $ script -fec 'sudo -E USE_DOCKER=true ./image_builder.sh ${ROOTFS_DIR}'
 >   variable in the previous command and ensure the `qemu-img` command is
 >   available on your system.
 >   - If `qemu-img` is not installed, you will likely see errors such as `ERROR: File /dev/loop19p1 is not a block device` and `losetup: /tmp/tmp.bHz11oY851: Warning: file is smaller than 512 bytes; the loop device may be useless or invisible for system tools`. These can be mitigated by installing the `qemu-img` command (available in the `qemu-img` package on Fedora or the `qemu-utils` package on Debian).
+> - If `loop` module is not probed, you will likely see errors such as `losetup: cannot find an unused loop device`. Execute `modprobe loop` could resolve it.
 
 
 ### Install the rootfs image
 
-```
-$ commit=$(git log --format=%h -1 HEAD)
-$ date=$(date +%Y-%m-%d-%T.%N%z)
+```bash
+$ pushd kata-containers/tools/osbuilder/image-builder
+$ commit="$(git log --format=%h -1 HEAD)"
+$ date="$(date +%Y-%m-%d-%T.%N%z)"
 $ image="kata-containers-${date}-${commit}"
 $ sudo install -o root -g root -m 0640 -D kata-containers.img "/usr/share/kata-containers/${image}"
 $ (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
+$ popd
 ```
 
 ## Create an initrd image - OPTIONAL
 ### Create a local rootfs for initrd image
-```
-$ export ROOTFS_DIR="${GOPATH}/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder/rootfs"
-$ sudo rm -rf ${ROOTFS_DIR}
-$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder
-$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true ./rootfs.sh ${distro}'
+
+```bash
+$ export distro="ubuntu" # example
+$ export ROOTFS_DIR="$(realpath kata-containers/tools/osbuilder/rootfs-builder/rootfs)"
+$ sudo rm -rf "${ROOTFS_DIR}"
+$ pushd kata-containers/tools/osbuilder/rootfs-builder/
+$ script -fec 'sudo -E AGENT_INIT=yes USE_DOCKER=true ./rootfs.sh "${distro}"'
+$ popd
 ```
 `AGENT_INIT` controls if the guest image uses the Kata agent as the guest `init` process. When you create an initrd image,
 always set `AGENT_INIT` to `yes`.
@@ -335,14 +387,14 @@ always set `AGENT_INIT` to `yes`.
 You MUST choose a distribution (e.g., `ubuntu`) for `${distro}`.
 You can get a supported distributions list in the Kata Containers by running the following.
 
-```
-$ ./rootfs.sh -l
+```bash
+$ ./kata-containers/tools/osbuilder/rootfs-builder/rootfs.sh -l
 ```
 
 If you want to build the agent without seccomp capability, you need to run the `rootfs.sh` script with `SECCOMP=no` as follows.
 
-```
-$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
+```bash
+$ script -fec 'sudo -E AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh "${distro}"'
 ```
 
 > **Note:**
@@ -351,28 +403,31 @@ $ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no
 
 Optionally, add your custom agent binary to the rootfs with the following commands. The default `$LIBC` used
 is `musl`, but on ppc64le and s390x, `gnu` should be used. Also, Rust refers to ppc64le as `powerpc64le`:
-```
-$ export ARCH=$(uname -m)
-$ [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
-$ [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
-$ sudo install -o root -g root -m 0550 -T ../../../src/agent/target/${ARCH}-unknown-linux-${LIBC}/release/kata-agent ${ROOTFS_DIR}/sbin/init
+```bash
+$ export ARCH="$(uname -m)"
+$ [ "${ARCH}" == "ppc64le" ] || [ "${ARCH}" == "s390x" ] && export LIBC=gnu || export LIBC=musl
+$ [ "${ARCH}" == "ppc64le" ] && export ARCH=powerpc64le
+$ sudo install -o root -g root -m 0550 -T "${ROOTFS_DIR}/../../../../src/agent/target/${ARCH}-unknown-linux-${LIBC}/release/kata-agent" "${ROOTFS_DIR}/sbin/init"
 ```
 
 ### Build an initrd image
 
-```
-$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/initrd-builder
-$ script -fec 'sudo -E AGENT_INIT=yes USE_DOCKER=true ./initrd_builder.sh ${ROOTFS_DIR}'
+```bash
+$ pushd kata-containers/tools/osbuilder/initrd-builder
+$ script -fec 'sudo -E AGENT_INIT=yes USE_DOCKER=true ./initrd_builder.sh "${ROOTFS_DIR}"'
+$ popd
 ```
 
 ### Install the initrd image
 
-```
-$ commit=$(git log --format=%h -1 HEAD)
-$ date=$(date +%Y-%m-%d-%T.%N%z)
+```bash
+$ pushd kata-containers/tools/osbuilder/initrd-builder
+$ commit="$(git log --format=%h -1 HEAD)"
+$ date="$(date +%Y-%m-%d-%T.%N%z)"
 $ image="kata-containers-initrd-${date}-${commit}"
 $ sudo install -o root -g root -m 0640 -D kata-containers-initrd.img "/usr/share/kata-containers/${image}"
 $ (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers-initrd.img)
+$ popd
 ```
 
 # Install guest kernel images
@@ -391,44 +446,44 @@ Kata Containers makes use of upstream QEMU branch. The exact version
 and repository utilized can be found by looking at the [versions file](../versions.yaml).
 
 Find the correct version of QEMU from the versions file:
-```
-$ source ${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging/scripts/lib.sh
-$ qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version")
-$ echo ${qemu_version}
+```bash
+$ source kata-containers/tools/packaging/scripts/lib.sh
+$ qemu_version="$(get_from_kata_deps "assets.hypervisor.qemu.version")"
+$ echo "${qemu_version}"
 ```
 Get source from the matching branch of QEMU:
-```
-$ go get -d github.com/qemu/qemu
-$ cd ${GOPATH}/src/github.com/qemu/qemu
-$ git checkout ${qemu_version}
-$ your_qemu_directory=${GOPATH}/src/github.com/qemu/qemu
+```bash
+$ git clone -b "${qemu_version}" https://github.com/qemu/qemu.git
+$ your_qemu_directory="$(realpath qemu)"
 ```
 
 There are scripts to manage the build and packaging of QEMU. For the examples below, set your
 environment as:
-```
-$ go get -d github.com/kata-containers/kata-containers
-$ packaging_dir="${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging"
+```bash
+$ packaging_dir="$(realpath kata-containers/tools/packaging)"
 ```
 
 Kata often utilizes patches for not-yet-upstream and/or backported fixes for components,
 including QEMU. These can be found in the [packaging/QEMU directory](../tools/packaging/qemu/patches),
 and it's *recommended* that you apply them. For example, suppose that you are going to build QEMU
 version 5.2.0, do:
-```
-$ cd $your_qemu_directory
-$ $packaging_dir/scripts/apply_patches.sh $packaging_dir/qemu/patches/5.2.x/
+```bash
+$ "$packaging_dir/scripts/apply_patches.sh" "$packaging_dir/qemu/patches/5.2.x/"
 ```
 
 To build utilizing the same options as Kata, you should make use of the `configure-hypervisor.sh` script. For example:
-```
-$ cd $your_qemu_directory
-$ $packaging_dir/scripts/configure-hypervisor.sh kata-qemu > kata.cfg
+```bash
+$ pushd "$your_qemu_directory"
+$ "$packaging_dir/scripts/configure-hypervisor.sh" kata-qemu > kata.cfg
 $ eval ./configure "$(cat kata.cfg)"
-$ make -j $(nproc)
+$ make -j $(nproc --ignore=1)
+# Optional
 $ sudo -E make install
+$ popd
 ```
 
+If you do not want to install the respective QEMU version, the configuration file can be modified to point to the correct binary. In `/etc/kata-containers/configuration.toml`, change `path = "/path/to/qemu/build/qemu-system-x86_64"` to point to the correct QEMU binary.
+
 See the [static-build script for QEMU](../tools/packaging/static-build/qemu/build-static-qemu.sh) for a reference on how to get, setup, configure and build QEMU for Kata.
 
 ### Build a custom QEMU for aarch64/arm64 - REQUIRED
@@ -439,11 +494,33 @@ See the [static-build script for QEMU](../tools/packaging/static-build/qemu/buil
 >   under upstream review for supporting NVDIMM on aarch64.
 >
 You could build the custom `qemu-system-aarch64` as required with the following command:
+```bash
+$ git clone https://github.com/kata-containers/tests.git
+$ script -fec 'sudo -E tests/.ci/install_qemu.sh'
 ```
-$ go get -d github.com/kata-containers/tests
-$ script -fec 'sudo -E ${GOPATH}/src/github.com/kata-containers/tests/.ci/install_qemu.sh'
+
+## Build `virtiofsd`
+
+When using the file system type virtio-fs (default), `virtiofsd` is required
+
+```bash
+$ pushd kata-containers/tools/packaging/static-build/virtiofsd
+$ ./build.sh
+$ popd
 ```
 
+Modify `/etc/kata-containers/configuration.toml` and update value `virtio_fs_daemon = "/path/to/kata-containers/tools/packaging/static-build/virtiofsd/virtiofsd/virtiofsd"` to point to the binary.
+
+# Check hardware requirements
+
+You can check if your system is capable of creating a Kata Container by running the following:
+
+```bash
+$ sudo kata-runtime check
+```
+
+If your system is *not* able to run Kata Containers, the previous command will error out and explain why.
+
 # Run Kata Containers with Containerd
 Refer to the [How to use Kata Containers and Containerd](how-to/containerd-kata.md) how-to guide.
 
@@ -465,7 +542,7 @@ script and paste its output directly into a
 > [runtime](../src/runtime) repository.
 
 To perform analysis on Kata logs, use the
-[`kata-log-parser`](https://github.com/kata-containers/tests/tree/main/cmd/log-parser)
+[`kata-log-parser`](../src/tools/log-parser)
 tool, which can convert the logs into formats (e.g. JSON, TOML, XML, and YAML).
 
 See [Set up a debug console](#set-up-a-debug-console).
@@ -474,7 +551,7 @@ See [Set up a debug console](#set-up-a-debug-console).
 
 ## Checking Docker default runtime
 
-```
+```bash
 $ sudo docker info 2>/dev/null | grep -i "default runtime" | cut -d: -f2- | grep -q runc  && echo "SUCCESS" || echo "ERROR: Incorrect default Docker runtime"
 ```
 ## Set up a debug console
@@ -491,7 +568,7 @@ contain either `/bin/sh` or `/bin/bash`.
 
 Enable debug_console_enabled in the `configuration.toml` configuration file:
 
-```
+```toml
 [agent.kata]
 debug_console_enabled = true
 ```
@@ -502,7 +579,7 @@ This will pass `agent.debug_console agent.debug_console_vport=1026` to agent as
 
 For Kata Containers `2.0.x` releases, the `kata-runtime exec` command depends on the`kata-monitor` running, in order to get the sandbox's `vsock` address to connect to. Thus, first start the `kata-monitor` process.
 
-```
+```bash
 $ sudo kata-monitor
 ```
 
@@ -510,10 +587,15 @@ $ sudo kata-monitor
 
 #### Connect to debug console
 
-Command `kata-runtime exec` is used to connect to the debug console.
+You need to start a container for example:
+```bash
+$ sudo ctr run  --runtime io.containerd.kata.v2 -d docker.io/library/ubuntu:latest testdebug
+```
+
+Then, you can use the command `kata-runtime exec <sandbox id>` to connect to the debug console.
 
 ```
-$ kata-runtime exec 1a9ab65be63b8b03dfd0c75036d27f0ed09eab38abb45337fea83acd3cd7bacd
+$ kata-runtime exec testdebug
 bash-4.2# id
 uid=0(root) gid=0(root) groups=0(root)
 bash-4.2# pwd
@@ -522,7 +604,7 @@ bash-4.2# exit
 exit
 ```
 
-`kata-runtime exec` has a command-line option `runtime-namespace`, which is used to specify under which [runtime namespace](https://github.com/containerd/containerd/blob/master/docs/namespaces.md) the particular pod was created. By default, it is set to `k8s.io` and works for containerd when configured
+`kata-runtime exec` has a command-line option `runtime-namespace`, which is used to specify under which [runtime namespace](https://github.com/containerd/containerd/blob/main/docs/namespaces.md) the particular pod was created. By default, it is set to `k8s.io` and works for containerd when configured
  with Kubernetes. For CRI-O, the namespace should set to `default` explicitly. This should not be confused with [Kubernetes namespaces](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/).
 For other CRI-runtimes and configurations, you may need to set the namespace utilizing the `runtime-namespace` option.
 
@@ -564,10 +646,10 @@ an additional `coreutils` package.
 
 For example using CentOS:
 
-```
-$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder
-$ export ROOTFS_DIR=${GOPATH}/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder/rootfs
-$ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true EXTRA_PKGS="bash coreutils" ./rootfs.sh centos'
+```bash
+$ pushd kata-containers/tools/osbuilder/rootfs-builder
+$ export ROOTFS_DIR="$(realpath ./rootfs)"
+$ script -fec 'sudo -E USE_DOCKER=true EXTRA_PKGS="bash coreutils" ./rootfs.sh centos'
 ```
 
 #### Build the debug image
@@ -579,12 +661,13 @@ section when using rootfs, or when using initrd, complete the steps in the [Buil
 
 Install the image:
 
->**Note**: When using an initrd image, replace the below rootfs image name `kata-containers.img` 
+>**Note**: When using an initrd image, replace the below rootfs image name `kata-containers.img`
 >with the initrd image name `kata-containers-initrd.img`.
 
-```
+```bash
 $ name="kata-containers-centos-with-debug-console.img"
 $ sudo install -o root -g root -m 0640 kata-containers.img "/usr/share/kata-containers/${name}"
+$ popd
 ```
 
 Next, modify the `image=` values in the `[hypervisor.qemu]` section of the
@@ -593,7 +676,7 @@ to specify the full path to the image name specified in the previous code
 section. Alternatively, recreate the symbolic link so it points to
 the new debug image:
 
-```
+```bash
 $ (cd /usr/share/kata-containers && sudo ln -sf "$name" kata-containers.img)
 ```
 
@@ -604,7 +687,7 @@ to avoid all subsequently created containers from using the debug image.
 
 Create a container as normal. For example using `crictl`:
 
-```
+```bash
 $ sudo crictl run -r kata container.yaml pod.yaml
 ```
 
@@ -612,25 +695,25 @@ $ sudo crictl run -r kata container.yaml pod.yaml
 
 The steps required to enable debug console for QEMU slightly differ with
 those for firecracker / cloud-hypervisor.
- 
+
 ##### Enabling debug console for QEMU
 
-Add `agent.debug_console` to the guest kernel command line to allow the agent process to start a debug console. 
+Add `agent.debug_console` to the guest kernel command line to allow the agent process to start a debug console.
 
-```
+```bash
 $ sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.debug_console"/g' "${kata_configuration_file}"
 ```
 
-Here `kata_configuration_file` could point to `/etc/kata-containers/configuration.toml` 
+Here `kata_configuration_file` could point to `/etc/kata-containers/configuration.toml`
 or `/usr/share/defaults/kata-containers/configuration.toml`
 or `/opt/kata/share/defaults/kata-containers/configuration-{hypervisor}.toml`, if
 you installed Kata Containers using `kata-deploy`.
 
 ##### Enabling debug console for cloud-hypervisor / firecracker
 
-Slightly different configuration is required in case of firecracker and cloud hypervisor. 
-Firecracker and cloud-hypervisor don't have a UNIX socket connected to `/dev/console`. 
-Hence, the kernel command line option `agent.debug_console` will not work for them. 
+Slightly different configuration is required in case of firecracker and cloud hypervisor.
+Firecracker and cloud-hypervisor don't have a UNIX socket connected to `/dev/console`.
+Hence, the kernel command line option `agent.debug_console` will not work for them.
 These hypervisors support `hybrid vsocks`,  which can be used for communication
 between the host and the guest. The kernel command line option `agent.debug_console_vport`
  was added to allow developers specify on which `vsock` port the debugging console should be connected.
@@ -638,12 +721,12 @@ between the host and the guest. The kernel command line option `agent.debug_cons
 
 Add the parameter `agent.debug_console_vport=1026` to the kernel command line
 as shown below:
-```
+```bash
 sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.debug_console_vport=1026"/g' "${kata_configuration_file}"
 ```
 
 > **Note** Ports 1024 and 1025 are reserved for communication with the agent
-> and gathering of agent logs respectively. 
+> and gathering of agent logs respectively.
 
 ##### Connecting to the debug console
 
@@ -651,7 +734,7 @@ Next, connect to the debug console. The VSOCKS paths vary slightly between each
 VMM solution.
 
 In case of cloud-hypervisor, connect to the `vsock` as shown:
-```
+```bash
 $ sudo su -c 'cd /var/run/vc/vm/${sandbox_id}/root/ && socat stdin unix-connect:clh.sock'
 CONNECT 1026
 ```
@@ -659,7 +742,7 @@ CONNECT 1026
 **Note**: You need to type `CONNECT 1026` and press `RETURN` key after entering the `socat` command.
 
 For firecracker, connect to the `hvsock` as shown:
-```
+```bash
 $ sudo su -c 'cd /var/run/vc/firecracker/${sandbox_id}/root/ && socat stdin unix-connect:kata.hvsock'
 CONNECT 1026
 ```
@@ -668,7 +751,7 @@ CONNECT 1026
 
 
 For QEMU, connect to the `vsock` as shown:
-```
+```bash
 $ sudo su -c 'cd /var/run/vc/vm/${sandbox_id} && socat "stdin,raw,echo=0,escape=0x11" "unix-connect:console.sock"'
 ```
 
@@ -681,7 +764,7 @@ If the image is created using
 [osbuilder](../tools/osbuilder), the following YAML
 file exists and contains details of the image and how it was created:
 
-```
+```bash
 $ cat /var/lib/osbuilder/osbuilder.yaml
 ```
 
@@ -700,11 +783,11 @@ options to have the kernel boot messages logged into the system journal.
 For generic information on enabling debug in the configuration file, see the
 [Enable full debug](#enable-full-debug) section.
 
-The kernel boot messages will appear in the `containerd` or `CRI-O` log appropriately,
+The kernel boot messages will appear in the `kata` logs (and in the `containerd` or `CRI-O` log appropriately).
 such as:
 
 ```bash
-$ sudo journalctl -t containerd
+$ sudo journalctl -t kata
 -- Logs begin at Thu 2020-02-13 16:20:40 UTC, end at Thu 2020-02-13 16:30:23 UTC. --
 ...
 time="2020-09-15T14:56:23.095113803+08:00" level=debug msg="reading guest console" console-protocol=unix console-url=/run/vc/vm/ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791/console.sock pid=107642 sandbox=ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791 source=virtcontainers subsystem=sandbox vmconsole="[    0.395399] brd: module loaded"
@@ -714,3 +797,4 @@ time="2020-09-15T14:56:23.105268162+08:00" level=debug msg="reading guest consol
 time="2020-09-15T14:56:23.121121598+08:00" level=debug msg="reading guest console" console-protocol=unix console-url=/run/vc/vm/ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791/console.sock pid=107642 sandbox=ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791 source=virtcontainers subsystem=sandbox vmconsole="[    0.421324] memmap_init_zone_device initialised 32768 pages in 12ms"
 ...
 ```
+Refer to the [kata-log-parser documentation](../src/tools/log-parser/README.md) which is useful to fetch these.

docs/Limitations.md

@@ -46,7 +46,7 @@ The following link shows the latest list of limitations:
 # Contributing
 
 If you would like to work on resolving a limitation, please refer to the
-[contributors guide](https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md).
+[contributors guide](https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md).
 If you wish to raise an issue for a new limitation, either
 [raise an issue directly on the runtime](https://github.com/kata-containers/kata-containers/issues/new)
 or see the
@@ -60,17 +60,26 @@ This section lists items that might be possible to fix.
 ## OCI CLI commands
 
 ### Docker and Podman support
-Currently Kata Containers does not support Docker or Podman.
+Currently Kata Containers does not support Podman.
 
 See issue https://github.com/kata-containers/kata-containers/issues/722 for more information.
 
+Docker supports Kata Containers since 22.06:
+
+```bash
+$ sudo docker run --runtime io.containerd.kata.v2
+```
+
+Kata Containers works perfectly with containerd, we recommend to use
+containerd's Docker-style command line tool [`nerdctl`](https://github.com/containerd/nerdctl).
+
 ## Runtime commands
 
 ### checkpoint and restore
 
 The runtime does not provide `checkpoint` and `restore` commands. There
 are discussions about using VM save and restore to give us a
-`[criu](https://github.com/checkpoint-restore/criu)`-like functionality,
+[`criu`](https://github.com/checkpoint-restore/criu)-like functionality,
 which might provide a solution.
 
 Note that the OCI standard does not specify `checkpoint` and `restore`
@@ -93,6 +102,42 @@ All other configurations are supported and are working properly.
 
 ## Networking
 
+### Host network
+
+Host network (`nerdctl/docker run --net=host`or [Kubernetes `HostNetwork`](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#hosts-namespaces)) is not supported.
+It is not possible to directly access the host networking configuration
+from within the VM.
+
+The `--net=host` option can still be used with `runc` containers and
+inter-mixed with running Kata Containers, thus enabling use of `--net=host`
+when necessary.
+
+It should be noted, currently passing the `--net=host` option into a
+Kata Container may result in the Kata Container networking setup
+modifying, re-configuring and therefore possibly breaking the host
+networking setup. Do not use `--net=host` with Kata Containers.
+
+### Support for joining an existing VM network
+
+Docker supports the ability for containers to join another containers
+namespace with the `docker run --net=containers` syntax. This allows
+multiple containers to share a common network namespace and the network
+interfaces placed in the network namespace. Kata Containers does not
+support network namespace sharing. If a Kata Container is setup to
+share the network namespace of a `runc` container, the runtime
+effectively takes over all the network interfaces assigned to the
+namespace and binds them to the VM. Consequently, the `runc` container loses
+its network connectivity.
+
+### docker run --link
+
+The runtime does not support the `docker run --link` command. This
+command is now deprecated by docker and we have no intention of adding support.
+Equivalent functionality can be achieved with the newer docker networking commands.
+
+See more documentation at
+[docs.docker.com](https://docs.docker.com/network/links/).
+
 ## Resource management
 
 Due to the way VMs differ in their CPU and memory allocation, and sharing
@@ -102,7 +147,8 @@ these commands is potentially challenging.
 See issue https://github.com/clearcontainers/runtime/issues/341 and [the constraints challenge](#the-constraints-challenge) for more information.
 
 For CPUs resource management see
-[CPU constraints](design/vcpu-handling.md).
+[CPU constraints(in runtime-go)](design/vcpu-handling-runtime-go.md).
+[CPU constraints(in runtime-rs)](design/vcpu-handling-runtime-rs.md).
 
 # Architectural limitations
 

docs/README.md

@@ -30,7 +30,6 @@ See the [how-to documentation](how-to).
 * [GPU Passthrough with Kata](./use-cases/GPU-passthrough-and-Kata.md)
 * [SR-IOV with Kata](./use-cases/using-SRIOV-and-kata.md)
 * [Intel QAT with Kata](./use-cases/using-Intel-QAT-and-kata.md)
-* [VPP with Kata](./use-cases/using-vpp-and-kata.md)
 * [SPDK vhost-user with Kata](./use-cases/using-SPDK-vhostuser-and-kata.md)
 * [Intel SGX with Kata](./use-cases/using-Intel-SGX-and-kata.md)
 

docs/Release-Process.md

@@ -4,11 +4,11 @@
 ## Requirements
 
 - [hub](https://github.com/github/hub)
-  * Using an [application token](https://github.com/settings/tokens) is required for hub.
+  * Using an [application token](https://github.com/settings/tokens) is required for hub (set to a GITHUB_TOKEN environment variable).
 
 - GitHub permissions to push tags and create releases in Kata repositories.
 
-- GPG configured to sign git tags. https://help.github.com/articles/generating-a-new-gpg-key/
+- GPG configured to sign git tags. https://docs.github.com/en/authentication/managing-commit-signature-verification/generating-a-new-gpg-key
 
 - You should configure your GitHub to use your ssh keys (to push to branches). See https://help.github.com/articles/adding-a-new-ssh-key-to-your-github-account/.
     * As an alternative, configure hub to push and fork with HTTPS, `git config --global hub.protocol https` (Not tested yet) *
@@ -28,27 +28,10 @@
   $ ./update-repository-version.sh -p "$NEW_VERSION" "$BRANCH"
   ```
 
-### Point tests repository to stable branch
-
-  If you create a new stable branch, i.e. if your release changes a major or minor version number (not a patch release), then
-  you should modify the `tests` repository to point to that newly created stable branch and not the `main` branch.
-  The objective is that changes in the CI on the main branch will not impact the stable branch.
-
-  In the test directory, change references the main branch in:
-  * `README.md`
-  * `versions.yaml`
-  * `cmd/github-labels/labels.yaml.in`
-  * `cmd/pmemctl/pmemctl.sh`
-  * `.ci/lib.sh`
-  * `.ci/static-checks.sh`
-
-  See the commits in [the corresponding PR for stable-2.1](https://github.com/kata-containers/tests/pull/3504) for an example of the changes.
-
-
 ### Merge all bump version Pull requests
 
   - The above step will create a GitHub pull request in the Kata projects. Trigger the CI using `/test` command on each bump Pull request.
-  - Trigger the test-kata-deploy workflow on the kata-containers repository bump Pull request using `/test_kata_deploy` (monitor under the "action" tab).
+  - Trigger the `test-kata-deploy` workflow which is under the `Actions` tab on the repository GitHub page (make sure to select the correct branch and validate it passes).
   - Check any failures and fix if needed.
   - Work with the Kata approvers to verify that the CI works and the pull requests are merged.
 
@@ -63,6 +46,24 @@
   $ ./tag_repos.sh -p -b "$BRANCH" tag
   ```
 
+### Point tests repository to stable branch
+
+  If your release changes a major or minor version number(not a patch release), then the above 
+  `./tag_repos.sh` script will create a new stable branch in all the repositories in addition to tagging them.
+  This happens when you are making the first `rc` release for a new major or minor version in Kata.
+  In this case, you should modify the `tests` repository to point to the newly created stable branch and not the `main` branch.
+  The objective is that changes in the CI on the main branch will not impact the stable branch.
+
+  In the test directory, change references of the `main` branch to the new stable branch in:
+  * `README.md`
+  * `versions.yaml`
+  * `cmd/github-labels/labels.yaml.in`
+  * `cmd/pmemctl/pmemctl.sh`
+  * `.ci/lib.sh`
+  * `.ci/static-checks.sh`
+
+  See the commits in [the corresponding PR for stable-2.1](https://github.com/kata-containers/tests/pull/3504) for an example of the changes.
+
 ### Check Git-hub Actions
 
   We make use of [GitHub actions](https://github.com/features/actions) in this [file](../.github/workflows/release.yaml) in the `kata-containers/kata-containers` repository to build and upload release artifacts. This action is auto triggered with the above step when a new tag is pushed to the `kata-containers/kata-containers` repository.

docs/Unit-Test-Advice.md

@@ -277,7 +277,9 @@ mod tests {
 
 ## Temporary files
 
-Always delete temporary files on success.
+Use `t.TempDir()` to create temporary directory. The directory created by
+`t.TempDir()` is automatically removed when the test and all its subtests
+complete.
 
 ### Golang temporary files
 
@@ -286,11 +288,7 @@ func TestSomething(t *testing.T) {
     assert := assert.New(t)
 
     // Create a temporary directory
-    tmpdir, err := os.MkdirTemp("", "")
-    assert.NoError(err)
-
-    // Delete it at the end of the test
-    defer os.RemoveAll(tmpdir) 
+    tmpdir := t.TempDir()
 
     // Add test logic that will use the tmpdir here...
 }
@@ -322,7 +320,7 @@ mod tests {
 
 ## Test user
 
-[Unit tests are run *twice*](https://github.com/kata-containers/tests/blob/main/.ci/go-test.sh):
+[Unit tests are run *twice*](../src/runtime/go-test.sh):
 
 - as the current user
 - as the `root` user (if different to the current user)
@@ -343,7 +341,7 @@ The main repository has the most comprehensive set of skip abilities. See:
 
 One method is to use the `nix` crate along with some custom macros:
 
-```
+```rust
 #[cfg(test)]
 mod tests {
     #[allow(unused_macros)]

docs/code-pr-advice.md

@@ -79,7 +79,7 @@ a "`BUG: feature X not implemented see {bug-url}`" type error.
 - Don't use multiple log calls when a single log call could be used.
 
 - Use structured logging where possible to allow
-  [standard tooling](https://github.com/kata-containers/tests/tree/main/cmd/log-parser)
+  [standard tooling](../src/tools/log-parser)
   be able to extract the log fields.
 
 ### Names

docs/design/README.md

@@ -6,12 +6,19 @@ Kata Containers design documents:
 - [API Design of Kata Containers](kata-api-design.md)
 - [Design requirements for Kata Containers](kata-design-requirements.md)
 - [VSocks](VSocks.md)
-- [VCPU handling](vcpu-handling.md)
+- [VCPU handling(in runtime-go)](vcpu-handling-runtime-go.md)
+- [VCPU handling(in runtime-rs)](vcpu-handling-runtime-rs.md)
+- [VCPU threads pinning](vcpu-threads-pinning.md)
 - [Host cgroups](host-cgroups.md)
+- [Agent systemd cgroup](agent-systemd-cgroup.md)
 - [`Inotify` support](inotify.md)
+- [`Hooks` support](hooks-handling.md)
 - [Metrics(Kata 2.0)](kata-2-0-metrics.md)
+- [Metrics in Rust Runtime(runtime-rs)](kata-metrics-in-runtime-rs.md)
 - [Design for Kata Containers `Lazyload` ability with `nydus`](kata-nydus-design.md)
-
+- [Design for direct-assigned volume](direct-blk-device-assignment.md)
+- [Design for core-scheduling](core-scheduling.md)
+- [Virtualization Reference Architecture](kata-vra.md)
 ---
 
 - [Design proposals](proposals)

docs/design/VSocks.md

@@ -67,22 +67,15 @@ Using a proxy for multiplexing the connections between the VM and the host uses
 4.5MB per [POD][2]. In a high density deployment this could add up to GBs of
 memory that could have been used to host more PODs. When we talk about density
 each kilobyte matters and it might be the decisive factor between run another
-POD or not. For example if you have 500 PODs running in a server, the same
-amount of [`kata-proxy`][3] processes will be running and consuming for around
-2250MB of RAM. Before making the decision not to use VSOCKs, you should ask
+POD or not. Before making the decision not to use VSOCKs, you should ask
 yourself, how many more containers can run with the memory RAM consumed by the
 Kata proxies?
 
 ### Reliability
 
-[`kata-proxy`][3] is in charge of multiplexing the connections between virtual
-machine and host processes, if it dies all connections get broken. For example
-if you have a [POD][2] with 10 containers running, if `kata-proxy` dies it would
-be impossible to contact your containers, though they would still be running.
 Since communication via VSOCKs is direct, the only way to lose communication
 with the containers is if the VM itself or the `containerd-shim-kata-v2` dies, if this happens
 the containers are removed automatically.
 
 [1]: https://wiki.qemu.org/Features/VirtioVsock
-[2]: ./vcpu-handling.md#virtual-cpus-and-kubernetes-pods
-[3]: https://github.com/kata-containers/proxy
+[2]: ./vcpu-handling-runtime-go.md#virtual-cpus-and-kubernetes-pods

docs/design/agent-systemd-cgroup.md

@@ -0,0 +1,84 @@
+# Systemd Cgroup for Agent
+
+As we know, we can interact with cgroups in two ways, **`cgroupfs`** and **`systemd`**. The former is achieved by reading and writing cgroup `tmpfs` files under `/sys/fs/cgroup` while the latter is done by configuring a transient unit by requesting systemd. Kata agent uses **`cgroupfs`** by default, unless you pass the parameter `--systemd-cgroup`.
+
+## usage
+
+For systemd, kata agent configures cgroups according to the following `linux.cgroupsPath` format standard provided by `runc` (`[slice]:[prefix]:[name]`). If you don't provide a valid `linux.cgroupsPath`, kata agent will treat it as `"system.slice:kata_agent:<container-id>"`. 
+
+> Here slice is a systemd slice under which the container is placed. If empty, it defaults to system.slice, except when cgroup v2 is used and rootless container is created, in which case it defaults to user.slice.
+>
+> Note that slice can contain dashes to denote a sub-slice (e.g. user-1000.slice is a correct notation, meaning a `subslice` of user.slice), but it must not contain slashes (e.g. user.slice/user-1000.slice is invalid).
+>
+> A slice of `-` represents a root slice.
+>
+> Next, prefix and name are used to compose the unit name, which is `<prefix>-<name>.scope`, unless name has `.slice` suffix, in which case prefix is ignored and the name is used as is.
+
+## supported properties
+
+The kata agent will translate the parameters in the `linux.resources` of `config.json` into systemd unit properties, and send it to systemd for configuration. Since systemd supports limited properties, only the following parameters in `linux.resources` will be applied. We will simply treat hybrid mode as legacy mode by the way.
+
+- CPU
+
+  - v1
+
+  | runtime spec resource | systemd property name |
+  | --------------------- | --------------------- |
+  | `cpu.shares`          | `CPUShares`           |
+
+  - v2
+
+  | runtime spec resource      | systemd property name      |
+  | -------------------------- | -------------------------- |
+  | `cpu.shares`               | `CPUShares`                |
+  | `cpu.period`               | `CPUQuotaPeriodUSec`(v242) |
+  | `cpu.period` & `cpu.quota` | `CPUQuotaPerSecUSec`       |
+
+- MEMORY
+
+  - v1
+
+  | runtime spec resource | systemd property name |
+  | --------------------- | --------------------- |
+  | `memory.limit`        | `MemoryLimit`         |
+
+  - v2
+
+  | runtime spec resource          | systemd property name |
+  | ------------------------------ | --------------------- |
+  | `memory.low`                   | `MemoryLow`           |
+  | `memory.max`                   | `MemoryMax`           |
+  | `memory.swap` & `memory.limit` | `MemorySwapMax`       |
+
+- PIDS
+
+  | runtime spec resource | systemd property name |
+  | --------------------- | --------------------- |
+  | `pids.limit `         | `TasksMax`            |
+
+- CPUSET
+
+  | runtime spec resource | systemd property name      |
+  | --------------------- | -------------------------- |
+  | `cpuset.cpus`         | `AllowedCPUs`(v244)        |
+  | `cpuset.mems`         | `AllowedMemoryNodes`(v244) |
+
+## Systemd Interface
+
+`session.rs` and `system.rs` in `src/agent/rustjail/src/cgroups/systemd/interface` are automatically generated by `zbus-xmlgen`, which is is an accompanying tool provided by `zbus` to generate Rust code from `D-Bus XML interface descriptions`. The specific commands to generate these two files are as follows: 
+
+```shell
+// system.rs
+zbus-xmlgen --system org.freedesktop.systemd1 /org/freedesktop/systemd1
+// session.rs
+zbus-xmlgen --session org.freedesktop.systemd1 /org/freedesktop/systemd1
+```
+
+The current implementation of `cgroups/systemd` uses `system.rs` while `session.rs` could be used to build rootless containers in the future.
+
+## references
+
+- [runc - systemd cgroup driver](https://github.com/opencontainers/runc/blob/main/docs/systemd.md)
+
+- [systemd.resource-control  — Resource control unit settings](https://www.freedesktop.org/software/systemd/man/systemd.resource-control.html)
+

docs/design/architecture/kubernetes.md

@@ -3,11 +3,11 @@
 [Kubernetes](https://github.com/kubernetes/kubernetes/), or K8s, is a popular open source
 container orchestration engine. In Kubernetes, a set of containers sharing resources
 such as networking, storage, mount, PID, etc. is called a
-[pod](https://kubernetes.io/docs/user-guide/pods/).
+[pod](https://kubernetes.io/docs/concepts/workloads/pods/).
 
 A node can have multiple pods, but at a minimum, a node within a Kubernetes cluster
 only needs to run a container runtime and a container agent (called a
-[Kubelet](https://kubernetes.io/docs/admin/kubelet/)).
+[Kubelet](https://kubernetes.io/docs/concepts/overview/components/#kubelet)).
 
 Kata Containers represents a Kubelet pod as a VM.
 
@@ -17,7 +17,7 @@ Kubelet instance is responsible for managing the lifecycle of pods
 within the nodes and eventually relies on a container runtime to
 handle execution. The Kubelet architecture decouples lifecycle
 management from container execution through a dedicated gRPC based
-[Container Runtime Interface (CRI)](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/node/container-runtime-interface-v1.md).
+[Container Runtime Interface (CRI)](https://github.com/kubernetes/design-proposals-archive/blob/main/node/container-runtime-interface-v1.md).
 
 In other words, a Kubelet is a CRI client and expects a CRI
 implementation to handle the server side of the interface.

docs/design/architecture/networking.md

@@ -36,7 +36,7 @@ compatibility, and performance on par with MACVTAP.
 Kata Containers has deprecated support for bridge due to lacking performance relative to TC-filter and MACVTAP.
 
 Kata Containers supports both
-[CNM](https://github.com/docker/libnetwork/blob/master/docs/design.md#the-container-network-model)
+[CNM](https://github.com/moby/libnetwork/blob/master/docs/design.md#the-container-network-model)
 and [CNI](https://github.com/containernetworking/cni) for networking management.
 
 ## Network Hotplug

docs/design/architecture/storage.md

@@ -1,5 +1,17 @@
 # Storage
 
+## Limits
+
+Kata Containers is [compatible](README.md#compatibility) with existing
+standards and runtime. From the perspective of storage, this means no
+limits are placed on the amount of storage a container
+[workload](README.md#workload) may use.
+
+Since cgroups are not able to set limits on storage allocation, if you
+wish to constrain the amount of storage a container uses, consider
+using an existing facility such as `quota(1)` limits or
+[device mapper](#devicemapper) limits.
+
 ## virtio SCSI
 
 If a block-based graph driver is [configured](README.md#configuration),
@@ -20,7 +32,7 @@ For virtio-fs, the [runtime](README.md#runtime) starts one `virtiofsd` daemon
 ## Devicemapper
 
 The
-[devicemapper `snapshotter`](https://github.com/containerd/containerd/tree/master/snapshots/devmapper)
+[devicemapper `snapshotter`](https://github.com/containerd/containerd/tree/main/snapshots/devmapper)
 is a special case. The `snapshotter` uses dedicated block devices
 rather than formatted filesystems, and operates at the block level
 rather than the file level. This knowledge is used to directly use the

docs/design/architecture_3.0/README.md

@@ -0,0 +1,169 @@
+# Kata 3.0 Architecture
+## Overview
+In cloud-native scenarios, there is an increased demand for container startup speed, resource consumption, stability, and security, areas where the present Kata Containers runtime is challenged relative to other runtimes. To achieve this, we propose a solid, field-tested and secure Rust version of the kata-runtime.
+
+Also, we provide the following designs:
+
+- Turn key solution with builtin `Dragonball` Sandbox
+- Async I/O to reduce resource consumption
+- Extensible framework for multiple services, runtimes and hypervisors
+- Lifecycle management for sandbox and container associated resources
+
+### Rationale for choosing Rust
+
+We chose Rust because it is designed as a system language with a focus on efficiency.
+In contrast to Go, Rust makes a variety of design trade-offs in order to obtain
+good execution performance, with innovative techniques that, in contrast to C or
+C++, provide reasonable protection against common memory errors (buffer
+overflow, invalid pointers, range errors), error checking (ensuring errors are
+dealt with), thread safety, ownership of resources, and more.
+
+These benefits were verified in our project when the Kata Containers guest agent
+was rewritten in Rust. We notably saw a significant reduction in memory usage
+with the Rust-based implementation.
+
+
+## Design
+### Architecture
+![architecture](./images/architecture.png)
+### Built-in VMM
+#### Current Kata 2.x architecture
+![not_builtin_vmm](./images/not_built_in_vmm.png)
+As shown in the figure, runtime and VMM are separate processes. The runtime process forks the VMM process and interacts through the inter-process RPC. Typically, process interaction consumes more resources than peers within the process, and it will result in relatively low efficiency. At the same time, the cost of resource operation and maintenance should be considered. For example, when performing resource recovery under abnormal conditions, the exception of any process must be detected by others and activate the appropriate resource recovery process. If there are additional processes, the recovery becomes even more difficult.
+#### How To Support Built-in VMM
+We provide `Dragonball` Sandbox to enable built-in VMM by integrating VMM's function into the Rust library. We could perform VMM-related functionalities by using the library. Because runtime and VMM  are in the same process, there is a benefit in terms of message processing speed and API synchronization. It can also guarantee the consistency of the runtime and the VMM life cycle, reducing resource recovery and exception handling maintenance, as shown in the figure:
+![builtin_vmm](./images/built_in_vmm.png)
+### Async Support
+#### Why Need Async
+**Async is already in stable Rust and allows us to write async code**
+
+- Async provides significantly reduced CPU and memory overhead, especially for workloads with a large amount of IO-bound tasks
+- Async is zero-cost in Rust, which means that you only pay for what you use. Specifically, you can use async without heap allocations and dynamic dispatch, which greatly improves efficiency
+- For more (see [Why Async?](https://rust-lang.github.io/async-book/01_getting_started/02_why_async.html) and [The State of Asynchronous Rust](https://rust-lang.github.io/async-book/01_getting_started/03_state_of_async_rust.html)).
+
+**There may be several problems if implementing kata-runtime with Sync Rust**
+
+- Too many threads with a new TTRPC connection
+   - TTRPC threads: reaper thread(1) + listener thread(1) + client handler(2)
+- Add 3 I/O threads with a new container
+- In Sync mode, implementing a timeout mechanism is challenging. For example, in TTRPC API interaction, the timeout mechanism is difficult to align with Golang
+#### How To Support Async
+The kata-runtime is controlled by TOKIO_RUNTIME_WORKER_THREADS to run the OS thread, which is 2 threads by default. For TTRPC and container-related threads run in the `tokio` thread in a unified manner, and related dependencies need to be switched to Async, such as Timer, File, Netlink, etc. With the help of Async, we can easily support no-block I/O and timer. Currently, we only utilize Async for kata-runtime. The built-in VMM keeps the OS thread because it can ensure that the threads are controllable.
+
+**For N tokio worker threads and M containers**
+
+- Sync runtime(both OS thread and `tokio` task are OS thread but without `tokio` worker thread)  OS thread number:  4 + 12*M
+- Async runtime(only OS thread is OS thread) OS thread number: 2 + N
+```shell
+├─ main(OS thread)
+├─ async-logger(OS thread)
+└─ tokio worker(N * OS thread)
+  ├─ agent log forwarder(1 * tokio task)
+  ├─ health check thread(1 * tokio task)
+  ├─ TTRPC reaper thread(M * tokio task)
+  ├─ TTRPC listener thread(M * tokio task)
+  ├─ TTRPC client handler thread(7 * M * tokio task)
+  ├─ container stdin io thread(M * tokio task)
+  ├─ container stdout io thread(M * tokio task)
+  └─ container stderr io thread(M * tokio task)
+```
+### Extensible Framework
+The Kata 3.x runtime is designed with the extension of service, runtime, and hypervisor, combined with configuration to meet the needs of different scenarios. At present, the service provides a register mechanism to support multiple services. Services could interact with runtime through messages. In addition, the runtime handler handles messages from services. To meet the needs of a binary that supports multiple runtimes and hypervisors, the startup must obtain the runtime handler type and hypervisor type through configuration.
+
+![framework](./images/framework.png)
+### Resource Manager
+In our case, there will be a variety of resources, and every resource has several subtypes. Especially for `Virt-Container`, every subtype of resource has different operations. And there may be dependencies, such as the share-fs rootfs and the share-fs volume will use share-fs resources to share files to the VM. Currently, network and share-fs are regarded as sandbox resources, while rootfs, volume, and cgroup are regarded as container resources. Also, we abstract a common interface for each resource and use subclass operations to evaluate the differences between different subtypes.
+![resource manager](./images/resourceManager.png)
+
+## Roadmap
+
+- Stage 1 (June): provide basic features (current delivered)
+- Stage 2 (September): support common features
+- Stage 3: support full features
+
+| **Class**                  | **Sub-Class**       | **Development Stage** | **Status** |
+| -------------------------- | ------------------- | --------------------- |------------|
+| Service                    | task service        | Stage 1               |  ✅        |
+|                            | extend service      | Stage 3               |  🚫        |
+|                            | image service       | Stage 3               |  🚫        |
+| Runtime handler            | `Virt-Container`    | Stage 1               |  ✅        |
+| Endpoint                   | VETH Endpoint       | Stage 1               |  ✅        |
+|                            | Physical Endpoint   | Stage 2               |  ✅        |
+|                            | Tap Endpoint        | Stage 2               |  ✅        |
+|                            | `Tuntap` Endpoint   | Stage 2               |  ✅        |
+|                            | `IPVlan` Endpoint   | Stage 2               |  ✅        |
+|                            | `MacVlan` Endpoint  | Stage 2               |  ✅        |
+|                            | MACVTAP Endpoint    | Stage 3               |  🚫        |
+|                            | `VhostUserEndpoint` | Stage 3               |  🚫        |
+| Network Interworking Model | Tc filter           | Stage 1               |  ✅        |
+|                            | `MacVtap`           | Stage 3               |  🚧        |
+| Storage                    | Virtio-fs           | Stage 1               |  ✅        |
+|                            | `nydus`             | Stage 2               |  🚧        |
+|                            | `device mapper`     | Stage 2               |  🚫        |
+| `Cgroup V2`                |                     | Stage 2               |  🚧        |
+| Hypervisor                 | `Dragonball`        | Stage 1               |  🚧        |
+|                            | QEMU                | Stage 2               |  🚫        |
+|                            | ACRN                | Stage 3               |  🚫        |
+|                            | Cloud Hypervisor    | Stage 3               |  🚫        |
+|                            | Firecracker         | Stage 3               |  🚫        |
+
+## FAQ
+
+- Are the "service", "message dispatcher" and "runtime handler" all part of the single Kata 3.x runtime binary?
+
+  Yes. They are components in Kata 3.x runtime. And they will be packed into one binary.
+  1. Service is an interface, which is responsible for handling multiple services like task service, image service and etc. 
+  2. Message dispatcher, it is used to match multiple requests from the service module. 
+  3. Runtime handler is used to deal with the operation for sandbox and container. 
+- What is the name of the Kata 3.x runtime binary?
+  
+  Apparently we can't use `containerd-shim-v2-kata` because it's already used. We are facing the hardest issue of "naming" again. Any suggestions are welcomed.
+  Internally we use `containerd-shim-v2-rund`.
+
+- Is the Kata 3.x design compatible with the containerd shimv2 architecture? 
+  
+  Yes. It is designed to follow the functionality of go version kata.  And it implements the `containerd shim v2` interface/protocol.
+
+- How will users migrate to the Kata 3.x architecture?
+  
+  The migration plan will be provided before the Kata 3.x is merging into the main branch.
+
+- Is `Dragonball` limited to its own built-in VMM? Can the `Dragonball` system be configured to work using an external `Dragonball` VMM/hypervisor?
+
+  The `Dragonball` could work as an external hypervisor. However, stability and performance is challenging in this case. Built in VMM could optimise the container overhead, and it's easy to maintain stability.
+
+  `runD` is the `containerd-shim-v2` counterpart of `runC` and can run a pod/containers. `Dragonball` is a `microvm`/VMM that is designed to run container workloads. Instead of `microvm`/VMM, we sometimes refer to it as secure sandbox.
+
+- QEMU, Cloud Hypervisor and Firecracker support are planned, but how that would work. Are they working in separate process?
+ 
+  Yes. They are unable to work as built in VMM.
+
+- What is `upcall`?
+  
+    The `upcall` is used to hotplug CPU/memory/MMIO devices, and it solves two issues.
+    1. avoid dependency on PCI/ACPI
+    2. avoid dependency on `udevd` within guest and get deterministic results for hotplug operations. So `upcall` is an alternative to ACPI based CPU/memory/device hotplug. And we may cooperate with the community to add support for ACPI based CPU/memory/device hotplug if needed.
+   
+    `Dbs-upcall` is a `vsock-based` direct communication tool between VMM and guests. The server side of the `upcall` is a driver in guest kernel (kernel patches are needed for this feature) and it'll start to serve the requests once the kernel has started. And the client side is in VMM , it'll be a thread that communicates with VSOCK through `uds`. We have accomplished device hotplug / hot-unplug directly through `upcall` in order to avoid virtualization of ACPI  to minimize virtual machine's overhead. And there could be many other usage through this direct communication channel. It's already open source.
+   https://github.com/openanolis/dragonball-sandbox/tree/main/crates/dbs-upcall 
+
+- The URL below says the kernel patches work with 4.19, but do they also work with 5.15+ ?
+  
+  Forward compatibility should be achievable, we have ported it to 5.10 based kernel.
+
+- Are these patches platform-specific or would they work for any architecture that supports VSOCK?
+  
+  It's almost platform independent, but some message related to CPU hotplug are platform dependent.
+
+- Could the kernel driver be replaced with a userland daemon in the guest using loopback VSOCK?
+  
+  We need to create device nodes for hot-added CPU/memory/devices, so it's not easy for userspace daemon to do these tasks.
+
+- The fact that `upcall` allows communication between the VMM and the guest suggests that this architecture might be incompatible with https://github.com/confidential-containers where the VMM should have no knowledge of what happens inside the VM.
+  
+  1. `TDX` doesn't support CPU/memory hotplug yet.
+  2. For ACPI based device hotplug, it depends on ACPI `DSDT` table, and the guest kernel will execute `ASL` code to handle during handling those hotplug event. And it should be easier to audit VSOCK based communication than ACPI `ASL` methods.
+
+- What is the security boundary for the monolithic / "Built-in VMM" case?
+  
+  It has the security boundary of virtualization. More details will be provided in next stage.

docs/design/core-scheduling.md

@@ -0,0 +1,12 @@
+# Core scheduling
+
+Core scheduling is a Linux kernel feature that allows only trusted tasks to run concurrently on
+CPUs sharing compute resources (for example, hyper-threads on a core).
+
+Containerd versions >= 1.6.4 leverage this to treat all of the processes associated with a
+given pod or container to be a single group of trusted tasks. To indicate this should be carried
+out, containerd sets the `SCHED_CORE` environment variable for each shim it spawns. When this is
+set, the Kata Containers shim implementation uses the `prctl` syscall to create a new core scheduling
+domain for the shim process itself as well as future VMM processes it will start.
+
+For more details on the core scheduling feature, see the [Linux documentation](https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/core-scheduling.html).

docs/design/direct-blk-device-assignment.md

@@ -0,0 +1,253 @@
+# Motivation
+Today, there exist a few gaps between Container Storage Interface (CSI) and virtual machine (VM) based runtimes such as Kata Containers 
+that prevent them from working together smoothly.
+
+First, it’s cumbersome to use a persistent volume (PV) with Kata Containers. Today, for a PV with Filesystem volume mode, Virtio-fs
+is the only way to surface it inside a Kata Container guest VM. But often mounting the filesystem (FS) within the guest operating system (OS) is 
+desired due to performance benefits, availability of native FS features and security benefits over the Virtio-fs mechanism.
+
+Second, it’s difficult if not impossible to resize a PV online with Kata Containers. While a PV can be expanded on the host OS, 
+the updated metadata needs to be propagated to the guest OS in order for the application container to use the expanded volume. 
+Currently, there is not a way to propagate the PV metadata from the host OS to the guest OS without restarting the Pod sandbox.
+
+# Proposed Solution
+
+Because of the OS boundary, these features cannot be implemented in the CSI node driver plugin running on the host OS 
+as is normally done in the runc container. Instead, they can be done by the Kata Containers agent inside the guest OS, 
+but it requires the CSI driver to pass the relevant information to the Kata Containers runtime. 
+An ideal long term solution would be to have the `kubelet` coordinating the communication between the CSI driver and 
+the container runtime, as described in [KEP-2857](https://github.com/kubernetes/enhancements/pull/2893/files). 
+However, as the KEP is still under review, we would like to propose a short/medium term solution to unblock our use case.
+
+The proposed solution is built on top of a previous [proposal](https://github.com/egernst/kata-containers/blob/da-proposal/docs/design/direct-assign-volume.md) 
+described by Eric Ernst. The previous proposal has two gaps:
+
+1. Writing a `csiPlugin.json` file to the volume root path introduced a security risk. A malicious user can gain unauthorized 
+access to a block device by writing their own `csiPlugin.json` to the above location through an ephemeral CSI plugin.  
+
+2. The proposal didn't describe how to establish a mapping between a volume and a kata sandbox, which is needed for 
+implementing CSI volume resize and volume stat collection APIs.
+
+This document particularly focuses on how to address these two gaps.
+
+## Assumptions and Limitations
+1. The proposal assumes that a block device volume will only be used by one Pod on a node at a time, which we believe 
+is the most common pattern in Kata Containers use cases. It’s also unsafe to have the same block device attached to more than 
+one Kata pod. In the context of Kubernetes, the `PersistentVolumeClaim` (PVC) needs to have the `accessMode` as `ReadWriteOncePod`. 
+2. More advanced Kubernetes volume features such as, `fsGroup`, `fsGroupChangePolicy`, and `subPath` are not supported. 
+
+## End User Interface
+
+1. The user specifies a PV as a direct-assigned volume. How a PV is specified as a direct-assigned volume is left for each CSI implementation to decide.
+There are a few options for reference:
+   1. A storage class parameter specifies whether it's a direct-assigned volume. This avoids any lookups of PVC 
+   or Pod information from the CSI plugin (as external provisioner takes care of these). However, all PVs in the storage class with the parameter set 
+   will have host mounts skipped.
+   2. Use a PVC annotation. This approach requires the CSI plugins have `--extra-create-metadata` [set](https://kubernetes-csi.github.io/docs/external-provisioner.html#persistentvolumeclaim-and-persistentvolume-parameters)
+   to be able to perform a lookup of the PVC annotations from the API server. Pro: API server lookup of annotations only required during creation of PV. 
+   Con: The CSI plugin will always skip host mounting of the PV.
+   3. The CSI plugin can also lookup pod `runtimeclass` during `NodePublish`. This approach can be found in the [ALIBABA CSI plugin](https://github.com/kubernetes-sigs/alibaba-cloud-csi-driver/blob/master/pkg/disk/nodeserver.go#L248).
+2. The CSI node driver delegates the direct assigned volume to the Kata Containers runtime. The CSI node driver APIs need to 
+   be modified to pass the volume mount information and collect volume information to/from the Kata Containers runtime by invoking `kata-runtime` command line commands.
+   * **NodePublishVolume** -- It invokes `kata-runtime direct-volume add --volume-path [volumePath] --mount-info [mountInfo]` 
+   to propagate the volume mount information to the Kata Containers runtime for it to carry out the filesystem mount operation.
+   The `volumePath` is the [target_path](https://github.com/container-storage-interface/spec/blob/master/csi.proto#L1364) in the CSI `NodePublishVolumeRequest`.
+   The `mountInfo` is a serialized JSON string. 
+   * **NodeGetVolumeStats** -- It invokes `kata-runtime direct-volume stats --volume-path [volumePath]` to retrieve the filesystem stats of direct-assigned volume.
+   * **NodeExpandVolume** -- It invokes `kata-runtime direct-volume resize --volume-path [volumePath] --size [size]` to send a resize request to the Kata Containers runtime to
+   resize the direct-assigned volume.
+   * **NodeStageVolume/NodeUnStageVolume** -- It invokes `kata-runtime direct-volume remove --volume-path [volumePath]` to remove the persisted metadata of a direct-assigned volume.
+
+The `mountInfo` object is defined as follows:
+```Golang
+type MountInfo struct {
+    // The type of the volume (ie. block)
+    VolumeType string `json:"volume-type"`
+    // The device backing the volume.
+    Device string `json:"device"`
+    // The filesystem type to be mounted on the volume.
+    FsType string `json:"fstype"`
+    // Additional metadata to pass to the agent regarding this volume.
+    Metadata map[string]string `json:"metadata,omitempty"`
+    // Additional mount options.
+    Options []string `json:"options,omitempty"`
+}
+```
+Notes: given that the `mountInfo` is persisted to the disk by the Kata runtime, it shouldn't container any secrets (such as SMB mount password).
+
+## Implementation Details
+
+### Kata runtime
+Instead of the CSI node driver writing the mount info into a `csiPlugin.json` file under the volume root, 
+as described in the original proposal, here we propose that the CSI node driver passes the mount information to 
+the Kata Containers runtime through a new `kata-runtime` commandline command. The `kata-runtime` then writes the mount 
+information to a `mountInfo.json` file in a predefined location (`/run/kata-containers/shared/direct-volumes/[volume_path]/`).
+
+When the Kata Containers runtime starts a container, it verifies whether a volume mount is a direct-assigned volume by checking 
+whether there is a `mountInfo` file under the computed Kata `direct-volumes` directory. If it is, the runtime parses the `mountInfo` file, 
+updates the mount spec with the data in `mountInfo`. The updated mount spec is then passed to the Kata agent in the guest VM together
+with other mounts. The Kata Containers runtime also creates a file named by the sandbox id under the `direct-volumes/[volume_path]/` 
+directory. The reason for adding a sandbox id file is to establish a mapping between the volume and the sandbox using it. 
+Later, when the Kata Containers runtime handles the `get-stats` and `resize` commands, it uses the sandbox id to identify 
+the endpoint of the corresponding `containerd-shim-kata-v2`.
+
+### containerd-shim-kata-v2 changes
+`containerd-shim-kata-v2` provides an API for sandbox management through a Unix domain socket. Two new handlers are proposed: `/direct-volume/stats` and `/direct-volume/resize`:
+
+Example:
+
+```bash
+$ curl --unix-socket "$shim_socket_path" -I -X GET 'http://localhost/direct-volume/stats/[urlSafeVolumePath]'
+$ curl --unix-socket "$shim_socket_path" -I -X POST 'http://localhost/direct-volume/resize' -d '{ "volumePath"": [volumePath], "Size": "123123" }'
+```
+
+The shim then forwards the corresponding request to the `kata-agent` to carry out the operations inside the guest VM. For `resize` operation, 
+the Kata runtime also needs to notify the hypervisor to resize the block device (e.g. call `block_resize` in QEMU). 
+
+### Kata agent changes
+
+The mount spec of a direct-assigned volume is passed to `kata-agent` through the existing `Storage` GRPC object. 
+Two new APIs and three new GRPC objects are added to GRPC protocol between the shim and agent for resizing and getting volume stats:
+```protobuf
+
+rpc GetVolumeStats(VolumeStatsRequest) returns (VolumeStatsResponse);
+rpc ResizeVolume(ResizeVolumeRequest) returns (google.protobuf.Empty);
+
+message VolumeStatsRequest {
+// The volume path on the guest outside the container
+    string volume_guest_path = 1;
+}
+
+message ResizeVolumeRequest {
+// Full VM guest path of the volume (outside the container)
+    string volume_guest_path = 1;
+    uint64 size = 2;
+}
+
+// This should be kept in sync with CSI NodeGetVolumeStatsResponse (https://github.com/container-storage-interface/spec/blob/v1.5.0/csi.proto)
+message VolumeStatsResponse {
+   // This field is OPTIONAL.
+   repeated VolumeUsage usage = 1;
+   // Information about the current condition of the volume.
+   // This field is OPTIONAL.
+   // This field MUST be specified if the VOLUME_CONDITION node
+   // capability is supported.
+   VolumeCondition volume_condition = 2;
+}
+message VolumeUsage {
+   enum Unit {
+      UNKNOWN = 0;
+      BYTES = 1;
+      INODES = 2;
+   }
+   // The available capacity in specified Unit. This field is OPTIONAL.
+   // The value of this field MUST NOT be negative.
+   uint64 available = 1;
+
+   // The total capacity in specified Unit. This field is REQUIRED.
+   // The value of this field MUST NOT be negative.
+   uint64 total = 2;
+
+   // The used capacity in specified Unit. This field is OPTIONAL.
+   // The value of this field MUST NOT be negative.
+   uint64 used = 3;
+
+   // Units by which values are measured. This field is REQUIRED.
+   Unit unit = 4;
+}
+
+// VolumeCondition represents the current condition of a volume.
+message VolumeCondition {
+
+   // Normal volumes are available for use and operating optimally.
+   // An abnormal volume does not meet these criteria.
+   // This field is REQUIRED.
+   bool abnormal = 1;
+
+   // The message describing the condition of the volume.
+   // This field is REQUIRED.
+   string message = 2;
+}
+
+```
+
+### Step by step walk-through
+
+Given the following definition:
+```YAML
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: app
+spec:
+  runtime-class: kata-qemu
+  containers:
+  - name: app
+    image: centos
+    command: ["/bin/sh"]
+    args: ["-c", "while true; do echo $(date -u) >> /data/out.txt; sleep 5; done"]
+    volumeMounts:
+    - name: persistent-storage
+      mountPath: /data
+  volumes:
+  - name: persistent-storage
+    persistentVolumeClaim:
+      claimName: ebs-claim
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  annotations:
+    skip-hostmount: "true"
+  name: ebs-claim
+spec:
+  accessModes:
+    - ReadWriteOncePod
+  volumeMode: Filesystem
+  storageClassName: ebs-sc
+  resources:
+    requests:
+      storage: 4Gi
+---
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: ebs-sc
+provisioner: ebs.csi.aws.com
+volumeBindingMode: WaitForFirstConsumer
+parameters:
+  csi.storage.k8s.io/fstype: ext4
+
+```
+Let’s assume that changes have been made in the `aws-ebs-csi-driver` node driver.
+
+**Node publish volume**
+1. In the node CSI driver, the `NodePublishVolume` API invokes: `kata-runtime direct-volume add --volume-path "/kubelet/a/b/c/d/sdf" --mount-info "{\"Device\": \"/dev/sdf\", \"fstype\": \"ext4\"}"`.
+2. The `Kata-runtime` writes the mount-info JSON to a file called `mountInfo.json` under `/run/kata-containers/shared/direct-volumes/kubelet/a/b/c/d/sdf`.
+
+**Node unstage volume**
+1. In the node CSI driver, the `NodeUnstageVolume` API invokes: `kata-runtime direct-volume remove --volume-path "/kubelet/a/b/c/d/sdf"`.
+2. Kata-runtime deletes the directory `/run/kata-containers/shared/direct-volumes/kubelet/a/b/c/d/sdf`.
+
+**Use the volume in sandbox**
+1. Upon the request to start a container, the `containerd-shim-kata-v2` examines the container spec,
+and iterates through the mounts. For each mount, if there is a `mountInfo.json` file under `/run/kata-containers/shared/direct-volumes/[mount source path]`,
+it generates a `storage` GRPC object after overwriting the mount spec with the information in `mountInfo.json`.
+2. The shim sends the storage objects to kata-agent through TTRPC.
+3. The shim writes a file with the sandbox id as the name under `/run/kata-containers/shared/direct-volumes/[mount source path]`.
+4. The kata-agent mounts the storage objects for the container.
+
+**Node expand volume**
+1. In the node CSI driver, the `NodeExpandVolume` API invokes: `kata-runtime direct-volume resize –-volume-path "/kubelet/a/b/c/d/sdf" –-size 8Gi`.
+2. The Kata runtime checks whether there is a sandbox id file under the directory `/run/kata-containers/shared/direct-volumes/kubelet/a/b/c/d/sdf`.
+3. The Kata runtime identifies the shim instance through the sandbox id, and sends a GRPC request to resize the volume.
+4. The shim handles the request, asks the hypervisor to resize the block device and sends a GRPC request to Kata agent to resize the filesystem.
+5. Kata agent receives the request and resizes the filesystem.
+
+**Node get volume stats**
+1. In the node CSI driver, the `NodeGetVolumeStats` API invokes: `kata-runtime direct-volume stats –-volume-path "/kubelet/a/b/c/d/sdf"`.
+2. The Kata runtime checks whether there is a sandbox id file under the directory `/run/kata-containers/shared/direct-volumes/kubelet/a/b/c/d/sdf`.
+3. The Kata runtime identifies the shim instance through the sandbox id, and sends a GRPC request to get the volume stats.
+4. The shim handles the request and forwards it to the Kata agent.
+5. Kata agent receives the request and returns the filesystem stats.

docs/design/hooks-handling.md

@@ -0,0 +1,63 @@
+# Kata Containers support for `Hooks`
+
+## Introduction
+
+During container's lifecycle, different Hooks can be executed to do custom actions. In Kata Containers, we support two types of Hooks, `OCI Hooks` and `Kata Hooks`.
+
+### OCI Hooks
+
+The OCI Spec stipulates six hooks that can be executed at different time points and namespaces, including `Prestart Hooks`, `CreateRuntime Hooks`, `CreateContainer Hooks`, `StartContainer Hooks`, `Poststart Hooks` and `Poststop Hooks`. We support these types of Hooks as compatible as possible in Kata Containers.
+
+The path and arguments of these hooks will be passed to Kata for execution via `bundle/config.json`. For example:
+```
+...
+"hooks": {
+  "prestart": [
+    {
+      "path": "/usr/bin/prestart-hook",
+      "args": ["prestart-hook", "arg1", "arg2"],
+      "env":  [ "key1=value1"]
+    }
+  ],
+  "createRuntime": [
+    {
+      "path": "/usr/bin/createRuntime-hook",
+      "args": ["createRuntime-hook", "arg1", "arg2"],
+      "env":  [ "key1=value1"]
+    }
+  ]
+}
+...
+```
+
+### Kata Hooks
+
+In Kata, we support another three kinds of hooks executed in guest VM, including `Guest Prestart Hook`, `Guest Poststart Hook`, `Guest Poststop Hook`.
+
+The executable files for Kata Hooks must be packaged in the *guest rootfs*. The file path to those guest hooks should be specified in the configuration file, and guest hooks must be stored in a subdirectory of `guest_hook_path` according to their hook type. For example:
+
++ In configuration file:
+```
+guest_hook_path="/usr/share/hooks"
+```
++ In guest rootfs, prestart-hook is stored in `/usr/share/hooks/prestart/prestart-hook`.
+
+## Execution
+The table below summarized when and where those different hooks will be executed in Kata Containers:
+
+| Hook Name | Hook Type | Hook Path | Exec Place | Exec Time |
+|---|---|---|---|---|
+| `Prestart(deprecated)` | OCI hook | host runtime namespace | host runtime namespace | After VM is started, before container is created. |
+| `CreateRuntime` | OCI hook | host runtime namespace | host runtime namespace | After VM is started, before container is created, after `Prestart` hooks. |
+| `CreateContainer` | OCI hook | host runtime namespace | host vmm namespace* | After VM is started, before container is created, after `CreateRuntime` hooks. |
+| `StartContainer` | OCI hook | guest container namespace | guest container namespace | After container is created, before container is started. |
+| `Poststart` | OCI hook | host runtime namespace | host runtime namespace | After container is started, before start operation returns. |
+| `Poststop` | OCI hook | host runtime namespace | host runtime namespace | After container is deleted, before delete operation returns. |
+| `Guest Prestart` | Kata hook | guest agent namespace | guest agent namespace | During start operation, before container command is executed. |
+| `Guest Poststart` | Kata hook | guest agent namespace | guest agent namespace | During start operation, after container command is executed, before start operation returns. |
+| `Guest Poststop` | Kata hook | guest agent namespace | guest agent namespace | During delete operation, after container is deleted, before delete operation returns. |
+
++ `Hook Path` specifies where hook's path be resolved.
++ `Exec Place` specifies in which namespace those hooks can be executed.
+  + For `CreateContainer` Hooks, OCI requires to run them inside the container namespace while the hook executable path is in the host runtime, which is a non-starter for VM-based containers. So we design to keep them running in the *host vmm namespace.* 
++ `Exec Time` specifies at which time point those hooks can be executed.

docs/design/host-cgroups.md

@@ -12,7 +12,7 @@ The OCI [runtime specification][linux-config] provides guidance on where the con
   > [`cgroupsPath`][cgroupspath]: (string, OPTIONAL) path to the cgroups. It can be used to either control the cgroups
   > hierarchy for containers or to run a new process in an existing container
 
-Cgroups are hierarchical, and this can be seen with the following pod example:
+The cgroups are hierarchical, and this can be seen with the following pod example:
 
 - Pod 1: `cgroupsPath=/kubepods/pod1`
   - Container 1: `cgroupsPath=/kubepods/pod1/container1`
@@ -247,14 +247,14 @@ cgroup size and constraints accordingly.
 
 # Supported cgroups
 
-Kata Containers currently only supports cgroups `v1`. 
+Kata Containers currently supports cgroups `v1` and `v2`. 
 
 In the following sections each cgroup is described briefly.
 
-## Cgroups V1
+## cgroups v1
 
-`Cgroups V1` are under a [`tmpfs`][1] filesystem mounted at `/sys/fs/cgroup`, where each cgroup is
-mounted under a separate cgroup filesystem. A `Cgroups v1` hierarchy may look like the following
+`cgroups v1` are under a [`tmpfs`][1] filesystem mounted at `/sys/fs/cgroup`, where each cgroup is
+mounted under a separate cgroup filesystem. A `cgroups v1` hierarchy may look like the following
 diagram:
 
 ```
@@ -301,13 +301,12 @@ diagram:
 A process can join a cgroup by writing its process id (`pid`) to `cgroup.procs` file,
 or join a cgroup partially by writing the task (thread) id (`tid`) to the `tasks` file.
 
-Kata Containers only supports `v1`.
 To know more about `cgroups v1`, see [cgroupsv1(7)][2].
 
-## Cgroups V2
+## cgroups v2
 
-`Cgroups v2` are also known as unified cgroups, unlike `cgroups v1`, the cgroups are
-mounted under the same cgroup filesystem. A `Cgroups v2` hierarchy may look like the following
+`cgroups v2` are also known as unified cgroups, unlike `cgroups v1`, the cgroups are
+mounted under the same cgroup filesystem. A `cgroups v2` hierarchy may look like the following
 diagram:
 
 ```
@@ -354,8 +353,6 @@ Same as `cgroups v1`, a process can join the cgroup by writing its process id (`
 `cgroup.procs` file, or join a cgroup partially by writing the task (thread) id (`tid`) to
 `cgroup.threads` file.
 
-Kata Containers does not support cgroups `v2` on the host.
-
 ### Distro Support
 
 Many Linux distributions do not yet support `cgroups v2`, as it is quite a recent addition.

docs/design/kata-2-0-metrics.md

@@ -51,6 +51,7 @@ The `kata-monitor` management agent should be started on each node where the Kat
 > **Note**: a *node* running Kata containers will be either a single host system or a worker node belonging to a K8s cluster capable of running Kata pods.
 
 - Aggregate sandbox metrics running on the node, adding the `sandbox_id` label to them.
+- Attach the additional `cri_uid`, `cri_name` and `cri_namespace` labels to the sandbox metrics, tracking the `uid`, `name` and `namespace` Kubernetes pod metadata.
 - Expose a new Prometheus target, allowing all node metrics coming from the Kata shim to be collected by Prometheus indirectly. This simplifies the targets count in Prometheus and avoids exposing shim's metrics by `ip:port`.
 
 Only one `kata-monitor` process runs in each node.

docs/design/kata-metrics-in-runtime-rs.md

@@ -0,0 +1,50 @@
+# Kata Metrics in Rust Runtime(runtime-rs)
+
+Rust Runtime(runtime-rs) is responsible for:
+
+- Gather metrics about `shim`.
+- Gather metrics from `hypervisor` (through `channel`).
+- Get metrics from `agent` (through `ttrpc`).
+
+---
+
+Here are listed all the metrics gathered by `runtime-rs`.
+
+> * Current status of each entry is marked as:
+>  * ✅：DONE
+>   * 🚧：TODO
+
+### Kata Shim
+
+| STATUS | Metric name                                                  | Type        | Units          | Labels                                                       |
+| ------ | ------------------------------------------------------------ | ----------- | -------------- | ------------------------------------------------------------ |
+| 🚧      | `kata_shim_agent_rpc_durations_histogram_milliseconds`: <br> RPC latency distributions. | `HISTOGRAM` | `milliseconds` | <ul><li>`action` (RPC actions of Kata agent)<ul><li>`grpc.CheckRequest`</li><li>`grpc.CloseStdinRequest`</li><li>`grpc.CopyFileRequest`</li><li>`grpc.CreateContainerRequest`</li><li>`grpc.CreateSandboxRequest`</li><li>`grpc.DestroySandboxRequest`</li><li>`grpc.ExecProcessRequest`</li><li>`grpc.GetMetricsRequest`</li><li>`grpc.GuestDetailsRequest`</li><li>`grpc.ListInterfacesRequest`</li><li>`grpc.ListProcessesRequest`</li><li>`grpc.ListRoutesRequest`</li><li>`grpc.MemHotplugByProbeRequest`</li><li>`grpc.OnlineCPUMemRequest`</li><li>`grpc.PauseContainerRequest`</li><li>`grpc.RemoveContainerRequest`</li><li>`grpc.ReseedRandomDevRequest`</li><li>`grpc.ResumeContainerRequest`</li><li>`grpc.SetGuestDateTimeRequest`</li><li>`grpc.SignalProcessRequest`</li><li>`grpc.StartContainerRequest`</li><li>`grpc.StatsContainerRequest`</li><li>`grpc.TtyWinResizeRequest`</li><li>`grpc.UpdateContainerRequest`</li><li>`grpc.UpdateInterfaceRequest`</li><li>`grpc.UpdateRoutesRequest`</li><li>`grpc.WaitProcessRequest`</li><li>`grpc.WriteStreamRequest`</li></ul></li><li>`sandbox_id`</li></ul> |
+| ✅      | `kata_shim_fds`: <br> Kata containerd shim v2 open FDs.      | `GAUGE`     |                | <ul><li>`sandbox_id`</li></ul>                               |
+| ✅      | `kata_shim_io_stat`: <br> Kata containerd shim v2 process IO statistics. | `GAUGE`     |                | <ul><li>`item` (see `/proc/<pid>/io`)<ul><li>`cancelledwritebytes`</li><li>`rchar`</li><li>`readbytes`</li><li>`syscr`</li><li>`syscw`</li><li>`wchar`</li><li>`writebytes`</li></ul></li><li>`sandbox_id`</li></ul> |
+| ✅      | `kata_shim_netdev`: <br> Kata containerd shim v2 network devices statistics. | `GAUGE`     |                | <ul><li>`interface` (network device name)</li><li>`item` (see `/proc/net/dev`)<ul><li>`recv_bytes`</li><li>`recv_compressed`</li><li>`recv_drop`</li><li>`recv_errs`</li><li>`recv_fifo`</li><li>`recv_frame`</li><li>`recv_multicast`</li><li>`recv_packets`</li><li>`sent_bytes`</li><li>`sent_carrier`</li><li>`sent_colls`</li><li>`sent_compressed`</li><li>`sent_drop`</li><li>`sent_errs`</li><li>`sent_fifo`</li><li>`sent_packets`</li></ul></li><li>`sandbox_id`</li></ul> |
+| 🚧      | `kata_shim_pod_overhead_cpu`: <br> Kata Pod overhead for CPU resources(percent). | `GAUGE`     | percent        | <ul><li>`sandbox_id`</li></ul>                               |
+| 🚧      | `kata_shim_pod_overhead_memory_in_bytes`: <br> Kata Pod overhead for memory resources(bytes). | `GAUGE`     | `bytes`        | <ul><li>`sandbox_id`</li></ul>                               |
+| ✅      | `kata_shim_proc_stat`: <br> Kata containerd shim v2 process statistics. | `GAUGE`     |                | <ul><li>`item` (see `/proc/<pid>/stat`)<ul><li>`cstime`</li><li>`cutime`</li><li>`stime`</li><li>`utime`</li></ul></li><li>`sandbox_id`</li></ul> |
+| ✅      | `kata_shim_proc_status`: <br> Kata containerd shim v2 process status. | `GAUGE`     |                | <ul><li>`item` (see `/proc/<pid>/status`)<ul><li>`hugetlbpages`</li><li>`nonvoluntary_ctxt_switches`</li><li>`rssanon`</li><li>`rssfile`</li><li>`rssshmem`</li><li>`vmdata`</li><li>`vmexe`</li><li>`vmhwm`</li><li>`vmlck`</li><li>`vmlib`</li><li>`vmpeak`</li><li>`vmpin`</li><li>`vmpmd`</li><li>`vmpte`</li><li>`vmrss`</li><li>`vmsize`</li><li>`vmstk`</li><li>`vmswap`</li><li>`voluntary_ctxt_switches`</li></ul></li><li>`sandbox_id`</li></ul> |
+| 🚧      | `kata_shim_process_cpu_seconds_total`: <br> Total user and system CPU time spent in seconds. | `COUNTER`   | `seconds`      | <ul><li>`sandbox_id`</li></ul>                               |
+| 🚧      | `kata_shim_process_max_fds`: <br> Maximum number of open file descriptors. | `GAUGE`     |                | <ul><li>`sandbox_id`</li></ul>                               |
+| 🚧      | `kata_shim_process_open_fds`: <br> Number of open file descriptors. | `GAUGE`     |                | <ul><li>`sandbox_id`</li></ul>                               |
+| 🚧      | `kata_shim_process_resident_memory_bytes`: <br> Resident memory size in bytes. | `GAUGE`     | `bytes`        | <ul><li>`sandbox_id`</li></ul>                               |
+| 🚧      | `kata_shim_process_start_time_seconds`: <br> Start time of the process since `unix` epoch in seconds. | `GAUGE`     | `seconds`      | <ul><li>`sandbox_id`</li></ul>                               |
+| 🚧      | `kata_shim_process_virtual_memory_bytes`: <br> Virtual memory size in bytes. | `GAUGE`     | `bytes`        | <ul><li>`sandbox_id`</li></ul>                               |
+| 🚧      | `kata_shim_process_virtual_memory_max_bytes`: <br> Maximum amount of virtual memory available in bytes. | `GAUGE`     | `bytes`        | <ul><li>`sandbox_id`</li></ul>                               |
+| 🚧      | `kata_shim_rpc_durations_histogram_milliseconds`: <br> RPC latency distributions. | `HISTOGRAM` | `milliseconds` | <ul><li>`action` (Kata shim v2 actions)<ul><li>`checkpoint`</li><li>`close_io`</li><li>`connect`</li><li>`create`</li><li>`delete`</li><li>`exec`</li><li>`kill`</li><li>`pause`</li><li>`pids`</li><li>`resize_pty`</li><li>`resume`</li><li>`shutdown`</li><li>`start`</li><li>`state`</li><li>`stats`</li><li>`update`</li><li>`wait`</li></ul></li><li>`sandbox_id`</li></ul> |
+| ✅      | `kata_shim_threads`: <br> Kata containerd shim v2 process threads. | `GAUGE`     |                | <ul><li>`sandbox_id`</li></ul>                               |
+
+### Kata Hypervisor
+
+Different from golang runtime, hypervisor and shim in runtime-rs belong to the **same process**, so all previous metrics for hypervisor and shim only need to be gathered once. Thus, we currently only collect previous metrics in kata shim.
+
+At the same time, we added the interface(`VmmAction::GetHypervisorMetrics`) to gather hypervisor metrics, in case we design tailor-made metrics for hypervisor in the future. Here're metrics exposed from [src/dragonball/src/metric.rs](https://github.com/kata-containers/kata-containers/blob/main/src/dragonball/src/metric.rs).
+
+| Metric name                                                  | Type       | Units | Labels                                                       |
+| ------------------------------------------------------------ | ---------- | ----- | ------------------------------------------------------------ |
+| `kata_hypervisor_scrape_count`: <br> Metrics scrape count    | `COUNTER`  |       | <ul><li>`sandbox_id`</li></ul>                               |
+| `kata_hypervisor_vcpu`: <br>Hypervisor metrics specific to VCPUs' mode of functioning. | `IntGauge` |       | <ul><li>`item`<ul><li>`exit_io_in`</li><li>`exit_io_out`</li><li>`exit_mmio_read`</li><li>`exit_mmio_write`</li><li>`failures`</li><li>`filter_cpuid`</li></ul></li><li>`sandbox_id`</li></ul> |
+| `kata_hypervisor_seccomp`: <br> Hypervisor metrics for the seccomp filtering. | `IntGauge` |       | <ul><li>`item`<ul><li>`num_faults`</li></ul></li><li>`sandbox_id`</li></ul> |
+| `kata_hypervisor_seccomp`: <br> Hypervisor metrics for the seccomp filtering. | `IntGauge` |       | <ul><li>`item`<ul><li>`sigbus`</li><li>`sigsegv`</li></ul></li><li>`sandbox_id`</li></ul> |

docs/design/kata-vra.md

@@ -0,0 +1,434 @@
+# Virtualization Reference Architecture
+
+## Subject to Change | © 2022 by NVIDIA Corporation. All rights reserved. | For test and development only_
+
+Before digging deeper into the virtualization reference architecture, let's
+first look at the various GPUDirect use cases in the following table. We’re
+distinguishing between two top-tier use cases where the devices are (1)
+passthrough and (2) virtualized, where a VM gets assigned a virtual function
+(VF) and not the physical function (PF). A combination of PF and VF would also
+be possible.
+
+| Device #1  (passthrough)  | Device #2 (passthrough) | P2P Compatibility and Mode                   |
+| ------------------------- | ----------------------- | -------------------------------------------- |
+| GPU PF                    | GPU PF                  | GPUDirect P2P                                |
+| GPU PF                    | NIC PF                  | GPUDirect RDMA                               |
+| MIG-slice                 | MIG-slice               | _No GPUDirect P2P_                           |
+| MIG-slice                 | NIC PF                  | GPUDirect RDMA                               |
+| **PDevice #1  (virtualized)** | **Device #2 (virtualized)** | **P2P Compatibility and   Mode**     |
+| Time-slice vGPU VF        | Time-slice vGPU VF      | _No GPUDirect P2P  but NVLINK P2P available_ |
+| Time-slice vGPU VF        | NIC VF                  | GPUDirect RDMA                               |
+| MIG-slice vGPU            | MIG-slice vGPU          | _No GPUDirect P2P_                           |
+| MIG-slice vGPU            | NIC VF                  | GPUDirect RDMA                               |
+
+In a virtualized environment we have several distinct features that may prevent
+Peer-to-peer (P2P) communication of two endpoints in a PCI Express topology. The
+IOMMU translates IO virtual addresses (IOVA) to physical addresses (PA). Each
+device behind an IOMMU has its own IOVA memory space, usually, no two devices
+share the same IOVA memory space but it’s up to the hypervisor or OS how it
+chooses to map devices to IOVA spaces.  Any PCI Express DMA transactions will
+use IOVAs, which the IOMMU must translate. By default, all the traffic is routed
+to the root complex and not issued directly to the peer device.
+
+An IOMMU can be used to isolate and protect devices even if virtualization is
+not used; since devices can only access memory regions that are mapped for it, a
+DMA from one device to another is not possible. DPDK uses the IOMMU to have
+better isolation between devices, another benefit is that IOVA space can be
+represented as a contiguous memory even if the PA space is heavily scattered.
+
+In the case of virtualization, the IOMMU is responsible for isolating the device
+and memory between VMs for safe device assignment without compromising the host
+and other guest OSes. Without an IOMMU, any device can access the entire system
+and perform DMA transactions _anywhere_.
+
+The second feature is ACS (Access Control Services), which controls which
+devices are allowed to communicate with one another and thus avoids improper
+routing of packets `irrespectively` of whether IOMMU is enabled or not.
+
+When IOMMU is enabled, ACS is normally configured to force all PCI Express DMA
+to go through the root complex so IOMMU can translate it, impacting performance
+between peers with higher latency and reduced bandwidth.
+
+A way to avoid the performance hit is to enable Address Translation Services
+(ATS). ATS-capable endpoints can prefetch IOVA -> PA translations from the IOMMU
+and then perform DMA transactions directly to another endpoint. Hypervisors
+enable this by enabling ATS in such endpoints, configuring ACS to enable Direct
+Translated P2P, and configuring the IOMMU to allow Address Translation requests.
+
+Another important factor is that the NVIDIA driver stack will use the PCI
+Express topology of the system it is running on to determine whether the
+hardware is capable of supporting P2P. The driver stack qualifies specific
+chipsets, and PCI Express switches for use with GPUDirect P2P. In virtual
+environments, the PCI Express topology is flattened and obfuscated to present a
+uniform environment to the software inside the VM, which breaks the GPUDirect
+P2P use case.
+
+On a bare metal machine, the driver stack groups GPUs into cliques that can
+perform GPUDirect P2P communication, excluding peer mappings where P2P
+communication is not possible, prominently if GPUs are attached to multiple CPU
+sockets.  
+
+CPUs and local memory banks are referred to as NUMA nodes. In a two-socket
+server, each of the CPUs has a local memory bank for a total of two NUMA nodes.
+Some servers provide the ability to configure additional NUMA nodes per CPU,
+which means a CPU socket can have two NUMA nodes  (some servers support four
+NUMA nodes per socket) with local memory banks and L3 NUMA domains for improved
+performance.
+
+One of the current solutions is that the hypervisor provides additional topology
+information that the driver stack can pick up and enable GPUDirect P2P between
+GPUs, even if the virtualized environment does not directly expose it. The PCI
+Express virtual P2P approval capability structure in the PCI configuration space
+is entirely emulated by the hypervisor of passthrough GPU devices.
+
+A clique ID is provided where GPUs with the same clique ID belong to a group of
+GPUs capable of P2P communication
+
+On vSphere, Azure, and other CPSs,  the hypervisor lays down a `topologies.xml`
+which NCCL can pick up and deduce the right P2P level[^1]. NCCL is leveraging
+Infiniband (IB) and/or Unified Communication X (UCX) for communication, and
+GPUDirect P2P and GPUDirect RDMA should just work in this case. The only culprit
+is that software or applications that do not use the XML file to deduce the
+topology will fail and not enable GPUDirect ( [`nccl-p2p-level`](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/env.html#nccl-p2p-level) )
+
+## Hypervisor PCI Express Topology
+
+To enable every part of the accelerator stack, we propose a virtualized
+reference architecture to enable GPUDirect P2P and GPUDirect RDMA for any
+hypervisor. The idea is split into two parts to enable the right PCI Express
+topology. The first part builds upon extending the PCI Express virtual P2P
+approval capability structure to every device that wants to do P2P in some way
+and groups devices by clique ID. The other part involves replicating a subset of
+the host topology so that applications running in the VM do not need to read
+additional information and enable the P2P capability like in the bare-metal use
+case described above. The driver stack can then deduce automatically if the
+topology presented in the VM is capable of P2P communication.
+
+We will work with the following host topology for the following sections. It is
+a system with two converged DPUs, each having an `A100X` GPU and two `ConnectX-6`
+network ports connected to the downstream ports of a PCI Express switch.
+
+```sh
++-00.0-[d8-df]----00.0-[d9-df]--+-00.0-[da-db]--+-00.0  Mellanox Tech MT42822 BlueField-2 integrated ConnectX-6 Dx network
+                                |               +-00.1  Mellanox Tech MT42822 BlueField-2 integrated ConnectX-6 Dx network
+                                |               \-00.2  Mellanox Tech MT42822 BlueField-2 SoC Management Interface
+                                 \-01.0-[dc-df]----00.0-[dd-df]----08.0-[de-df]----00.0  NVIDIA Corporation GA100 [A100X]
+
++-00.0-[3b-42]----00.0-[3c-42]--+-00.0-[3d-3e]--+-00.0  Mellanox Tech MT42822 BlueField-2 integrated ConnectX-6 Dx network
+                                |               +-00.1  Mellanox Tech MT42822 BlueField-2 integrated ConnectX-6 Dx network
+                                |               \-00.2  Mellanox Tech MT42822 BlueField-2 SoC Management Interface
+                                 \-01.0-[3f-42]----00.0-[40-42]----08.0-[41-42]----00.0  NVIDIA Corporation GA100 [A100X]
+```
+
+The green path highlighted above is the optimal and preferred path for
+efficient P2P communication.
+
+## PCI Express Virtual P2P Approval Capability
+
+Most of the time, the PCI Express topology is flattened and obfuscated to ensure
+easy migration of the VM image between different physical hardware `topologies`.
+In Kata, we can configure the hypervisor to use PCI Express root ports to
+hotplug the VFIO  devices one is passing through. A user can select how many PCI
+Express root ports to allocate depending on how many devices are passed through.
+A recent addition to Kata will detect the right amount of PCI Express devices
+that need hotplugging and bail out if the number of root ports is insufficient.
+In Kata, we do not automatically increase the number of root ports, we want the
+user to be in full control of the topology.
+
+```toml
+# /etc/kata-containers/configuration.toml
+
+# VFIO devices are hotplugged on a bridge by default.
+# Enable hot-plugging on the root bus. This may be required for devices with
+# a large PCI bar, as this is a current limitation with hot-plugging on
+# a bridge.
+# Default “bridge-port”
+hotplug_vfio = "root-port"
+
+# Before hot plugging a PCIe device, you need to add a pcie_root_port device.
+# Use this parameter when using some large PCI bar devices, such as NVIDIA GPU
+# The value means the number of pcie_root_port
+# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35"
+# Default 0
+pcie_root_port = 8
+```
+
+VFIO devices are hotplugged on a PCIe-PCI bridge by default. Hotplug of PCI
+Express devices is only supported on PCI Express root or downstream ports. With
+this configuration set, if we start up a Kata container, we can inspect our
+topology and see the allocated PCI Express root ports and the hotplugged
+devices.
+
+```sh
+$ lspci -tv
+ -[0000:00]-+-00.0  Intel Corporation 82G33/G31/P35/P31 Express DRAM Controller
+           +-01.0  Red Hat, Inc. Virtio console
+           +-02.0  Red Hat, Inc. Virtio SCSI
+           +-03.0  Red Hat, Inc. Virtio RNG
+           +-04.0-[01]----00.0  Mellanox Technologies MT42822 BlueField-2 integrated ConnectX-6
+           +-05.0-[02]----00.0  Mellanox Technologies MT42822 BlueField-2 integrated ConnectX-6
+           +-06.0-[03]----00.0  NVIDIA Corporation Device 20b8
+           +-07.0-[04]----00.0  NVIDIA Corporation Device 20b8
+           +-08.0-[05]--
+           +-09.0-[06]--
+           +-0a.0-[07]--
+           +-0b.0-[08]--
+           +-0c.0  Red Hat, Inc. Virtio socket
+           +-0d.0  Red Hat, Inc. Virtio file system
+           +-1f.0  Intel Corporation 82801IB (ICH9) LPC Interface Controller
+           +-1f.2  Intel Corporation 82801IR/IO/IH (ICH9R/DO/DH) 6 port SATA Controller
+           \-1f.3  Intel Corporation 82801I (ICH9 Family) SMBus Controller
+```
+
+For devices with huge BARs (Base Address Registers) like the GPU (we need to
+configure the PCI Express root port properly and allocate enough memory for
+mapping), we have added a heuristic to Kata to deduce the right settings. Hence,
+the BARs can be mapped correctly. This functionality is added to
+[`nvidia/go-nvlib1](https://gitlab.com/nvidia/cloud-native/go-nvlib) which is part
+of Kata now.
+
+```sh
+$ sudo dmesg | grep BAR
+[    0.179960] pci 0000:00:04.0: BAR 7: assigned [io  0x1000-0x1fff]
+[    0.179962] pci 0000:00:05.0: BAR 7: assigned [io  0x2000-0x2fff]
+[    0.179963] pci 0000:00:06.0: BAR 7: assigned [io  0x3000-0x3fff]
+[    0.179964] pci 0000:00:07.0: BAR 7: assigned [io  0x4000-0x4fff]
+[    0.179966] pci 0000:00:08.0: BAR 7: assigned [io  0x5000-0x5fff]
+[    0.179967] pci 0000:00:09.0: BAR 7: assigned [io  0x6000-0x6fff]
+[    0.179968] pci 0000:00:0a.0: BAR 7: assigned [io  0x7000-0x7fff]
+[    0.179969] pci 0000:00:0b.0: BAR 7: assigned [io  0x8000-0x8fff]
+[    2.115912] pci 0000:01:00.0: BAR 0: assigned [mem 0x13000000000-0x13001ffffff 64bit pref]
+[    2.116203] pci 0000:01:00.0: BAR 2: assigned [mem 0x13002000000-0x130027fffff 64bit pref]
+[    2.683132] pci 0000:02:00.0: BAR 0: assigned [mem 0x12000000000-0x12001ffffff 64bit pref]
+[    2.683419] pci 0000:02:00.0: BAR 2: assigned [mem 0x12002000000-0x120027fffff 64bit pref]
+[    2.959155] pci 0000:03:00.0: BAR 1: assigned [mem 0x11000000000-0x117ffffffff 64bit pref]
+[    2.959345] pci 0000:03:00.0: BAR 3: assigned [mem 0x11800000000-0x11801ffffff 64bit pref]
+[    2.959523] pci 0000:03:00.0: BAR 0: assigned [mem 0xf9000000-0xf9ffffff]
+[    2.966119] pci 0000:04:00.0: BAR 1: assigned [mem 0x10000000000-0x107ffffffff 64bit pref]
+[    2.966295] pci 0000:04:00.0: BAR 3: assigned [mem 0x10800000000-0x10801ffffff 64bit pref]
+[    2.966472] pci 0000:04:00.0: BAR 0: assigned [mem 0xf7000000-0xf7ffffff]
+```
+
+The NVIDIA driver stack in this case would refuse to do P2P communication since
+(1) the topology is not what it expects, (2)  we do not have a qualified
+chipset. Since our P2P devices are not connected to a PCI Express switch port,
+we need to provide additional information to support the P2P functionality. One
+way of providing such meta information would be to annotate the container; most
+of the settings in Kata's configuration file can be overridden via annotations,
+but this limits the flexibility, and a user would need to update all the
+containers that he wants to run with Kata. The goal is to make such things as
+transparent as possible, so we also introduced
+[CDI](https://github.com/container-orchestrated-devices/container-device-interface)
+(Container Device Interface) to Kata. CDI is a[
+specification](https://github.com/container-orchestrated-devices/container-device-interface/blob/main/SPEC.md)
+for container runtimes to support third-party devices.
+
+As written before, we can provide a clique ID for the devices that belong
+together and are capable of doing P2P. This information is provided to the
+hypervisor, which will set up things in the VM accordingly. Let's suppose the
+user wanted to do GPUDirect RDMA with the first GPU and the NIC that reside on
+the same DPU, one could provide the specification telling the hypervisor that
+they belong to the same clique.
+
+```yaml
+# /etc/cdi/nvidia.yaml
+cdiVersion: 0.4.0
+kind: nvidia.com/gpu
+devices:
+- name: gpu0
+  annotations:
+    bdf: “41:00.0”
+    clique-id: “0”
+  containerEdits:
+    deviceNodes:
+    - path: “/dev/vfio/71"
+
+# /etc/cdi/mellanox.yaml
+cdiVersion: 0.4.0
+kind: mellanox.com/nic
+devices:
+- name: nic0
+  annotations:
+    bdf: “3d:00.0”
+    clique-id: “0”
+    attach-pci: “true”
+  containerEdits:
+    deviceNodes:
+    - path: "/dev/vfio/66"
+```
+
+Since this setting is bound to the device and not the container we do not need
+to alter the container just allocate the right resource and GPUDirect RDMA would
+be set up correctly. Rather than exposing them separately, an idea would be to
+expose a GPUDirect RDMA device via NFD (Node Feature Discovery) that combines
+both of them; this way, we could make sure that the right pair is allocated and
+used more on  Kubernetes deployment in the next section.
+
+The GPU driver stack is leveraging the PCI Express virtual P2P approval
+capability, but the NIC stack does not use this now. One of the action items is
+to enable MOFED to read the P2P approval capability and enable ATS and ACS
+settings as described above.
+
+This way, we could enable GPUDirect P2P and GPUDirect RDMA on any topology
+presented to the VM application. It is the responsibility of the administrator
+or infrastructure engineer to provide the right information either via
+annotations or a CDI specification.
+
+## Host Topology Replication
+
+The other way to represent the PCI Express topology in the VM is to replicate a
+subset of the topology needed to support the P2P use case inside the VM. Similar
+to the configuration for the root ports, we can easily configure the usage of
+PCI Express switch ports to hotplug the devices.
+
+```toml
+# /etc/kata-containers/configuration.toml
+
+# VFIO devices are hotplugged on a bridge by default.
+# Enable hot plugging on the root bus. This may be required for devices with
+# a large PCI bar, as this is a current limitation with hot plugging on
+# a bridge.
+# Default “bridge-port”
+hotplug_vfio = "switch-port"
+
+# Before hot plugging a PCIe device, you need to add a pcie_root_port device.
+# Use this parameter when using some large PCI bar devices, such as Nvidia GPU
+# The value means the number of pcie_root_port
+# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35"
+# Default 0
+pcie_switch_port = 8
+```
+
+Each device that is passed through is attached to a PCI Express downstream port
+as illustrated below. We can even replicate the host’s two DPUs `topologies` with
+added metadata through the CDI. Most of the time, a container only needs one
+pair of GPU and NIC for GPUDirect RDMA. This is more of a showcase of what we
+can do with the power of Kata and CDI. One could even think of adding groups of
+devices that support P2P, even from different CPU sockets or NUMA nodes, into
+one container; indeed, the first group is NUMA node 0 (red), and the second
+group is NUMA node 1 (green). Since they are grouped correctly, P2P would be
+enabled naturally inside a group, aka clique ID.
+
+```sh
+$ lspci -tv
+ -[0000:00]-+-00.0  Intel Corporation 82G33/G31/P35/P31 Express DRAM Controller
+            +-01.0  Red Hat, Inc. Virtio console
+            +-02.0  Red Hat, Inc. Virtio SCSI
+            +-03.0  Red Hat, Inc. Virtio RNG
+            +-04.0-[01-04]----00.0-[02-04]--+-00.0-[03]----00.0  NVIDIA Corporation Device 20b8
+            |                               \-01.0-[04]----00.0  Mellanox Tech MT42822 BlueField-2 integrated ConnectX-6 Dx
+            +-05.0-[05-08]----00.0-[06-08]--+-00.0-[07]----00.0  Mellanox Tech MT42822 BlueField-2 integrated ConnectX-6 Dx
+            |                               \-01.0-[08]----00.0  NVIDIA Corporation Device 20b8
+            +-06.0  Red Hat, Inc. Virtio socket
+            +-07.0  Red Hat, Inc. Virtio file system
+            +-1f.0  Intel Corporation 82801IB (ICH9) LPC Interface Controller
+            +-1f.2  Intel Corporation 82801IR/IO/IH (ICH9R/DO/DH) 6 port SATA Controller [AHCI mode]
+            \-1f.3  Intel Corporation 82801I (ICH9 Family) SMBus Controller
+            \-1f.3  Intel Corporation 82801I (ICH9 Family) SMBus Controller
+```
+
+The configuration of using either the root port or switch port can be applied on
+a per Container or Pod basis, meaning we can switch PCI Express `topologies` on
+each run of an application.
+
+## Hypervisor Resource Limits
+
+Every hypervisor will have resource limits in terms of how many PCI Express root
+ports, switch ports, or bridge ports can be created, especially with devices
+that need to reserve a 4K IO range per PCI specification. Each instance of root
+or switch port will consume 4K IO of very limited capacity, 64k is the maximum.
+
+Simple math brings us to the conclusion that we can have a maximum of 16 PCI
+Express root ports or 16 PCI Express switch ports in QEMU if devices with IO
+BARs are used in the PCI Express hierarchy.
+
+Additionally, one can have 32 slots on the PCI root bus and a maximum of 256
+slots for the complete PCI(e) topology.
+
+Per default, QEMU will attach a multi-function device in the last slot on the
+PCI root bus,
+
+```sh
+ +-1f.0  Intel Corporation 82801IB (ICH9) LPC Interface Controller
+ +-1f.2  Intel Corporation 82801IR/IO/IH (ICH9R/DO/DH) 6 port SATA Controller [AHCI mode]
+ \-1f.3  Intel Corporation 82801I (ICH9 Family) SMBus Controller
+```
+
+Kata will additionally add `virtio-xxx-pci` devices consuming (5 slots) plus a
+PCIe-PCI-bridge (1 slot) and a DRAM controller (1 slot), meaning per default, we
+have already eight slots used. This leaves us 24 slots for adding other devices
+to the root bus.
+
+The problem that arises here is one use-case from a customer that uses recent
+RTX GPUs with Kata. The user wanted to pass through eight of these GPUs into one
+container and ran into issues. The problem is that those cards often consist of
+four individual device nodes: GPU, Audio, and two USB controller devices (some
+cards have a USB-C output).
+
+These devices are grouped into one IOMMU group. Since one needs to pass through
+the complete IOMMU group into the VM, we need to allocate 32 PCI Express root
+ports or 32 PCI Express switch ports, which is technically impossible due to the
+resource limits outlined above. Since all the devices appear as PCI Express
+devices, we need to hotplug those into a root or switch port.
+
+The solution to this problem is leveraging CDI. For each device, add the
+information if it is going to be hotplugged as a PCI Express or PCI device,
+which results in either using a PCI Express root/switch port or an ordinary PCI
+bridge. PCI bridges are not affected by the limited IO range. This way, the GPU
+is attached as a PCI Express device to a root/switch port and the other three
+PCI devices to a PCI bridge, leaving enough resources to create the needed PCI
+Express root/switch ports.  For example, we’re going to attach the GPUs to a PCI
+Express root port and the NICs to a PCI bridge.
+
+```jsonld
+# /etc/cdi/mellanox.json
+cdiVersion: 0.4.0
+kind: mellanox.com/nic
+devices:
+- name: nic0
+  annotations:
+    bdf: “3d:00.0”
+    clique-id: “0”
+    attach-pci: “true”
+  containerEdits:
+    deviceNodes:
+    - path: "/dev/vfio/66"
+- name: nic1
+  annotations:
+    bdf: “3d:00.1”
+    clique-id: “1”
+    attach-pci: “true”
+  containerEdits:
+    deviceNodes:
+    - path: "/dev/vfio/67”
+```
+
+The configuration is set to use eight root ports for the GPUs and attach the
+NICs to a PCI bridge which is connected to a PCI Express-PCI bridge which is the
+preferred way of introducing a PCI topology in a PCI Express machine.
+
+```sh
+$ lspci -tv
+-[0000:00]-+-00.0  Intel Corporation 82G33/G31/P35/P31 Express DRAM Controller
+           +-01.0  Red Hat, Inc. Virtio console
+           +-02.0  Red Hat, Inc. Virtio SCSI
+           +-03.0  Red Hat, Inc. Virtio RNG
+           +-04.0-[01]----00.0  NVIDIA Corporation Device 20b8
+           +-05.0-[02]----00.0  NVIDIA Corporation Device 20b8
+           +-06.0-[03]--
+           +-07.0-[04]--
+           +-08.0-[05]--
+           +-09.0-[06]--
+           +-0a.0-[07]--
+           +-0b.0-[08]--
+           +-0c.0-[09-0a]----00.0-[0a]--+-00.0  Mellanox Tech MT42822 BlueField-2 ConnectX-6
+           |                             \-01.0  Mellanox Tech MT42822 BlueField-2 ConnectX-6
+           +-0d.0  Red Hat, Inc. Virtio socket
+           +-0e.0  Red Hat, Inc. Virtio file system
+           +-1f.0  Intel Corporation 82801IB (ICH9) LPC Interface Controller
+           +-1f.2  Intel Corporation 82801IR/IO/IH (ICH9R/DO/DH) 6 port SATA Controller
+           \-1f.3  Intel Corporation 82801I (ICH9 Family) SMBus Controller
+```
+
+The PCI devices will consume a slot of which we have 256 in the PCI(e) topology
+and leave scarce resources for the needed PCI Express devices.

docs/design/vcpu-handling-runtime-go.md

@@ -2,24 +2,15 @@
 
 ## Default number of virtual CPUs
 
-Before starting a container, the [runtime][6] reads the `default_vcpus` option
-from the [configuration file][7] to determine the number of virtual CPUs
+Before starting a container, the [runtime][4] reads the `default_vcpus` option
+from the [configuration file][5] to determine the number of virtual CPUs
 (vCPUs) needed to start the virtual machine. By default, `default_vcpus` is
 equal to 1 for fast boot time and a small memory footprint per virtual machine.
 Be aware that increasing this value negatively impacts the virtual machine's
 boot time and memory footprint.
 In general, we recommend that you do not edit this variable, unless you know
 what are you doing. If your container needs more than one vCPU, use
-[docker `--cpus`][1], [docker update][4], or [Kubernetes `cpu` limits][2] to
-assign more resources.
-
-*Docker*
-
-```sh
-$ docker run --name foo -ti --cpus 2 debian bash
-$ docker update --cpus 4 foo
-```
-
+[Kubernetes `cpu` limits][1] to assign more resources.
 
 *Kubernetes*
 
@@ -49,7 +40,7 @@ $ sudo -E kubectl create -f ~/cpu-demo.yaml
 ## Virtual CPUs and Kubernetes pods
 
 A Kubernetes pod is a group of one or more containers, with shared storage and
-network, and a specification for how to run the containers [[specification][3]].
+network, and a specification for how to run the containers [[specification][2]].
 In Kata Containers this group of containers, which is called a sandbox, runs inside
 the same virtual machine. If you do not specify a CPU constraint, the runtime does
 not add more vCPUs and the container is not placed inside a CPU cgroup.
@@ -73,13 +64,7 @@ constraints with each container trying to consume 100% of vCPU, the resources
 divide in two parts, 50% of vCPU for each container because your virtual
 machine does not have enough resources to satisfy containers needs. If you want
 to give access to a greater or lesser portion of vCPUs to a specific container,
-use [`docker --cpu-shares`][1] or [Kubernetes `cpu` requests][2].
-
-*Docker*
-
-```sh
-$ docker run -ti --cpus-shares=512 debian bash
-```
+use [Kubernetes `cpu` requests][1].
 
 *Kubernetes*
 
@@ -109,10 +94,9 @@ $ sudo -E kubectl create -f ~/cpu-demo.yaml
 Before running containers without CPU constraint, consider that your containers
 are not running alone. Since your containers run inside a virtual machine other
 processes use the vCPUs as well (e.g. `systemd` and the Kata Containers
-[agent][5]). In general, we recommend setting `default_vcpus` equal to 1 to
+[agent][3]). In general, we recommend setting `default_vcpus` equal to 1 to
 allow non-container processes to run on this vCPU and to specify a CPU
-constraint for each container. If your container is already running and needs
-more vCPUs, you can add more using [docker update][4].
+constraint for each container.
 
 ## Container with CPU constraint
 
@@ -121,7 +105,7 @@ constraints using the following formula: `vCPUs = ceiling( quota / period )`, wh
 `quota` specifies the number of microseconds per CPU Period that the container is
 guaranteed CPU access and `period` specifies the CPU CFS scheduler period of time
 in microseconds. The result determines the number of vCPU to hot plug into the
-virtual machine. Once the vCPUs have been added, the [agent][5] places the
+virtual machine. Once the vCPUs have been added, the [agent][3] places the
 container inside a CPU cgroup. This placement allows the container to use only
 its assigned resources.
 
@@ -138,25 +122,6 @@ the virtual machine starts with 8 vCPUs and 1 vCPUs is added and assigned
 to the container. Non-container processes might be able to use 8 vCPUs but they
 use a maximum 1 vCPU, hence 7 vCPUs might not be used.
 
-
-*Container without CPU constraint*
-
-```sh
-$ docker run -ti debian bash -c "nproc; cat /sys/fs/cgroup/cpu,cpuacct/cpu.cfs_*"
-1       # number of vCPUs
-100000  # cfs period
--1      # cfs quota
-```
-
-*Container with CPU constraint*
-
-```sh
-docker run --cpus 4 -ti debian bash -c "nproc; cat /sys/fs/cgroup/cpu,cpuacct/cpu.cfs_*"
-5       # number of vCPUs
-100000  # cfs period
-400000  # cfs quota
-```
-
 ## Virtual CPU handling without hotplug
 
 In some cases, the hardware and/or software architecture being utilized does not support
@@ -183,11 +148,8 @@ the container's `spec` will provide the sizing information directly. If these ar
 calculate the number of CPUs required for the workload and augment this by `default_vcpus`
 configuration option, and use this for the virtual machine size.
 
-
-[1]: https://docs.docker.com/config/containers/resource_constraints/#cpu
-[2]: https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource
-[3]: https://kubernetes.io/docs/concepts/workloads/pods/pod/
-[4]: https://docs.docker.com/engine/reference/commandline/update/
-[5]: ../../src/agent
-[6]: ../../src/runtime
-[7]: ../../src/runtime/README.md#configuration
+[1]: https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource
+[2]: https://kubernetes.io/docs/concepts/workloads/pods/pod/
+[3]: ../../src/agent
+[4]: ../../src/runtime
+[5]: ../../src/runtime/README.md#configuration

docs/design/vcpu-handling-runtime-rs.md

@@ -0,0 +1,51 @@
+# Virtual machine vCPU sizing in Kata Containers 3.0
+
+> Preview: 
+> [Kubernetes(since 1.23)][1] and [Containerd(since 1.6.0-beta4)][2] will help calculate `Sandbox Size` info and pass it to Kata Containers through annotations.
+> In order to adapt to this beneficial change and be compatible with the past, we have implemented the new vCPUs handling way in `runtime-rs`, which is slightly different from the original `runtime-go`'s design.
+
+## When do we need to handle vCPUs size?
+vCPUs sizing should be determined by the container workloads. So throughout the life cycle of Kata Containers, there are several points in time when we need to think about how many vCPUs should be at the time. Mainly including the time points of `CreateVM`, `CreateContainer`, `UpdateContainer`, and `DeleteContainer`.
+* `CreateVM`: When creating a sandbox, we need to know how many vCPUs to start the VM with.
+* `CreateContainer`: When creating a new container in the VM, we may need to hot-plug the vCPUs according to the requirements in container's spec.
+* `UpdateContainer`: When receiving the `UpdateContainer` request, we may need to update the vCPU resources according to the new requirements of the container.
+* `DeleteContainer`: When a container is removed from the VM, we may need to hot-unplug the vCPUs to reclaim the vCPU resources introduced by the container.
+
+## On what basis do we calculate the number of vCPUs?
+When Kata calculate the number of vCPUs, We have three data sources, the `default_vcpus` and `default_maxvcpus` specified in the configuration file (named `TomlConfig` later in the doc), the `io.kubernetes.cri.sandbox-cpu-quota` and `io.kubernetes.cri.sandbox-cpu-period` annotations passed by the upper layer runtime, and the corresponding CPU resource part in the container's spec for the container when `CreateContainer`/`UpdateContainer`/`DeleteContainer` is requested.
+
+Our understanding and priority of these resources are as follows, which will affect how we calculate the number of vCPUs later.
+
+* From `TomlConfig`:
+  * `default_vcpus`: default number of vCPUs when starting a VM.
+  * `default_maxvcpus`: maximum number of vCPUs.
+* From `Annotation`:
+  * `InitialSize`: we call the size of the resource passed from the annotations as `InitialSize`. Kubernetes will calculate the sandbox size according to the Pod's statement, which is the `InitialSize` here. This size should be the size we want to prioritize. 
+* From `Container Spec`:
+  * The amount of CPU resources that the Container wants to use will be declared through the spec. Including the aforementioned annotations, we mainly consider `cpu quota` and `cpuset` when calculating the number of vCPUs.
+  * `cpu quota`: `cpu quota` is the most common way to declare the amount of CPU resources. The number of vCPUs introduced by `cpu quota` declared in a container's spec is: `vCPUs = ceiling( quota / period )`.
+  * `cpuset`: `cpuset` is often used to bind the CPUs that tasks can run on. The number of vCPUs may introduced by `cpuset` declared in a container's spec is the number of CPUs specified in the set that do not overlap with other containers.
+
+
+## How to calculate and adjust the vCPUs size:
+There are two types of vCPUs that we need to consider, one is the number of vCPUs when starting the VM (named `Boot Size` in the doc). The second is the number of vCPUs when `CreateContainer`/`UpdateContainer`/`DeleteContainer` request is received (`Real-time Size` in the doc).
+
+### `Boot Size`
+The main considerations are `InitialSize` and `default_vcpus`. There are the following principles:
+`InitialSize` has priority over `default_vcpus` declared in `TomlConfig`.
+1. When there is such an annotation statement, the originally `default_vcpus` will be modified to the number of vCPUs in the `InitialSize` as the `Boot Size`. (Because not all runtimes support this annotation for the time being, we still keep the `default_cpus` in `TomlConfig`.)
+2. When the specs of all containers are aggregated for sandbox size calculation, the method is consistent with the calculation method of `InitialSize` here.
+
+### `Real-time Size`
+When we receive an OCI request, it may be for a single container. But what we have to consider is the number of vCPUs for the entire VM. So we will maintain a list. Every time there is a demand for adjustment, the entire list will be traversed to calculate a value for the number of vCPUs. In addition, there are the following principles:
+1. Do not cut computing power and try to keep the number of vCPUs specified by `InitialSize`.
+   * So the number of vCPUs after will not be less than the `Boot Size`.
+2. `cpu quota` takes precedence over `cpuset` and the setting history are took into account.
+   * We think quota describes the CPU time slice that a cgroup can use, and `cpuset` describes the actual CPU number that a cgroup can use. Quota can better describe the size of the CPU time slice that a cgroup actually wants to use. The `cpuset` only describes which CPUs the cgroup can use, but the cgroup can use the specified CPU but consumes a smaller time slice, so the quota takes precedence over the `cpuset`.
+   * On the one hand, when both `cpu quota` and `cpuset` are specified, we will calculate the number of vCPUs based on `cpu quota` and ignore `cpuset`. On the other hand, if `cpu quota` was used to control the number of vCPUs in the past, and only `cpuset` was updated during `UpdateContainer`, we will not adjust the number of vCPUs at this time.
+3. `StaticSandboxResourceMgmt` controls hotplug.
+   * Some VMMs and kernels of some architectures do not support hotplugging. We can accommodate this situation through `StaticSandboxResourceMgmt`. When `StaticSandboxResourceMgmt = true` is set, we don't make any further attempts to update the number of vCPUs after booting.
+
+
+[1]: https://github.com/kubernetes/kubernetes/pull/104886
+[2]: https://github.com/containerd/containerd/pull/6155

docs/design/vcpu-threads-pinning.md

@@ -0,0 +1,37 @@
+# Design Doc for Kata Containers' VCPUs Pinning Feature
+
+## Background
+By now, vCPU threads of Kata Containers are scheduled randomly to CPUs. And each pod would request a specific set of CPUs which we call it CPU set (just the CPU set meaning in Linux cgroups).    
+
+If the number of vCPU threads are equal to that of CPUs claimed in CPU set, we can then pin each vCPU thread to one specified CPU, to reduce the cost of random scheduling. 
+
+## Detailed Design
+
+### Passing Config Parameters
+Two ways are provided to use this vCPU thread pinning feature: through `QEMU` configuration file and through annotations. Finally the pinning parameter is passed to `HypervisorConfig`.
+
+### Related Linux Thread Scheduling API
+
+| API Info          | Value                                                     |
+|-------------------|-----------------------------------------------------------|
+| Package           | `golang.org/x/sys/unix`                                     |
+| Method            | `unix.SchedSetaffinity(thread_id, &unixCPUSet)`             |
+| Official Doc Page | https://pkg.go.dev/golang.org/x/sys/unix#SchedSetaffinity |
+
+### When is VCPUs Pinning Checked?
+
+As shown in Section 1, when `num(vCPU threads) == num(CPUs in CPU set)`, we shall pin each vCPU thread to a specified CPU. And when this condition is broken, we should restore to the original random scheduling pattern.  
+So when may `num(CPUs in CPU set)` change? There are 5 possible scenes:
+
+| Possible scenes                   | Related Code                               |
+|-----------------------------------|--------------------------------------------|
+| when creating a container         | File Sandbox.go, in method `CreateContainer`  |
+| when starting a container         | File Sandbox.go, in method `StartContainer`   |
+| when deleting a container         | File Sandbox.go, in method `DeleteContainer`  |
+| when updating a container         | File Sandbox.go, in method `UpdateContainer`  |
+| when creating multiple containers | File Sandbox.go, in method `createContainers` |
+
+### Core Pinning Logics
+
+We can split the whole process into the following steps. Related methods are `checkVCPUsPinning` and `resetVCPUsPinning`, in file Sandbox.go.
+![](arch-images/vcpus-pinning-process.png) 

docs/design/virtualization.md

@@ -39,7 +39,7 @@ Details of each solution and a summary are provided below.
 Kata Containers with QEMU has complete compatibility with Kubernetes.
 
 Depending on the host architecture, Kata Containers supports various machine types,
-for example `pc` and `q35` on x86 systems, `virt` on ARM systems and `pseries` on IBM Power systems. The default Kata Containers
+for example `q35` on x86 systems, `virt` on ARM systems and `pseries` on IBM Power systems. The default Kata Containers
 machine type is `q35`. The machine type and its [`Machine accelerators`](#machine-accelerators) can
 be changed by editing the runtime [`configuration`](architecture/README.md#configuration) file.
 
@@ -60,9 +60,8 @@ Machine accelerators are architecture specific and can be used to improve the pe
 and enable specific features of the machine types. The following machine accelerators
 are used in Kata Containers:
 
-- NVDIMM: This machine accelerator is x86 specific and only supported by `pc` and
-`q35` machine types. `nvdimm` is used to provide the root filesystem as a persistent
-memory device to the Virtual Machine.
+- NVDIMM: This machine accelerator is x86 specific and only supported by `q35` machine types.
+`nvdimm` is used to provide the root filesystem as a persistent memory device to the Virtual Machine.
 
 #### Hotplug devices
 
@@ -111,7 +110,7 @@ Devices and features used:
 - VFIO
 - hotplug
 - seccomp filters
-- [HTTP OpenAPI](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/vmm/src/api/openapi/cloud-hypervisor.yaml)
+- [HTTP OpenAPI](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/main/vmm/src/api/openapi/cloud-hypervisor.yaml)
 
 ### Summary
 

docs/how-to/README.md

@@ -5,7 +5,7 @@
 - [Run Kata containers with `crictl`](run-kata-with-crictl.md)
 - [Run Kata Containers with Kubernetes](run-kata-with-k8s.md)
 - [How to use Kata Containers and Containerd](containerd-kata.md)
-- [How to use Kata Containers and CRI (containerd) with Kubernetes](how-to-use-k8s-with-cri-containerd-and-kata.md)
+- [How to use Kata Containers and containerd with Kubernetes](how-to-use-k8s-with-containerd-and-kata.md)
 - [Kata Containers and service mesh for Kubernetes](service-mesh.md)
 - [How to import Kata Containers logs into Fluentd](how-to-import-kata-logs-with-fluentd.md)
 
@@ -15,6 +15,11 @@
 - `qemu`
 - `cloud-hypervisor`
 - `firecracker`
+
+   In the case of `firecracker` the use of a block device `snapshotter` is needed
+   for the VM rootfs. Refer to the following guide for additional configuration
+   steps:
+   - [Setup Kata containers with `firecracker`](how-to-use-kata-containers-with-firecracker.md)
 - `ACRN`
 
   While `qemu` , `cloud-hypervisor` and `firecracker` work out of the box with installation of Kata,
@@ -37,4 +42,7 @@
 - [How to setup swap devices in guest kernel](how-to-setup-swap-devices-in-guest-kernel.md)
 - [How to run rootless vmm](how-to-run-rootless-vmm.md)
 - [How to run Docker with Kata Containers](how-to-run-docker-with-kata.md)
-- [How to run Kata Containers with `nydus`](how-to-use-virtio-fs-nydus-with-kata.md)
+- [How to run Kata Containers with `nydus`](how-to-use-virtio-fs-nydus-with-kata.md)
+- [How to run Kata Containers with AMD SEV-SNP](how-to-run-kata-containers-with-SNP-VMs.md)
+- [How to use EROFS to build rootfs in Kata Containers](how-to-use-erofs-build-rootfs.md)
+- [How to run Kata Containers with kinds of Block Volumes](how-to-run-kata-containers-with-kinds-of-Block-Volumes.md)

docs/how-to/containerd-kata.md

@@ -40,7 +40,7 @@ use `RuntimeClass` instead of the deprecated annotations.
 ### Containerd Runtime V2 API: Shim V2 API
 
 The [`containerd-shim-kata-v2` (short as `shimv2` in this documentation)](../../src/runtime/cmd/containerd-shim-kata-v2/)
-implements the [Containerd Runtime V2 (Shim API)](https://github.com/containerd/containerd/tree/master/runtime/v2) for Kata.
+implements the [Containerd Runtime V2 (Shim API)](https://github.com/containerd/containerd/tree/main/runtime/v2) for Kata.
 With `shimv2`, Kubernetes can launch Pod and OCI-compatible containers with one shim per Pod. Prior to `shimv2`, `2N+1` 
 shims (i.e. a `containerd-shim` and a `kata-shim` for each container and the Pod sandbox itself) and no standalone `kata-proxy` 
 process were used, even with VSOCK not available.
@@ -72,14 +72,13 @@ $ command -v containerd
 
 ### Install CNI plugins
 
-> **Note:** You do not need to install CNI plugins if you do not want to use containerd with Kubernetes.
 > If you have installed Kubernetes with `kubeadm`, you might have already installed the CNI plugins.
 
 You can manually install CNI plugins as follows:
 
 ```bash
-$ go get github.com/containernetworking/plugins
-$ pushd $GOPATH/src/github.com/containernetworking/plugins
+$ git clone https://github.com/containernetworking/plugins.git
+$ pushd plugins
 $ ./build_linux.sh
 $ sudo mkdir /opt/cni
 $ sudo cp -r bin /opt/cni/
@@ -94,8 +93,8 @@ $ popd
 You can install the `cri-tools` from source code:
 
 ```bash
-$ go get github.com/kubernetes-incubator/cri-tools
-$ pushd $GOPATH/src/github.com/kubernetes-incubator/cri-tools
+$ git clone https://github.com/kubernetes-sigs/cri-tools.git
+$ pushd cri-tools
 $ make
 $ sudo -E make install
 $ popd
@@ -131,74 +130,42 @@ For
 
 The `RuntimeClass` is suggested.
 
-The following configuration includes three runtime classes:
+The following configuration includes two runtime classes:
 - `plugins.cri.containerd.runtimes.runc`: the runc, and it is the default runtime.
-- `plugins.cri.containerd.runtimes.kata`: The function in containerd (reference [the document here](https://github.com/containerd/containerd/tree/master/runtime/v2#binary-naming)) 
+- `plugins.cri.containerd.runtimes.kata`: The function in containerd (reference [the document here](https://github.com/containerd/containerd/tree/main/runtime/v2#binary-naming)) 
   where the dot-connected string `io.containerd.kata.v2` is translated to `containerd-shim-kata-v2` (i.e. the 
-  binary name of the Kata implementation of [Containerd Runtime V2 (Shim API)](https://github.com/containerd/containerd/tree/master/runtime/v2)).
-- `plugins.cri.containerd.runtimes.katacli`: the `containerd-shim-runc-v1` calls `kata-runtime`, which is the legacy process.
+  binary name of the Kata implementation of [Containerd Runtime V2 (Shim API)](https://github.com/containerd/containerd/tree/main/runtime/v2)).
 
 ```toml
     [plugins.cri.containerd]
       no_pivot = false
     [plugins.cri.containerd.runtimes]
-      [plugins.cri.containerd.runtimes.runc]
-         runtime_type = "io.containerd.runc.v1"
-         [plugins.cri.containerd.runtimes.runc.options]
-           NoPivotRoot = false
-           NoNewKeyring = false
-           ShimCgroup = ""
-           IoUid = 0
-           IoGid = 0
-           BinaryName = "runc"
-           Root = ""
-           CriuPath = ""
-           SystemdCgroup = false
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+         privileged_without_host_devices = false
+         runtime_type = "io.containerd.runc.v2"
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+            BinaryName = ""
+            CriuImagePath = ""
+            CriuPath = ""
+            CriuWorkPath = ""
+            IoGid = 0
       [plugins.cri.containerd.runtimes.kata]
          runtime_type = "io.containerd.kata.v2"
-      [plugins.cri.containerd.runtimes.katacli]
-         runtime_type = "io.containerd.runc.v1"
-         [plugins.cri.containerd.runtimes.katacli.options]
-           NoPivotRoot = false
-           NoNewKeyring = false
-           ShimCgroup = ""
-           IoUid = 0
-           IoGid = 0
-           BinaryName = "/usr/bin/kata-runtime"
-           Root = ""
-           CriuPath = ""
-           SystemdCgroup = false
-```
-
-From Containerd v1.2.4 and Kata v1.6.0, there is a new runtime option supported, which allows you to specify a specific Kata configuration file as follows:
-
-```toml
-      [plugins.cri.containerd.runtimes.kata]
-         runtime_type = "io.containerd.kata.v2"
-	 privileged_without_host_devices = true
-	 [plugins.cri.containerd.runtimes.kata.options]
-	   ConfigPath = "/etc/kata-containers/config.toml"
+         privileged_without_host_devices = true
+         pod_annotations = ["io.katacontainers.*"]
+         container_annotations = ["io.katacontainers.*"]
+         [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options]
+            ConfigPath = "/opt/kata/share/defaults/kata-containers/configuration.toml"
 ```
 
 `privileged_without_host_devices` tells containerd that a privileged Kata container should not have direct access to all host devices. If unset, containerd will pass all host devices to Kata container, which may cause security issues.
 
+`pod_annotations` is the list of pod annotations passed to both the pod sandbox as well as container through the OCI config.
+
+`container_annotations` is the list of container annotations passed through to the OCI config of the containers.
+
 This `ConfigPath` option is optional. If you do not specify it, shimv2 first tries to get the configuration file from the environment variable `KATA_CONF_FILE`. If neither are set, shimv2 will use the default Kata configuration file paths (`/etc/kata-containers/configuration.toml` and `/usr/share/defaults/kata-containers/configuration.toml`).
 
-If you use Containerd older than v1.2.4 or a version of Kata older than v1.6.0  and also want to specify a configuration file, you can use the following workaround, since the shimv2 accepts an environment variable, `KATA_CONF_FILE` for the configuration file path. Then, you can create a
-shell script with the following:
-
-```bash
-#!/usr/bin/env bash
-KATA_CONF_FILE=/etc/kata-containers/firecracker.toml containerd-shim-kata-v2 $@
-```
-
-Name it as `/usr/local/bin/containerd-shim-katafc-v2` and reference it in the configuration of containerd:
-
-```toml
-      [plugins.cri.containerd.runtimes.kata-firecracker]
-         runtime_type = "io.containerd.katafc.v2"
-```
-
 #### Kata Containers as the runtime for untrusted workload
 
 For cases without `RuntimeClass` support, we can use the legacy annotation method to support using Kata Containers 
@@ -218,28 +185,8 @@ and then, run an untrusted workload with Kata Containers:
       runtime_type = "io.containerd.kata.v2"
 ```
 
-For the earlier versions of Kata Containers and containerd that do not support Runtime V2 (Shim API), you can use the following alternative configuration:
-
-```toml
-    [plugins.cri.containerd]
-  
-    # "plugins.cri.containerd.default_runtime" is the runtime to use in containerd.
-    [plugins.cri.containerd.default_runtime]
-      # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
-      runtime_type = "io.containerd.runtime.v1.linux"
-
-    # "plugins.cri.containerd.untrusted_workload_runtime" is a runtime to run untrusted workloads on it.
-    [plugins.cri.containerd.untrusted_workload_runtime]
-      # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
-      runtime_type = "io.containerd.runtime.v1.linux"
-
-      # runtime_engine is the name of the runtime engine used by containerd.
-      runtime_engine = "/usr/bin/kata-runtime"
-```
-
 You can find more information on the [Containerd config documentation](https://github.com/containerd/cri/blob/master/docs/config.md)
 
-
 #### Kata Containers as the default runtime
 
 If you want to set Kata Containers as the only runtime in the deployment, you can simply configure as follows:
@@ -250,15 +197,6 @@ If you want to set Kata Containers as the only runtime in the deployment, you ca
       runtime_type = "io.containerd.kata.v2"
 ```
 
-Alternatively, for the earlier versions of Kata Containers and containerd that do not support Runtime V2 (Shim API), you can use the following alternative configuration:
-
-```toml
-    [plugins.cri.containerd]
-    [plugins.cri.containerd.default_runtime]
-      runtime_type = "io.containerd.runtime.v1.linux"
-      runtime_engine = "/usr/bin/kata-runtime"
-```
-
 ### Configuration for `cri-tools`
 
 > **Note:** If you skipped the [Install `cri-tools`](#install-cri-tools) section, you can skip this section too.
@@ -312,11 +250,55 @@ To run a container with Kata Containers through the containerd command line, you
 
 ```bash
 $ sudo ctr image pull docker.io/library/busybox:latest
-$ sudo ctr run --runtime io.containerd.run.kata.v2 -t --rm docker.io/library/busybox:latest hello sh
+$ sudo ctr run --cni --runtime io.containerd.run.kata.v2 -t --rm docker.io/library/busybox:latest hello sh
 ```
 
 This launches a BusyBox container named `hello`, and it will be removed by `--rm` after it quits.
+The `--cni` flag enables CNI networking for the container. Without this flag, a container with just a
+loopback interface is created.
 
+### Launch containers using `ctr` command line with rootfs bundle
+
+#### Get rootfs
+Use the script to create rootfs
+```bash
+ctr i pull quay.io/prometheus/busybox:latest
+ctr i export rootfs.tar quay.io/prometheus/busybox:latest
+
+rootfs_tar=rootfs.tar
+bundle_dir="./bundle"
+mkdir -p "${bundle_dir}"
+
+# extract busybox rootfs
+rootfs_dir="${bundle_dir}/rootfs"
+mkdir -p "${rootfs_dir}"
+layers_dir="$(mktemp -d)"
+tar -C "${layers_dir}" -pxf "${rootfs_tar}"
+for ((i=0;i<$(cat ${layers_dir}/manifest.json | jq -r ".[].Layers | length");i++)); do
+  tar -C ${rootfs_dir} -xf ${layers_dir}/$(cat ${layers_dir}/manifest.json | jq -r ".[].Layers[${i}]")
+done
+```
+#### Get `config.json`
+Use runc spec to generate `config.json`
+```bash
+cd ./bundle/rootfs
+runc spec
+mv config.json ../
+```
+Change the root `path` in `config.json` to the absolute path of rootfs
+
+```JSON
+"root":{
+    "path":"/root/test/bundle/rootfs",
+    "readonly": false
+},
+```
+
+#### Run container
+```bash
+sudo ctr run -d --runtime io.containerd.run.kata.v2 --config bundle/config.json hello
+sudo ctr t exec --exec-id ${ID} -t hello sh
+```
 ### Launch Pods with `crictl` command line
 
 With the `crictl` command line of `cri-tools`, you can specify runtime class with `-r` or `--runtime` flag.

docs/how-to/data/kata-monitor-daemonset.yml

@@ -45,6 +45,9 @@ spec:
         - name: containerdsocket
           mountPath: /run/containerd/containerd.sock
           readOnly: true
+        - name: sbs
+          mountPath: /run/vc/sbs/
+          readOnly: true
       terminationGracePeriodSeconds: 30
       volumes:
       - name: containerdtask
@@ -53,3 +56,6 @@ spec:
       - name: containerdsocket
         hostPath:
           path: /run/containerd/containerd.sock
+      - name: sbs
+        hostPath:
+          path: /run/vc/sbs/

docs/how-to/how-to-hotplug-memory-arm64.md

@@ -15,6 +15,18 @@ $ sudo .ci/aarch64/install_rom_aarch64.sh
 $ popd
 ```
 
+## Config KATA QEMU
+
+After executing the above script, two files will be generated under the directory `/usr/share/kata-containers/` by default, namely `kata-flash0.img` and `kata-flash1.img`. Next we need to change the configuration file of `kata qemu`, which is in `/opt/kata/share/defaults/kata-containers/configuration-qemu.toml` by default, specify in the configuration file to use the UEFI ROM installed above. The above is an example of `kata deploy` installation. For package management installation, please use `kata-runtime env` to find the location of the configuration file. Please refer to the following configuration.
+
+```
+[hypervisor.qemu]
+
+# -pflash can add image file to VM. The arguments of it should be in format
+# of ["/path/to/flash0.img", "/path/to/flash1.img"]
+pflashes = ["/usr/share/kata-containers/kata-flash0.img", "/usr/share/kata-containers/kata-flash1.img"]
+```
+
 ## Run for test
 
 Let's test if the memory hotplug is ready for Kata after install the UEFI ROM. Make sure containerd is ready to run Kata before test.

docs/how-to/how-to-import-kata-logs-with-fluentd.md

@@ -68,7 +68,7 @@ the Kata logs import to the EFK stack.
 > stack they are able to utilise in order to modify and test as necessary.
 
 Minikube by default
-[configures](https://github.com/kubernetes/minikube/blob/master/deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/etc/systemd/journald.conf)
+[configures](https://github.com/kubernetes/minikube/blob/master/deploy/iso/minikube-iso/board/minikube/x86_64/rootfs-overlay/etc/systemd/journald.conf)
  the `systemd-journald` with the
 [`Storage=volatile`](https://www.freedesktop.org/software/systemd/man/journald.conf.html) option,
 which results in the journal being stored in `/run/log/journal`. Unfortunately, the Minikube EFK
@@ -163,7 +163,7 @@ sub-filter on, for instance, the `SYSLOG_IDENTIFIER` to differentiate the Kata c
 on the `PRIORITY` to filter out critical issues etc.
 
 Kata generates a significant amount of Kata specific information, which can be seen as
-[`logfmt`](https://github.com/kata-containers/tests/tree/main/cmd/log-parser#logfile-requirements).
+[`logfmt`](../../src/tools/log-parser/README.md#logfile-requirements).
 data contained in the `MESSAGE` field. Imported as-is, there is no easy way to filter on that data
 in Kibana:
 

docs/how-to/how-to-run-docker-with-kata.md

@@ -48,9 +48,9 @@ Running Docker containers Kata Containers requires care because `VOLUME`s specif
 kataShared on / type virtiofs (rw,relatime,dax)
 ```
 
-`kataShared` mount types are powered by [`virtio-fs`][virtio-fs], a marked improvement over `virtio-9p`, thanks to [PR #1016](https://github.com/kata-containers/runtime/pull/1016). While `virtio-fs` is normally an excellent choice, in the case of DinD workloads `virtio-fs` causes an issue -- [it *cannot* be used as a "upper layer" of `overlayfs` without a custom patch](http://lists.katacontainers.io/pipermail/kata-dev/2020-January/001216.html).
+`kataShared` mount types are powered by [`virtio-fs`](https://virtio-fs.gitlab.io/), a marked improvement over `virtio-9p`, thanks to [PR #1016](https://github.com/kata-containers/runtime/pull/1016). While `virtio-fs` is normally an excellent choice, in the case of DinD workloads `virtio-fs` causes an issue -- [it *cannot* be used as a "upper layer" of `overlayfs` without a custom patch](http://lists.katacontainers.io/pipermail/kata-dev/2020-January/001216.html).
 
-As `/var/lib/docker` is a `VOLUME` specified by DinD (i.e. the `docker` images tagged `*-dind`/`*-dind-rootless`), `docker` fill fail to start (or even worse, silently pick a worse storage driver like `vfs`) when started in a Kata Container. Special measures must be taken when running DinD-powered workloads in Kata Containers.
+As `/var/lib/docker` is a `VOLUME` specified by DinD (i.e. the `docker` images tagged `*-dind`/`*-dind-rootless`), `docker` will fail to start (or even worse, silently pick a worse storage driver like `vfs`) when started in a Kata Container. Special measures must be taken when running DinD-powered workloads in Kata Containers.
 
 ## Workarounds/Solutions
 
@@ -58,7 +58,7 @@ Thanks to various community contributions (see [issue references below](#referen
 
 ### Use a memory backed volume
 
-For small workloads (small container images, without much generated filesystem load), a memory-backed volume is sufficient. Kubernetes supports a variant of  [the `EmptyDir` volume][k8s-emptydir], which allows for memdisk-backed storage -- the [the `medium: Memory` ][k8s-memory-volume-type]. An example of a `Pod` using such a setup [was contributed](https://github.com/kata-containers/runtime/issues/1429#issuecomment-477385283), and is reproduced below:
+For small workloads (small container images, without much generated filesystem load), a memory-backed volume is sufficient. Kubernetes supports a variant of  [the `EmptyDir` volume](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir), which allows for memdisk-backed storage -- the the `medium: Memory`. An example of a `Pod` using such a setup [was contributed](https://github.com/kata-containers/runtime/issues/1429#issuecomment-477385283), and is reproduced below:
 
 ```yaml
 apiVersion: v1

docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md

@@ -0,0 +1,158 @@
+# Kata Containers with AMD SEV-SNP VMs
+
+## Disclaimer
+
+This guide is designed for developers and is - same as the Developer Guide - not intended for production systems or end users. It is advisable to only follow this guide on non-critical development systems.
+
+## Prerequisites
+
+To run  Kata Containers in SNP-VMs, the following software stack is used.
+
+![Kubernetes integration with shimv2](./images/SNP-stack.svg)
+
+The host BIOS and kernel must be capable of supporting AMD SEV-SNP and configured accordingly. For Kata Containers, the host kernel with branch [`sev-snp-iommu-avic_5.19-rc6_v3`](https://github.com/AMDESE/linux/tree/sev-snp-iommu-avic_5.19-rc6_v3) and commit [`3a88547`](https://github.com/AMDESE/linux/commit/3a885471cf89156ea555341f3b737ad2a8d9d3d0) is known to work in conjunction with SEV Firmware version 1.51.3 (0xh\_1.33.03) available on AMD's [SEV developer website](https://developer.amd.com/sev/). See [AMD's guide](https://github.com/AMDESE/AMDSEV/tree/sev-snp-devel) to configure the host accordingly. Verify that you are able to run SEV-SNP encrypted VMs first. The guest components required for Kata Containers are built as described below.
+
+**Tip**: It is easiest to first have Kata Containers running on your system and then modify it to run containers in SNP-VMs. Follow the [Developer guide](../Developer-Guide.md#warning) and then follow the below steps. Nonetheless, you can just follow this guide from the start.
+
+## How to build
+
+Follow all of the below steps to install Kata Containers with SNP-support from scratch. These steps mostly follow the developer guide with modifications to support SNP
+
+__Steps from the Developer Guide:__
+- Get all the [required components](../Developer-Guide.md#requirements-to-build-individual-components) for building the kata-runtime
+- [Build the and install kata-runtime](../Developer-Guide.md#build-and-install-the-kata-containers-runtime)
+- [Build a custom agent](../Developer-Guide.md#build-a-custom-kata-agent---optional)
+- [Create an initrd image](../Developer-Guide.md#create-an-initrd-image---optional) by first building a rootfs, then building the initrd based on the rootfs, use a custom agent and install. `ubuntu` works as the distribution of choice.
+- Get the [required components](../../tools/packaging/kernel/README.md#requirements) to build a custom kernel
+
+__SNP-specific steps:__
+- Build the SNP-specific kernel as shown below (see this [guide](../../tools/packaging/kernel/README.md#build-kata-containers-kernel) for more information)
+```bash
+$ pushd kata-containers/tools/packaging/kernel/
+$ ./build-kernel.sh -a x86_64 -x snp setup
+$ ./build-kernel.sh -a x86_64 -x snp build
+$ sudo -E PATH="${PATH}" ./build-kernel.sh -x snp install
+$ popd
+```
+- Build a current OVMF capable of SEV-SNP:
+```bash
+$ pushd kata-containers/tools/packaging/static-build/ovmf
+$ ./build.sh
+$ tar -xvf edk2-x86_64.tar.gz
+$ popd
+```
+- Build a custom QEMU
+```bash
+$ source kata-containers/tools/packaging/scripts/lib.sh
+$ qemu_url="$(get_from_kata_deps "assets.hypervisor.qemu-snp-experimental.url")"
+$ qemu_tag="$(get_from_kata_deps "assets.hypervisor.qemu-snp-experimental.tag")"
+$ git clone "${qemu_url}"
+$ pushd qemu
+$ git checkout "${qemu_tag}"
+$ ./configure --enable-virtfs --target-list=x86_64-softmmu --enable-debug
+$ make -j "$(nproc)"
+$ popd
+```
+
+### Kata Containers Configuration for SNP
+
+The configuration file located at `/etc/kata-containers/configuration.toml` must be adapted as follows to support SNP-VMs:
+- Use the SNP-specific kernel for the guest VM (change path)
+```toml
+kernel = "/usr/share/kata-containers/vmlinuz-snp.container"
+```
+- Enable the use of an initrd (uncomment)
+```toml
+initrd = "/usr/share/kata-containers/kata-containers-initrd.img"
+```
+- Disable the use of a rootfs (comment out)
+```toml
+# image = "/usr/share/kata-containers/kata-containers.img"
+```
+- Use the custom QEMU capable of SNP (change path)
+```toml
+path = "/path/to/qemu/build/qemu-system-x86_64"
+```
+- Use `virtio-9p` device since `virtio-fs` is unsupported due to bugs / shortcomings in QEMU version [`snp-v3`](https://github.com/AMDESE/qemu/tree/snp-v3) for SEV and SEV-SNP (change value)
+```toml
+shared_fs = "virtio-9p"
+```
+- Disable `virtiofsd` since it is no longer required (comment out)
+```toml
+# virtio_fs_daemon = "/usr/libexec/virtiofsd"
+```
+- Disable NVDIMM (uncomment)
+```toml
+disable_image_nvdimm = true
+```
+- Disable shared memory (uncomment)
+```toml
+file_mem_backend = ""
+```
+- Enable confidential guests (uncomment)
+```toml
+confidential_guest = true
+```
+- Enable SNP-VMs (uncomment)
+```toml
+sev_snp_guest = true
+```
+  - Configure an OVMF (add path)
+```toml
+firmware = "/path/to/kata-containers/tools/packaging/static-build/ovmf/opt/kata/share/ovmf/OVMF.fd"
+```
+
+## Test Kata Containers with Containerd
+
+With Kata Containers configured to support SNP-VMs, we use containerd to test and deploy containers in these VMs.
+
+### Install Containerd
+If not already present, follow [this guide](./containerd-kata.md#install) to install containerd and its related components including `CNI` and the `cri-tools` (skip Kata Containers since we already installed it)
+
+### Containerd Configuration
+
+Follow [this guide](./containerd-kata.md#configuration) to configure containerd to use Kata Containers
+
+## Run Kata Containers in SNP-VMs
+
+Run the below commands to start a container. See [this guide](./containerd-kata.md#run) for more information
+```bash
+$ sudo ctr image pull docker.io/library/busybox:latest
+$ sudo ctr run --cni --runtime io.containerd.run.kata.v2 -t --rm docker.io/library/busybox:latest hello sh
+```
+
+### Check for active SNP:
+
+Inside the running container, run the following commands to check if SNP is active. It should look something like this:
+```
+/ # dmesg | grep -i sev
+[    0.299242] Memory Encryption Features active: AMD SEV SEV-ES SEV-SNP
+[    0.472286] SEV: Using SNP CPUID table, 31 entries present.
+[    0.514574] SEV: SNP guest platform device initialized.
+[    0.885425] sev-guest sev-guest: Initialized SEV guest driver (using vmpck_id 0)
+```
+
+### Obtain an SNP Attestation Report
+
+To obtain an attestation report inside the container, the `/dev/sev-guest` must first be configured. As of now, the VM does not perform this step, however it can be performed inside the container, either in the terminal or in code.
+
+Example for shell:
+```
+/ # SNP_MAJOR=$(cat /sys/devices/virtual/misc/sev-guest/dev | awk -F: '{print $1}')
+/ # SNP_MINOR=$(cat /sys/devices/virtual/misc/sev-guest/dev | awk -F: '{print $2}')
+/ # mknod -m 600 /dev/sev-guest c "${SNP_MAJOR}" "${SNP_MINOR}"
+```
+
+## Known Issues
+
+- Support for cgroups v2 is still [work in progress](https://github.com/kata-containers/kata-containers/issues/927). If issues occur due to cgroups v2 becoming the default in newer systems, one possible solution is to downgrade cgroups to v1:
+```bash
+sudo sed -i 's/^\(GRUB_CMDLINE_LINUX=".*\)"/\1 systemd.unified_cgroup_hierarchy=0"/' /etc/default/grub
+sudo update-grub
+sudo reboot
+```
+- If both SEV and SEV-SNP are supported by the host, Kata Containers uses SEV-SNP by default. You can verify what features are enabled by checking `/sys/module/kvm_amd/parameters/sev` and `sev_snp`. This means that Kata Containers can not run both SEV-SNP-VMs and SEV-VMs at the same time. If SEV is to be used by Kata Containers instead, reload the `kvm_amd` kernel module without SNP-support, this will disable SNP-support for the entire platform.
+```bash
+sudo rmmod kvm_amd && sudo modprobe kvm_amd sev_snp=0
+```
+

docs/how-to/how-to-run-kata-containers-with-kinds-of-Block-Volumes.md

@@ -0,0 +1,226 @@
+# A new way for Kata Containers to use Kinds of Block Volumes
+
+> **Note:** This guide is only available for runtime-rs with default Hypervisor Dragonball.
+> Now, other hypervisors are still ongoing, and it'll be updated when they're ready.
+
+
+## Background
+
+Currently, there is no widely applicable and convenient method available for users to use some kinds of backend storages, such as File on host based block volume, SPDK based volume or VFIO device based volume for Kata Containers, so we adopt [Proposal: Direct Block Device Assignment](https://github.com/kata-containers/kata-containers/blob/main/docs/design/direct-blk-device-assignment.md) to address it.
+
+## Solution
+
+According to the proposal, it requires to use the `kata-ctl direct-volume` command to add a direct assigned block volume device to the Kata Containers runtime. 
+
+And then with the help of method [get_volume_mount_info](https://github.com/kata-containers/kata-containers/blob/099b4b0d0e3db31b9054e7240715f0d7f51f9a1c/src/libs/kata-types/src/mount.rs#L95), get information from JSON file: `(mountinfo.json)` and parse them into structure [Direct Volume Info](https://github.com/kata-containers/kata-containers/blob/099b4b0d0e3db31b9054e7240715f0d7f51f9a1c/src/libs/kata-types/src/mount.rs#L70) which is used to save device-related information. 
+
+We only fill the `mountinfo.json`, such as `device` ,`volume_type`, `fs_type`, `metadata` and `options`, which correspond to the fields in [Direct Volume Info](https://github.com/kata-containers/kata-containers/blob/099b4b0d0e3db31b9054e7240715f0d7f51f9a1c/src/libs/kata-types/src/mount.rs#L70), to describe a device. 
+
+The JSON file `mountinfo.json` placed in a sub-path `/kubelet/kata-test-vol-001/volume001` which under fixed path `/run/kata-containers/shared/direct-volumes/`. 
+And the full path looks like: `/run/kata-containers/shared/direct-volumes/kubelet/kata-test-vol-001/volume001`, But for some security reasons. it is 
+encoded as `/run/kata-containers/shared/direct-volumes/L2t1YmVsZXQva2F0YS10ZXN0LXZvbC0wMDEvdm9sdW1lMDAx`.
+
+Finally, when running a Kata Containers with `ctr run --mount type=X, src=Y, dst=Z,,options=rbind:rw`, the `type=X` should be specified a proprietary type specifically designed for some kind of volume. 
+
+Now, supported types: 
+
+- `directvol` for direct volume
+- `vfiovol` for VFIO device based volume
+- `spdkvol` for SPDK/vhost-user based volume
+
+
+## Setup Device and Run a Kata-Containers
+
+### Direct Block Device Based Volume
+
+#### create raw block based backend storage
+
+> **Tips:** raw block based backend storage MUST be formatted with `mkfs`.
+
+```bash
+$ sudo dd if=/dev/zero of=/tmp/stor/rawdisk01.20g bs=1M count=20480
+$ sudo mkfs.ext4 /tmp/stor/rawdisk01.20g
+```
+
+#### setup direct block device for kata-containers
+
+```json
+{
+  "device": "/tmp/stor/rawdisk01.20g", 
+  "volume_type": "directvol", 
+  "fs_type": "ext4", 
+  "metadata":"{}", 
+  "options": []
+}
+```
+
+```bash
+$ sudo kata-ctl direct-volume add /kubelet/kata-direct-vol-002/directvol002 "{\"device\": \"/tmp/stor/rawdisk01.20g\", \"volume_type\": \"directvol\", \"fs_type\": \"ext4\", \"metadata\":"{}", \"options\": []}"
+$# /kubelet/kata-direct-vol-002/directvol002 <==> /run/kata-containers/shared/direct-volumes/W1lMa2F0ZXQva2F0YS10a2F0DAxvbC0wMDEvdm9sdW1lMDAx
+$ cat W1lMa2F0ZXQva2F0YS10a2F0DAxvbC0wMDEvdm9sdW1lMDAx/mountInfo.json 
+{"volume_type":"directvol","device":"/tmp/stor/rawdisk01.20g","fs_type":"ext4","metadata":{},"options":[]}
+```
+
+#### Run a Kata container with direct block device volume
+
+```bash
+$ # type=disrectvol,src=/kubelet/kata-direct-vol-002/directvol002,dst=/disk002,options=rbind:rw
+$ sudo ctr run -t --rm --runtime io.containerd.kata.v2 --mount type=directvol,src=/kubelet/kata-direct-vol-002/directvol002,dst=/disk002,options=rbind:rw "$image" kata-direct-vol-xx05302045 /bin/bash
+```
+
+
+### VFIO Device Based Block Volume
+
+#### create VFIO device based backend storage
+
+> **Tip:** It only supports `vfio-pci` based PCI device passthrough mode.
+
+In this scenario, the device's host kernel driver will be replaced by `vfio-pci`, and IOMMU group ID generated.
+And either device's BDF or its VFIO IOMMU group ID in `/dev/vfio/` is fine for "device" in `mountinfo.json`.
+
+```bash
+$ lspci -nn -k -s 45:00.1 
+45:00.1 SCSI storage controller 
+...
+Kernel driver in use: vfio-pci
+...
+$ ls /dev/vfio/110
+/dev/vfio/110
+$ ls /sys/kernel/iommu_groups/110/devices/
+0000:45:00.1
+```
+
+#### setup VFIO device for kata-containers
+
+First, configure the `mountinfo.json`, as below:
+
+- (1) device with `BB:DD:F`
+
+```json
+{
+  "device": "45:00.1",
+  "volume_type": "vfiovol", 
+  "fs_type": "ext4", 
+  "metadata":"{}", 
+  "options": []
+}
+```
+
+- (2) device with `DDDD:BB:DD:F`
+
+```json
+{
+  "device": "0000:45:00.1",
+  "volume_type": "vfiovol", 
+  "fs_type": "ext4", 
+  "metadata":"{}", 
+  "options": []
+}
+```
+
+- (3) device with `/dev/vfio/X`
+
+```json
+{
+  "device": "/dev/vfio/110", 
+  "volume_type": "vfiovol", 
+  "fs_type": "ext4", 
+  "metadata":"{}", 
+  "options": []
+}
+```
+
+Second, run kata-containers with device(`/dev/vfio/110`) as an example:
+
+```bash
+$ sudo kata-ctl direct-volume add /kubelet/kata-vfio-vol-003/vfiovol003 "{\"device\": \"/dev/vfio/110\", \"volume_type\": \"vfiovol\", \"fs_type\": \"ext4\", \"metadata\":"{}", \"options\": []}"
+$ # /kubelet/kata-vfio-vol-003/directvol003 <==> /run/kata-containers/shared/direct-volumes/F0va22F0ZvaS12F0YS10a2F0DAxvbC0F0ZXvdm9sdF0Z0YSx
+$ cat F0va22F0ZvaS12F0YS10a2F0DAxvbC0F0ZXvdm9sdF0Z0YSx/mountInfo.json 
+{"volume_type":"vfiovol","device":"/dev/vfio/110","fs_type":"ext4","metadata":{},"options":[]}
+```
+
+#### Run a Kata container with VFIO block device based volume
+
+```bash
+$ # type=disrectvol,src=/kubelet/kata-vfio-vol-003/vfiovol003,dst=/disk003,options=rbind:rw
+$ sudo ctr run -t --rm --runtime io.containerd.kata.v2 --mount type=vfiovol,src=/kubelet/kata-vfio-vol-003/vfiovol003,dst=/disk003,options=rbind:rw "$image" kata-vfio-vol-xx05302245 /bin/bash
+```
+
+
+### SPDK Device Based Block Volume
+
+SPDK vhost-user devices in runtime-rs, unlike runtime (golang version), there is no need to `mknod` device node under `/dev/` any more.
+Just using the `kata-ctl direct-volume add ..` to make a mount info config is enough.
+
+#### Run SPDK vhost target and Expose vhost block device
+
+Run a SPDK vhost target and get vhost-user block controller as an example:
+
+First, run SPDK vhost target:
+
+> **Tips:** If driver `vfio-pci` supported, you can run SPDK with `DRIVER_OVERRIDE=vfio-pci`
+> Otherwise, Just run without it `sudo HUGEMEM=4096 ./scripts/setup.sh`.
+
+```bash
+$ SPDK_DEVEL=/xx/spdk
+$ VHU_UDS_PATH=/tmp/vhu-targets
+$ RAW_DISKS=/xx/rawdisks
+$ # Reset first
+$ ${SPDK_DEVEL}/scripts/setup.sh reset
+$ sudo sysctl -w vm.nr_hugepages=2048
+$ #4G Huge Memory for spdk
+$ sudo HUGEMEM=4096 DRIVER_OVERRIDE=vfio-pci ${SPDK_DEVEL}/scripts/setup.sh
+$ sudo ${SPDK_DEVEL}/build/bin/spdk_tgt -S $VHU_UDS_PATH -s 1024 -m 0x3 &
+```
+
+Second, create a vhost controller:
+
+```bash
+$ sudo dd if=/dev/zero of=${RAW_DISKS}/rawdisk01.20g bs=1M count=20480
+$ sudo ${SPDK_DEVEL}/scripts/rpc.py bdev_aio_create ${RAW_DISKS}/rawdisk01.20g vhu-rawdisk01.20g 512
+$ sudo ${SPDK_DEVEL}/scripts/rpc.py vhost_create_blk_controller vhost-blk-rawdisk01.sock vhu-rawdisk01.20g
+```
+
+Here, a vhost controller `vhost-blk-rawdisk01.sock` is created, and the controller will
+be passed to Hypervisor, such as Dragonball, Cloud-Hypervisor, Firecracker or QEMU.
+
+
+#### setup vhost-user block device for kata-containers
+
+
+First, `mkdir` a sub-path `kubelet/kata-test-vol-001/` under `/run/kata-containers/shared/direct-volumes/`.
+
+Second, fill fields in `mountinfo.json`, it looks like as below:
+```json
+{
+  "device": "/tmp/vhu-targets/vhost-blk-rawdisk01.sock",
+  "volume_type": "spdkvol",
+  "fs_type": "ext4",
+  "metadata":"{}",
+  "options": []
+}
+```
+
+Third, with the help of `kata-ctl direct-volume` to add block device to generate `mountinfo.json`, and run a kata container with `--mount`.
+
+```bash
+$ # kata-ctl direct-volume add
+$ sudo kata-ctl direct-volume add /kubelet/kata-test-vol-001/volume001 "{\"device\": \"/tmp/vhu-targets/vhost-blk-rawdisk01.sock\", \"volume_type\":\"spdkvol\", \"fs_type\": \"ext4\", \"metadata\":"{}", \"options\": []}"
+$ # /kubelet/kata-test-vol-001/volume001 <==> /run/kata-containers/shared/direct-volumes/L2t1YmVsZXQva2F0YS10ZXN0LXZvbC0wMDEvdm9sdW1lMDAx
+$ cat L2t1YmVsZXQva2F0YS10ZXN0LXZvbC0wMDEvdm9sdW1lMDAx/mountInfo.json
+$ {"volume_type":"spdkvol","device":"/tmp/vhu-targets/vhost-blk-rawdisk01.sock","fs_type":"ext4","metadata":{},"options":[]}
+```
+
+As `/run/kata-containers/shared/direct-volumes/` is a fixed path , we will be able to run a kata pod with `--mount` and set
+`src` sub-path. And the `--mount` argument looks like: `--mount type=spdkvol,src=/kubelet/kata-test-vol-001/volume001,dst=/disk001`.
+
+
+#### Run a Kata container with SPDK vhost-user block device
+
+
+In the case, `ctr run --mount type=X, src=source, dst=dest`, the X will be set `spdkvol` which is a proprietary type specifically designed for SPDK volumes.
+
+```bash
+$ # ctr run with --mount type=spdkvol,src=/kubelet/kata-test-vol-001/volume001,dst=/disk001
+$ sudo ctr run -t --rm --runtime io.containerd.kata.v2 --mount type=spdkvol,src=/kubelet/kata-test-vol-001/volume001,dst=/disk001,options=rbind:rw "$image" kata-spdk-vol-xx0530 /bin/bash
+```

docs/how-to/how-to-run-rootless-vmm.md

@@ -1,5 +1,5 @@
 ## Introduction
-To improve security, Kata Container supports running the VMM process (currently only QEMU) as a non-`root` user.
+To improve security, Kata Container supports running the VMM process (QEMU and cloud-hypervisor) as a non-`root` user. 
 This document describes how to enable the rootless VMM mode and its limitations.
 
 ## Pre-requisites
@@ -27,7 +27,7 @@ Another necessary change is to move the hypervisor runtime files (e.g. `vhost-fs
 ## Limitations
 
 1. Only the VMM process is running as a non-root user. Other processes such as Kata Container shimv2 and `virtiofsd` still run as the root user.
-2. Currently, this feature is only supported in QEMU. Still need to bring it to Firecracker and Cloud Hypervisor (see https://github.com/kata-containers/kata-containers/issues/2567).
+2. Currently, this feature is only supported in QEMU and cloud-hypervisor. For firecracker, you can use jailer to run the VMM process with a non-root user.
 3. Certain features will not work when rootless VMM is enabled, including:
    1. Passing devices to the guest (`virtio-blk`, `virtio-scsi`) will not work if the non-privileged user does not have permission to access it (leading to a permission denied error). A more permissive permission (e.g. 666) may overcome this issue. However, you need to be aware of the potential security implications of reducing the security on such devices.
    2. `vfio` device will also not work because of permission denied error.

docs/how-to/how-to-set-prometheus-in-k8s.md

@@ -19,7 +19,7 @@ Also you should ensure that `kubectl` working correctly.
 > **Note**: More information about Kubernetes integrations:
 >   - [Run Kata Containers with Kubernetes](run-kata-with-k8s.md)
 >   - [How to use Kata Containers and Containerd](containerd-kata.md)
->   - [How to use Kata Containers and CRI (containerd plugin) with Kubernetes](how-to-use-k8s-with-cri-containerd-and-kata.md)
+>   - [How to use Kata Containers and containerd with Kubernetes](how-to-use-k8s-with-containerd-and-kata.md)
 
 ## Configure Prometheus
 

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -57,6 +57,7 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.enable_iothreads` | `boolean`| enable IO to be processed in a separate thread. Supported currently for virtio-`scsi` driver |
 | `io.katacontainers.config.hypervisor.enable_mem_prealloc` | `boolean` | the memory space used for `nvdimm` device by the hypervisor |
 | `io.katacontainers.config.hypervisor.enable_vhost_user_store` | `boolean` | enable vhost-user storage device (QEMU) |
+| `io.katacontainers.config.hypervisor.vhost_user_reconnect_timeout_sec` | `string`| the timeout for reconnecting vhost user socket (QEMU) 
 | `io.katacontainers.config.hypervisor.enable_virtio_mem` | `boolean` | enable virtio-mem (QEMU) |
 | `io.katacontainers.config.hypervisor.entropy_source` (R) | string| the path to a host source of entropy (`/dev/random`, `/dev/urandom` or real hardware RNG device) |
 | `io.katacontainers.config.hypervisor.file_mem_backend` (R) | string | file based memory backend root directory |
@@ -87,10 +88,11 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.use_vsock` | `boolean` | specify use of `vsock` for agent communication |
 | `io.katacontainers.config.hypervisor.vhost_user_store_path` (R) | `string` | specify the directory path where vhost-user devices related folders, sockets and device nodes should be (QEMU) |
 | `io.katacontainers.config.hypervisor.virtio_fs_cache_size` | uint32 | virtio-fs DAX cache size in `MiB` |
-| `io.katacontainers.config.hypervisor.virtio_fs_cache` | string | the cache mode for virtio-fs, valid values are `always`, `auto` and `none` |
+| `io.katacontainers.config.hypervisor.virtio_fs_cache` | string | the cache mode for virtio-fs, valid values are `always`, `auto` and `never` |
 | `io.katacontainers.config.hypervisor.virtio_fs_daemon` | string | virtio-fs `vhost-user` daemon path |
 | `io.katacontainers.config.hypervisor.virtio_fs_extra_args` | string | extra options passed to `virtiofs` daemon |
 | `io.katacontainers.config.hypervisor.enable_guest_swap` | `boolean` | enable swap in the guest |
+| `io.katacontainers.config.hypervisor.use_legacy_serial` | `boolean` | uses legacy serial device for guest's console (QEMU) |
 
 ## Container Options
 | Key | Value Type | Comments |
@@ -172,7 +174,7 @@ kind: Pod
 metadata:
   name: pod2
   annotations:
-    io.katacontainers.config.runtime.disable_guest_seccomp: false
+    io.katacontainers.config.runtime.disable_guest_seccomp: "false"
 spec:
   runtimeClassName: kata
   containers:

docs/how-to/how-to-setup-swap-devices-in-guest-kernel.md

@@ -17,9 +17,9 @@ Enable setup swap device in guest kernel as follows:
 $ sudo sed -i -e 's/^#enable_guest_swap.*$/enable_guest_swap = true/g' /etc/kata-containers/configuration.toml
 ```
 
-## Run a Kata Container utilizing swap device
+## Run a Kata Containers utilizing swap device
 
-Use following command to start a Kata Container with swappiness 60 and 1GB swap device (swap_in_bytes - memory_limit_in_bytes).
+Use following command to start a Kata Containers with swappiness 60 and 1GB swap device (swap_in_bytes - memory_limit_in_bytes).
 ```
 $ pod_yaml=pod.yaml
 $ container_yaml=container.yaml
@@ -43,12 +43,12 @@ command:
 - top
 EOF
 $ sudo crictl pull $image
-$ podid=$(sudo crictl runp $pod_yaml)
+$ podid=$(sudo crictl runp --runtime kata $pod_yaml)
 $ cid=$(sudo crictl create $podid $container_yaml $pod_yaml)
 $ sudo crictl start $cid
 ```
 
-Kata Container setups swap device for this container only when `io.katacontainers.container.resource.swappiness` is set.
+Kata Containers setups swap device for this container only when `io.katacontainers.container.resource.swappiness` is set.
 
 The following table shows the swap size how to decide if `io.katacontainers.container.resource.swappiness` is set.
 |`io.katacontainers.container.resource.swap_in_bytes`|`memory_limit_in_bytes`|swap size|

docs/how-to/how-to-use-erofs-build-rootfs.md

@@ -0,0 +1,90 @@
+# Configure Kata Containers to use EROFS build rootfs
+
+## Introduction
+For kata containers, rootfs is used in the read-only way. EROFS can noticeably decrease metadata overhead.
+
+`mkfs.erofs` can generate compressed and uncompressed EROFS images.
+
+For uncompressed images, no files are compressed. However, it is optional to inline the data blocks at the end of the file with the metadata.
+
+For compressed images, each file will be compressed using the lz4 or lz4hc algorithm, and it will be confirmed whether it can save space. Use No compression of the file if compression does not save space.
+
+## Performance comparison
+|                 | EROFS | EXT4 | XFS |
+|-----------------|-------| --- | --- |
+| Image Size [MB] | 106(uncompressed) | 256 | 126 |
+
+
+## Guidance
+### Install the `erofs-utils`
+#### `apt/dnf` install
+On newer `Ubuntu/Debian` systems, it can be installed directly using the `apt` command, and on `Fedora` it can be installed directly using the `dnf` command.
+
+```shell
+# Debian/Ubuntu
+$ apt install erofs-utils
+# Fedora
+$ dnf install erofs-utils
+```
+
+#### Source install
+[https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git](https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git)
+
+##### Compile dependencies
+If you need to enable the `Lz4` compression feature, `Lz4 1.8.0+` is required, and `Lz4 1.9.3+` is strongly recommended.
+
+##### Compilation process
+For some old lz4 versions (lz4-1.8.0~1.8.3), if lz4-static is not installed, the lz4hc algorithm will not be supported. lz4-static can be installed with apt install lz4-static.x86_64. However, these versions have some bugs in compression, and it is not recommended to use these versions directly.
+If you use `lz4 1.9.0+`, you can directly use the following command to compile.
+
+```shell
+$ ./autogen.sh
+$ ./configure
+$ make
+```
+
+The compiled `mkfs.erofs` program will be saved in the `mkfs` directory. Afterwards, the generated tools can be installed to a system directory using make install (requires root privileges).
+
+### Create a local rootfs
+```shell
+$ export distro="ubuntu"
+$ export FS_TYPE="erofs"
+$ export ROOTFS_DIR="realpath kata-containers/tools/osbuilder/rootfs-builder/rootfs"
+$ sudo rm -rf "${ROOTFS_DIR}"
+$ pushd kata-containers/tools/osbuilder/rootfs-builder
+$ script -fec 'sudo -E SECCOMP=no ./rootfs.sh "${distro}"'
+$ popd
+```
+
+### Add a custom agent to the image - OPTIONAL
+> Note:
+> - You should only do this step if you are testing with the latest version of the agent.
+```shell
+$ sudo install -o root -g root -m 0550 -t "${ROOTFS_DIR}/usr/bin" "${ROOTFS_DIR}/../../../../src/agent/target/x86_64-unknown-linux-musl/release/kata-agent"
+$ sudo install -o root -g root -m 0440 "${ROOTFS_DIR}/../../../../src/agent/kata-agent.service" "${ROOTFS_DIR}/usr/lib/systemd/system/"
+$ sudo install -o root -g root -m 0440 "${ROOTFS_DIR}/../../../../src/agent/kata-containers.target" "${ROOTFS_DIR}/usr/lib/systemd/system/"
+```
+
+### Build a root image
+```shell
+$ pushd  kata-containers/tools/osbuilder/image-builder
+$ script -fec 'sudo -E ./image_builder.sh "${ROOTFS_DIR}"'
+$ popd
+```
+
+### Install the rootfs image
+```shell
+$ pushd kata-containers/tools/osbuilder/image-builder
+$ commit="$(git log --format=%h -1 HEAD)"
+$ date="$(date +%Y-%m-%d-%T.%N%z)"
+$ rootfs="erofs"
+$ image="kata-containers-${rootfs}-${date}-${commit}"
+$ sudo install -o root -g root -m 0640 -D kata-containers.img "/usr/share/kata-containers/${image}"
+$ (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
+$ popd
+```
+
+### Use `EROFS` in the runtime
+```shell
+$ sudo sed -i -e 's/^# *\(rootfs_type\).*=.*$/\1 = erofs/g' /etc/kata-containers/configuration.toml
+```