Merge pull request #2541 from fidencio/2.2.0-branch-bump

# Kata Containers 2.2.0

.github/ISSUE_TEMPLATE.md

@@ -1,17 +0,0 @@
-# Description of problem
-
-(replace this text with the list of steps you followed)
-
-# Expected result
-
-(replace this text with an explanation of what you thought would happen)
-
-# Actual result
-
-(replace this text with details of what actually happened)
-
----
-
-(replace this text with the output of the `kata-collect-data.sh` script, after
-you have reviewed its content to ensure it does not contain any private
-information).

.github/workflows/commit-message-check.yaml

@@ -10,7 +10,7 @@ env:
   error_msg: |+
     See the document below for help on formatting commits for the project.
 
-    https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md#patch-forma
+    https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md#patch-format
 
 jobs:
   commit-message-check:
@@ -48,7 +48,25 @@ jobs:
       uses: tim-actions/commit-message-checker-with-regex@v0.3.1
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
-        pattern: '^.+(\n.{0,72})*$|^.+\n\s*[^a-zA-Z\s\n]|^.+\n\S+$'
+        # Notes:
+        #
+        # - The subject line is not enforced here (see other check), but has
+        #   to be specified at the start of the regex as the action is passed
+        #   the entire commit message.
+        #
+        # - Body lines *can* be longer than the maximum if they start
+        #   with a non-alphabetic character.
+        #
+        #   This allows stack traces, log files snippets, emails, long URLs,
+        #   etc to be specified. Some of these naturally "work" as they start
+        #   with numeric timestamps or addresses. Emails can but quoted using
+        #   the normal ">" character, markdown bullets ("-", "*") are also
+        #   useful for lists of URLs, but it is always possible to override
+        #   the check by simply space indenting the content you need to add.
+        #
+        # - A SoB comment can be any length (as it is unreasonable to penalise
+        #   people with long names/email addresses :)
+        pattern: '^.+(\n([a-zA-Z].{0,149}|[^a-zA-Z\n].*|Signed-off-by:.*|))+$'
         error: 'Body line too long (max 72)'
         post_error: ${{ env.error_msg }}
 

.github/workflows/gather-artifacts.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-# Copyright (c) 2019 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-set -o errexit
-set -o pipefail
-
-pushd kata-artifacts >>/dev/null
-for c in ./*.tar.gz
-do
-    echo "untarring tarball $c"
-    tar -xvf $c
-done
-
-tar cvfJ ../kata-static.tar.xz ./opt
-popd >>/dev/null

.github/workflows/generate-artifact-tarball.sh

@@ -1,36 +0,0 @@
-#!/bin/bash
-# Copyright (c) 2019 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-set -o errexit
-set -o pipefail
-
-
-main() {
-    artifact_stage=${1:-}
-    artifact=$(echo  ${artifact_stage} | sed -n -e 's/^install_//p' | sed -r 's/_/-/g')
-    if [ -z "${artifact}" ]; then
-        "Scripts needs artifact name to build"
-        exit 1
-    fi
-
-    tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-    export GOPATH=$HOME/go
-
-    go get github.com/kata-containers/packaging || true
-    pushd $GOPATH/src/github.com/kata-containers/packaging/release >>/dev/null
-    git checkout $tag
-    pushd ../obs-packaging
-    ./gen_versions_txt.sh $tag
-    popd
-
-    source ./kata-deploy-binaries.sh
-    ${artifact_stage} $tag
-    popd
-
-    mv $HOME/go/src/github.com/kata-containers/packaging/release/kata-static-${artifact}.tar.gz .
-}
-
-main $@

.github/workflows/generate-local-artifact-tarball.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-# Copyright (c) 2019 Intel Corporation
-# Copyright (c) 2020 Ant Group
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-set -o errexit
-set -o pipefail
-
-
-main() {
-    artifact_stage=${1:-}
-    artifact=$(echo  ${artifact_stage} | sed -n -e 's/^install_//p' | sed -r 's/_/-/g')
-    if [ -z "${artifact}" ]; then
-        "Scripts needs artifact name to build"
-        exit 1
-    fi
-
-    tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-    pushd $GITHUB_WORKSPACE/tools/packaging/obs-packaging
-    git checkout $tag
-    ./gen_versions_txt.sh $tag
-    popd
-
-    pushd $GITHUB_WORKSPACE/tools/packaging/release
-    source ./kata-deploy-binaries.sh
-    ${artifact_stage} $tag
-    popd
-
-    mv $GITHUB_WORKSPACE/tools/packaging/release/kata-static-${artifact}.tar.gz .
-}
-
-main $@

.github/workflows/kata-deploy-push.yaml

@@ -0,0 +1,58 @@
+name: kata-deploy-build
+
+on: push
+
+jobs:
+  build-asset:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        asset:
+          - kernel
+          - shim-v2
+          - qemu
+          - cloud-hypervisor
+          - firecracker
+          - rootfs-image
+          - rootfs-initrd
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install docker
+        run: |
+          curl -fsSL https://test.docker.com -o test-docker.sh
+          sh test-docker.sh
+
+      - name: Build ${{ matrix.asset }}
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh --build="${KATA_ASSET}"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r --preserve=all "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: build-asset
+    steps:
+      - uses: actions/checkout@v2
+      - name: get-artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: kata-artifacts
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: kata-static-tarball
+          path: kata-static.tar.xz

.github/workflows/kata-deploy-test.yaml

@@ -0,0 +1,65 @@
+on:
+  issue_comment:
+    types: [created, edited]
+
+name: test-kata-deploy
+
+jobs:
+  check_comments:
+    if: ${{ github.event.issue.pull_request }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check for Command
+        id: command
+        uses: kata-containers/slash-command-action@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          command: "test_kata_deploy"
+          reaction: "true"
+          reaction-type: "eyes"
+          allow-edits: "false"
+          permission-level: admin
+      - name: verify command arg is kata-deploy
+        run: |
+           echo "The command was '${{ steps.command.outputs.command-name }}' with arguments '${{ steps.command.outputs.command-arguments }}'"
+
+  create-and-test-container:
+    needs: check_comments
+    runs-on: ubuntu-latest
+    steps:
+      - name: get-PR-ref
+        id: get-PR-ref
+        run: |
+            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
+            echo "reference for PR: " ${ref}
+            echo "##[set-output name=pr-ref;]${ref}"
+
+      - name: check out
+        uses: actions/checkout@v2
+        with:
+           ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
+
+      - name: build-container-image
+        id: build-container-image
+        run: |
+            PR_SHA=$(git log --format=format:%H -n1)
+            VERSION="2.0.0"
+            ARTIFACT_URL="https://github.com/kata-containers/kata-containers/releases/download/${VERSION}/kata-static-${VERSION}-x86_64.tar.xz"
+            wget "${ARTIFACT_URL}" -O tools/packaging/kata-deploy/kata-static.tar.xz
+            docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:${PR_SHA} -t quay.io/kata-containers/kata-deploy-ci:${PR_SHA} ./tools/packaging/kata-deploy
+            docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
+            docker push katadocker/kata-deploy-ci:$PR_SHA
+            docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
+            docker push quay.io/kata-containers/kata-deploy-ci:$PR_SHA
+            echo "##[set-output name=pr-sha;]${PR_SHA}"
+
+      - name: test-kata-deploy-ci-in-aks
+        uses: ./tools/packaging/kata-deploy/action
+        with:
+          packaging-sha: ${{ steps.build-container-image.outputs.pr-sha }}
+        env:
+          PKG_SHA: ${{ steps.build-container-image.outputs.pr-sha }}
+          AZ_APPID: ${{ secrets.AZ_APPID }}
+          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
+          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}

.github/workflows/main.yaml

@@ -38,12 +38,12 @@ jobs:
         run: |
          if grep -q $buildstr ./artifact-list/artifact-list.txt; then
            $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo ::set-env name=artifact-built::true
+           echo "artifact-built=true" >> $GITHUB_ENV
          else
-           echo ::set-env name=artifact-built::false
+           echo "artifact-built=false" >> $GITHUB_ENV
          fi
       - name: store-artifacts
-        if: env.artifact-built == 'true'
+        if: ${{ env.artifact-built }} == 'true'
         uses: actions/upload-artifact@master
         with:
           name: kata-artifacts
@@ -66,12 +66,12 @@ jobs:
         run: |
          if grep -q $buildstr ./artifact-list/artifact-list.txt; then
            $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo ::set-env name=artifact-built::true
+           echo "artifact-built=true" >> $GITHUB_ENV
          else
-           echo ::set-env name=artifact-built::false
+           echo "artifact-built=false" >> $GITHUB_ENV
          fi
       - name: store-artifacts
-        if: env.artifact-built == 'true'
+        if: ${{ env.artifact-built }} == 'true'
         uses: actions/upload-artifact@master
         with:
           name: kata-artifacts
@@ -92,70 +92,17 @@ jobs:
         run: |
          if grep -q $buildstr ./artifact-list/artifact-list.txt; then
            $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo ::set-env name=artifact-built::true
+           echo "artifact-built=true" >> $GITHUB_ENV
          else
-           echo ::set-env name=artifact-built::false
+           echo "artifact-built=false" >> $GITHUB_ENV
          fi
       - name: store-artifacts
-        if: env.artifact-built == 'true'
+        if: ${{ env.artifact-built }} == 'true'
         uses: actions/upload-artifact@master
         with:
           name: kata-artifacts
           path: kata-static-qemu.tar.gz
 
-  build-nemu:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_nemu"
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifact-list
-        uses: actions/download-artifact@master
-        with:
-          name: artifact-list
-      - name: build-nemu
-        run: |
-         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo ::set-env name=artifact-built::true
-         else
-           echo ::set-env name=artifact-built::false
-         fi
-      - name: store-artifacts
-        if: env.artifact-built == 'true'
-        uses: actions/upload-artifact@master
-        with:
-          name: kata-artifacts
-          path: kata-static-nemu.tar.gz
-
-  # Job for building the QEMU binaries with virtiofs support
-  build-qemu-virtiofsd:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_qemu_virtiofsd"
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifact-list
-        uses: actions/download-artifact@master
-        with:
-          name: artifact-list
-      - name: build-qemu-virtiofsd
-        run: |
-         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo ::set-env name=artifact-built::true
-         else
-           echo ::set-env name=artifact-built::false
-         fi
-      - name: store-artifacts
-        if: env.artifact-built == 'true'
-        uses: actions/upload-artifact@master
-        with:
-          name: kata-artifacts
-          path: kata-static-qemu-virtiofsd.tar.gz
-
   # Job for building the image
   build-image:
     runs-on: ubuntu-16.04
@@ -172,12 +119,12 @@ jobs:
         run: |
          if grep -q $buildstr ./artifact-list/artifact-list.txt; then
            $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo ::set-env name=artifact-built::true
+           echo "artifact-built=true" >> $GITHUB_ENV
          else
-           echo ::set-env name=artifact-built::false
+           echo "artifact-built=false" >> $GITHUB_ENV
          fi
       - name: store-artifacts
-        if: env.artifact-built == 'true'
+        if: ${{ env.artifact-built }} == 'true'
         uses: actions/upload-artifact@master
         with:
           name: kata-artifacts
@@ -199,12 +146,12 @@ jobs:
         run: |
          if grep -q $buildstr ./artifact-list/artifact-list.txt; then
            $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo ::set-env name=artifact-built::true
+           echo "artifact-built=true" >> $GITHUB_ENV
          else
-           echo ::set-env name=artifact-built::false
+           echo "artifact-built=false" >> $GITHUB_ENV
          fi
       - name: store-artifacts
-        if: env.artifact-built == 'true'
+        if: ${{ env.artifact-built }} == 'true'
         uses: actions/upload-artifact@master
         with:
           name: kata-artifacts
@@ -226,12 +173,12 @@ jobs:
         run: |
          if grep -q $buildstr ./artifact-list/artifact-list.txt; then
            $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo ::set-env name=artifact-built::true
+           echo "artifact-built=true" >> $GITHUB_ENV
          else
-           echo ::set-env name=artifact-built::false
+           echo "artifact-built=false" >> $GITHUB_ENV
          fi
       - name: store-artifacts
-        if: env.artifact-built == 'true'
+        if: ${{ env.artifact-built }} == 'true'
         uses: actions/upload-artifact@master
         with:
           name: kata-artifacts
@@ -253,12 +200,12 @@ jobs:
         run: |
          if grep -q $buildstr ./artifact-list/artifact-list.txt; then
            $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo ::set-env name=artifact-built::true
+           echo "artifact-built=true" >> $GITHUB_ENV
          else
-           echo ::set-env name=artifact-built::false
+           echo "artifact-built=false" >> $GITHUB_ENV
          fi
       - name: store-artifacts
-        if: env.artifact-built == 'true'
+        if: ${{ env.artifact-built }} == 'true'
         uses: actions/upload-artifact@master
         with:
           name: kata-artifacts
@@ -266,7 +213,7 @@ jobs:
 
   gather-artifacts:
     runs-on: ubuntu-16.04
-    needs: [build-experimental-kernel, build-kernel, build-qemu, build-qemu-virtiofsd, build-image, build-firecracker, build-kata-components, build-nemu, build-clh]
+    needs: [build-experimental-kernel, build-kernel, build-qemu, build-image, build-firecracker, build-kata-components, build-clh]
     steps:
       - uses: actions/checkout@v1
       - name: get-artifacts
@@ -300,12 +247,12 @@ jobs:
           pkg_sha=$(git rev-parse HEAD)
           popd
           mv release-candidate/kata-static.tar.xz ./packaging/kata-deploy/kata-static.tar.xz
-          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:$pkg_sha ./packaging/kata-deploy
+          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:$pkg_sha -t quay.io/kata-containers/kata-deploy-ci:$pkg_sha ./packaging/kata-deploy
           docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
           docker push katadocker/kata-deploy-ci:$pkg_sha
-
-          echo "##[set-output name=PKG_SHA;]${pkg_sha}"
-          echo ::set-env name=TAG::$tag
+          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
+          docker push quay.io/kata-containers/kata-deploy-ci:$pkg_sha
+          echo "::set-output name=PKG_SHA::${pkg_sha}"
       - name: test-kata-deploy-ci-in-aks
         uses: ./packaging/kata-deploy/action
         with:

.github/workflows/release.yaml

@@ -5,239 +5,45 @@ on:
      - '2.*'
 
 jobs:
-  get-artifact-list:
+  build-asset:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        asset:
+          - cloud-hypervisor
+          - firecracker
+          - kernel
+          - qemu
+          - rootfs-image
+          - rootfs-initrd
+          - shim-v2
     steps:
       - uses: actions/checkout@v2
-      - name: get the list
+      - name: Install docker
         run: |
-         pushd $GITHUB_WORKSPACE
-         tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-         git checkout $tag
-         popd
-         $GITHUB_WORKSPACE/tools/packaging/artifact-list.sh > artifact-list.txt
-      - name: save-artifact-list
-        uses: actions/upload-artifact@v2
-        with:
-          name: artifact-list
-          path: artifact-list.txt
+          curl -fsSL https://test.docker.com -o test-docker.sh
+          sh test-docker.sh
 
-  build-kernel:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_kernel"
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifact-list
-        uses: actions/download-artifact@v2
-        with:
-          name: artifact-list
-      - run: |
-         sudo apt-get update && sudo apt install -y flex bison libelf-dev bc iptables
-      - name: build-kernel
+      - name: Build ${{ matrix.asset }}
         run: |
-         if grep -q $buildstr artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-local-artifact-tarball.sh $buildstr
-           echo ::set-env name=artifact-built::true
-         else
-           echo ::set-env name=artifact-built::false
-         fi
-      - name: store-artifacts
-        if: env.artifact-built == 'true'
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh --build="${KATA_ASSET}"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+
+      - name: store-artifact ${{ matrix.asset }}
         uses: actions/upload-artifact@v2
         with:
           name: kata-artifacts
-          path: kata-static-kernel.tar.gz
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          if-no-files-found: error
 
-  build-experimental-kernel:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_experimental_kernel"
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifact-list
-        uses: actions/download-artifact@v2
-        with:
-          name: artifact-list
-      - run: |
-         sudo apt-get update && sudo apt install -y flex bison libelf-dev bc iptables
-      - name: build-experimental-kernel
-        run: |
-         if grep -q $buildstr artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-local-artifact-tarball.sh $buildstr
-           echo ::set-env name=artifact-built::true
-         else
-           echo ::set-env name=artifact-built::false
-         fi
-      - name: store-artifacts
-        if: env.artifact-built == 'true'
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-static-experimental-kernel.tar.gz
-
-  build-qemu:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_qemu"
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifact-list
-        uses: actions/download-artifact@v2
-        with:
-          name: artifact-list
-      - name: build-qemu
-        run: |
-         if grep -q $buildstr artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-local-artifact-tarball.sh $buildstr
-           echo ::set-env name=artifact-built::true
-         else
-           echo ::set-env name=artifact-built::false
-         fi
-      - name: store-artifacts
-        if: env.artifact-built == 'true'
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-static-qemu.tar.gz
-
-  build-qemu-virtiofsd:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_qemu_virtiofsd"
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifact-list
-        uses: actions/download-artifact@v2
-        with:
-          name: artifact-list
-      - name: build-qemu-virtiofsd
-        run: |
-         if grep -q $buildstr artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-local-artifact-tarball.sh $buildstr
-           echo ::set-env name=artifact-built::true
-         else
-           echo ::set-env name=artifact-built::false
-         fi
-      - name: store-artifacts
-        if: env.artifact-built == 'true'
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-static-qemu-virtiofsd.tar.gz
-
-  build-image:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_image"
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifact-list
-        uses: actions/download-artifact@v2
-        with:
-          name: artifact-list
-      - name: build-image
-        run: |
-         if grep -q $buildstr artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-local-artifact-tarball.sh $buildstr
-           echo ::set-env name=artifact-built::true
-         else
-           echo ::set-env name=artifact-built::false
-         fi
-      - name: store-artifacts
-        if: env.artifact-built == 'true'
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-static-image.tar.gz
-
-  build-firecracker:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_firecracker"
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifact-list
-        uses: actions/download-artifact@v2
-        with:
-          name: artifact-list
-      - name: build-firecracker
-        run: |
-         if grep -q $buildstr artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-local-artifact-tarball.sh $buildstr
-           echo ::set-env name=artifact-built::true
-         else
-           echo ::set-env name=artifact-built::false
-         fi
-      - name: store-artifacts
-        if: env.artifact-built == 'true'
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-static-firecracker.tar.gz
-
-
-  build-clh:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_clh"
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifact-list
-        uses: actions/download-artifact@v2
-        with:
-          name: artifact-list
-      - name: build-clh
-        run: |
-         if grep -q $buildstr artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-local-artifact-tarball.sh $buildstr
-           echo ::set-env name=artifact-built::true
-         else
-           echo ::set-env name=artifact-built::false
-         fi
-      - name: store-artifacts
-        if: env.artifact-built == 'true'
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-static-clh.tar.gz
-
-  build-kata-components:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_kata_components"
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifact-list
-        uses: actions/download-artifact@v2
-        with:
-          name: artifact-list
-      - name: build-kata-components
-        run: |
-         if grep -q $buildstr artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-local-artifact-tarball.sh $buildstr
-           echo ::set-env name=artifact-built::true
-         else
-           echo ::set-env name=artifact-built::false
-         fi
-      - name: store-artifacts
-        if: env.artifact-built == 'true'
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-static-kata-components.tar.gz
-
-  gather-artifacts:
-    runs-on: ubuntu-16.04
-    needs: [build-experimental-kernel, build-kernel, build-qemu, build-qemu-virtiofsd, build-image, build-firecracker, build-kata-components, build-clh]
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: build-asset
     steps:
       - uses: actions/checkout@v2
       - name: get-artifacts
@@ -245,24 +51,24 @@ jobs:
         with:
           name: kata-artifacts
           path: kata-artifacts
-      - name: colate-artifacts
+      - name: merge-artifacts
         run: |
-          $GITHUB_WORKSPACE/.github/workflows/gather-artifacts.sh
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
       - name: store-artifacts
         uses: actions/upload-artifact@v2
         with:
-          name: release-candidate
+          name: kata-static-tarball
           path: kata-static.tar.xz
 
   kata-deploy:
-    needs: gather-artifacts
+    needs: create-kata-tarball
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: get-artifacts
+      - name: get-kata-tarball
         uses: actions/download-artifact@v2
         with:
-          name: release-candidate
+          name: kata-static-tarball
       - name: build-and-push-kata-deploy-ci
         id: build-and-push-kata-deploy-ci
         run: |
@@ -272,14 +78,14 @@ jobs:
           pkg_sha=$(git rev-parse HEAD)
           popd
           mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
-          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
+          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:$pkg_sha -t quay.io/kata-containers/kata-deploy-ci:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
           docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
           docker push katadocker/kata-deploy-ci:$pkg_sha
-
-          echo "##[set-output name=PKG_SHA;]${pkg_sha}"
-          echo ::set-env name=TAG::$tag
+          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
+          docker push quay.io/kata-containers/kata-deploy-ci:$pkg_sha
           mkdir -p packaging/kata-deploy
           ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
+          echo "::set-output name=PKG_SHA::${pkg_sha}"
       - name: test-kata-deploy-ci-in-aks
         uses: ./packaging/kata-deploy/action
         with:
@@ -295,7 +101,9 @@ jobs:
           # tag the container image we created and push to DockerHub
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
           docker tag katadocker/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} katadocker/kata-deploy:${tag}
+          docker tag quay.io/kata-containers/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} quay.io/kata-containers/kata-deploy:${tag}
           docker push katadocker/kata-deploy:${tag}
+          docker push quay.io/kata-containers/kata-deploy:${tag}
 
   upload-static-tarball:
     needs: kata-deploy
@@ -305,7 +113,7 @@ jobs:
       - name: download-artifacts
         uses: actions/download-artifact@v2
         with:
-          name: release-candidate
+          name: kata-static-tarball
       - name: install hub
         run: |
           HUB_VER=$(curl -s "https://api.github.com/repos/github/hub/releases/latest" | jq -r .tag_name | sed 's/^v//')

.github/workflows/require-pr-porting-labels.yaml

@@ -12,6 +12,9 @@ on:
       - reopened
       - labeled
       - unlabeled
+   pull_request:
+     branches:
+      - main
 
 jobs:
   check-pr-porting-labels:

.github/workflows/snap-release.yaml

@@ -0,0 +1,39 @@
+name: Release Kata 2.x in snapcraft store
+on:
+  push:
+    tags:
+      - '2.*'
+jobs:
+  release-snap:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Check out Git repository
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Install Snapcraft
+        uses: samuelmeuli/action-snapcraft@v1
+        with:
+          snapcraft_token: ${{ secrets.snapcraft_token }}
+
+      - name: Build snap
+        run: |
+          sudo apt-get install -y git git-extras
+          kata_url="https://github.com/kata-containers/kata-containers"
+          latest_version=$(git ls-remote --tags ${kata_url}  | egrep -o "refs.*" | egrep -v "\-alpha|\-rc|{}" | egrep -o "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+" | sort -V -r | head -1)
+          current_version="$(echo ${GITHUB_REF} | cut -d/ -f3)"
+          # Check semantic versioning format (x.y.z) and if the current tag is the latest tag
+          if echo "${current_version}" | grep -q "^[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+$" && echo -e "$latest_version\n$current_version" | sort -C -V; then
+            # Current version is the latest version, build it
+            snapcraft -d snap --destructive-mode
+          fi
+
+      - name: Upload snap
+        run: |
+          snap_version="$(echo ${GITHUB_REF} | cut -d/ -f3)"
+          snap_file="kata-containers_${snap_version}_amd64.snap"
+          # Upload the snap if it exists
+          if [ -f ${snap_file} ]; then
+            snapcraft upload --release=stable ${snap_file}
+          fi

.github/workflows/snap.yaml

@@ -0,0 +1,17 @@
+name: snap CI
+on: ["pull_request"]
+jobs:
+  test:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Check out
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Install Snapcraft
+        uses: samuelmeuli/action-snapcraft@v1
+
+      - name: Build snap
+        run: |
+          snapcraft -d snap --destructive-mode

.github/workflows/static-checks.yaml

@@ -0,0 +1,86 @@
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+      - labeled
+      - unlabeled
+
+name: Static checks
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.15.x, 1.16.x]
+        os: [ubuntu-20.04]
+    runs-on: ${{ matrix.os }}
+    env:
+      TRAVIS: "true"
+      TRAVIS_BRANCH: ${{ github.base_ref }}
+      TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }}
+      TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }}
+      RUST_BACKTRACE: "1"
+      target_branch: ${{ github.base_ref }}
+    steps:
+    - name: Install Go
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+      env:
+        GOPATH: ${{ runner.workspace }}/kata-containers
+    - name: Setup GOPATH
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}"
+        echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}"
+        echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}"
+        echo "TRAVIS: ${TRAVIS}"
+    - name: Set env
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
+        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+    - name: Checkout code
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+        path: ./src/github.com/${{ github.repository }}
+    - name: Setup travis references
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}"
+        target_branch=${TRAVIS_BRANCH}
+    - name: Setup
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
+      env:
+        GOPATH: ${{ runner.workspace }}/kata-containers
+    - name: Building rust
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh
+        PATH=$PATH:"$HOME/.cargo/bin"
+        rustup target add x86_64-unknown-linux-musl
+        rustup component add rustfmt clippy
+    # Check whether the vendored code is up-to-date & working as the first thing
+    - name: Check vendored code
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && make vendor
+    - name: Static Checks
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && make static-checks
+    - name: Run Compiler Checks
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && make check
+    - name: Run Unit Tests
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && make test

.gitignore

@@ -1,5 +1,12 @@
 **/*.bk
+**/*~
 **/*.orig
 **/*.rej
 **/target
 **/.vscode
+pkg/logging/Cargo.lock
+src/agent/src/version.rs
+src/agent/kata-agent.service
+src/agent/protocols/src/*.rs
+!src/agent/protocols/src/lib.rs
+

.travis.yml

@@ -1,46 +0,0 @@
-# Copyright (c) 2019 Ant Financial
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-dist: bionic
-os: linux
-language: go
-go: 1.14.4
-env: target_branch=$TRAVIS_BRANCH
-
-before_install:
-  - git remote set-branches --add origin "${TRAVIS_BRANCH}"
-  - git fetch
-  - "ci/setup.sh"
-
-# we use install to run check agent
-# so that it is easy to skip for non-amd64 platform
-install:
-  - "ci/install_rust.sh"
-  - export PATH=$PATH:"$HOME/.cargo/bin"
-  - export RUST_AGENT=yes
-  - make -C ${TRAVIS_BUILD_DIR}/src/agent
-  - make -C ${TRAVIS_BUILD_DIR}/src/agent check
-  - sudo -E PATH=$PATH make -C ${TRAVIS_BUILD_DIR}/src/agent check
-
-before_script:
-  - "ci/install_go.sh"
-  - "ci/install_vc.sh"
-  - make -C ${TRAVIS_BUILD_DIR}/src/runtime
-  - make -C ${TRAVIS_BUILD_DIR}/src/runtime test
-  - sudo -E PATH=$PATH GOPATH=$GOPATH make -C ${TRAVIS_BUILD_DIR}/src/runtime test
-
-script:
-  - "ci/static-checks.sh"
-
-jobs:
-  include:
-  - name: x86_64 test
-    os: linux
-  - name: ppc64le test
-    os: linux-ppc64le
-    install: skip
-  allow_failures:
-    - name: ppc64le test
-  fast_finish: true

CODEOWNERS

@@ -1,4 +1,4 @@
-# Copyright 2019 Intel Corporation.
+# Copyright (c) 2019 Intel Corporation
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -10,4 +10,3 @@
 # used. See https://help.github.com/articles/about-code-owners/
 
 *.md    @kata-containers/documentation
-

Glossary.md

@@ -0,0 +1,94 @@
+# Glossary
+
+[A](#a), [B](#b), [C](#c), [D](#d), [E](#e), [F](#f), [G](#g), [H](#h), [I](#i), [J](#j), [K](#k), [L](#l), [M](#m), [N](#n), [O](#o), [P](#p), [Q](#q), [R](#r), [S](#s), [T](#t), [U](#u), [V](#v), [W](#w), [X](#x), [Y](#y), [Z](#z)
+
+## A
+
+### Auto Scaling
+a method used in cloud computing, whereby the amount of computational resources in a server farm, typically measured in terms of the number of active servers, which vary automatically based on the load on the farm.
+
+## B
+
+## C
+
+### Container Security Solutions
+The process of implementing security tools and policies that will give you the assurance that everything in your container is running as intended, and only as intended.
+
+### Container Software
+A standard unit of software that packages up code and all its dependencies so the application runs quickly and reliably from one computing environment to another.
+
+### Container Runtime Interface
+A plugin interface which enables Kubelet to use a wide variety of container runtimes, without the need to recompile.
+
+### Container Virtualization
+A container is a virtual runtime environment that runs on top of a single operating system (OS) kernel and emulates an operating system rather than the underlying hardware.
+
+## D
+
+## E
+
+## F
+
+## G
+
+## H
+
+## I
+
+### Infrastructure Architecture
+A structured and modern approach for supporting an organization and facilitating innovation within an enterprise.
+
+## J
+
+## K
+
+### Kata Containers
+Kata containers is an open source project delivering increased container security and Workload isolation through an implementation of lightweight virtual machines.
+
+## L
+
+## M
+
+## N
+
+## O
+
+## P
+
+### Pod Containers
+A Group of one or more containers , with shared storage/network, and a specification for how to run the containers.
+
+### Private Cloud
+A computing model that offers a proprietary environment dedicated to a single business entity.
+
+### Public Cloud
+Computing services offered by third-party providers over the public Internet, making them available to anyone who wants to use or purchase them.
+
+## Q
+
+## R
+
+## S
+
+### Serverless Containers
+An architecture in which code is executed on-demand. Serverless workloads are typically in the cloud, but on-premises serverless platforms exist, too.
+
+## T
+
+## U
+
+## V
+
+### Virtual Machine Monitor
+Computer software, firmware or hardware that creates and runs virtual machines.
+
+### Virtual Machine Software
+A software program or operating system that not only exhibits the behavior of a separate computer, but is also capable of performing tasks such as running applications and programs like a separate computer.
+
+## W
+
+## X
+
+## Y
+
+## Z

Makefile

@@ -15,7 +15,7 @@ TOOLS =
 
 TOOLS += agent-ctl
 
-STANDARD_TARGETS = build check clean install test
+STANDARD_TARGETS = build check clean install test vendor
 
 include utils.mk
 
@@ -29,4 +29,14 @@ $(eval $(call create_all_rules,$(COMPONENTS),$(TOOLS),$(STANDARD_TARGETS)))
 generate-protocols:
 	make -C src/agent generate-protocols
 
-.PHONY: all default
+# Some static checks rely on generated source files of components.
+static-checks: build
+	bash ci/static-checks.sh
+
+binary-tarball:
+	make -f ./tools/packaging/kata-deploy/local-build/Makefile
+
+install-binary-tarball:
+	make -f ./tools/packaging/kata-deploy/local-build/Makefile install
+
+.PHONY: all default static-checks binary-tarball install-binary-tarball

README.md

@@ -2,136 +2,90 @@
 
 # Kata Containers
 
-* [Raising issues](#raising-issues)
-* [Kata Containers repositories](#kata-containers-repositories)
-    * [Code Repositories](#code-repositories)
-        * [Kata Containers-developed components](#kata-containers-developed-components)
-            * [Agent](#agent)
-            * [KSM throttler](#ksm-throttler)
-            * [Runtime](#runtime)
-            * [Trace forwarder](#trace-forwarder)
-        * [Additional](#additional)
-            * [Hypervisor](#hypervisor)
-            * [Kernel](#kernel)
-    * [CI](#ci)
-    * [Community](#community)
-    * [Documentation](#documentation)
-    * [Packaging](#packaging)
-    * [Test code](#test-code)
-    * [Utilities](#utilities)
-        * [OS builder](#os-builder)
-    * [Web content](#web-content)
-
----
-
 Welcome to Kata Containers!
 
-The purpose of this repository is to act as a "top level" site for the project. Specifically it is used:
+This repository is the home of the Kata Containers code for the 2.0 and newer
+releases.
 
-- To provide a list of the various *other* [Kata Containers repositories](#kata-containers-repositories),
-  along with a brief explanation of their purpose.
+If you want to learn about Kata Containers, visit the main
+[Kata Containers website](https://katacontainers.io).
 
-- To provide a general area for [Raising Issues](#raising-issues).
+## Introduction
 
-## Raising issues
+Kata Containers is an open source project and community working to build a
+standard implementation of lightweight Virtual Machines (VMs) that feel and
+perform like containers, but provide the workload isolation and security
+advantages of VMs.
 
-This repository is used for [raising
-issues](https://github.com/kata-containers/kata-containers/issues/new):
+## Getting started
 
-- That might affect multiple code repositories.
+See the [installation documentation](docs/install).
 
-- Where the raiser is unsure which repositories are affected.
+## Documentation
+
+See the [official documentation](docs)
+(including [installation guides](docs/install),
+[the developer guide](docs/Developer-Guide.md),
+[design documents](docs/design) and more).
+
+## Community
+
+To learn more about the project, its community and governance, see the
+[community repository](https://github.com/kata-containers/community). This is
+the first place to go if you wish to contribute to the project.
+
+## Getting help
+
+See the [community](#community) section for ways to contact us.
+
+### Raising issues
+
+Please raise an issue
+[in this repository](https://github.com/kata-containers/kata-containers/issues).
 
 > **Note:**
-> 
-> - If an issue affects only a single component, it should be raised in that
->   components repository.
+> If you are reporting a security issue, please follow the [vulnerability reporting process](https://github.com/kata-containers/community#vulnerability-handling)
 
-## Kata Containers repositories
+## Developers
 
-### CI
+### Components
 
-The [CI](https://github.com/kata-containers/ci) repository stores the Continuous
-Integration (CI) system configuration information.
+### Main components
 
-### Community
+The table below lists the core parts of the project:
 
-The [Community](https://github.com/kata-containers/community) repository is
-the first place to go if you want to use or contribute to the project.
+| Component | Type | Description |
+|-|-|-|
+| [runtime](src/runtime) | core | Main component run by a container manager and providing a containerd shimv2 runtime implementation. |
+| [agent](src/agent) | core | Management process running inside the virtual machine / POD that sets up the container environment. |
+| [documentation](docs) | documentation | Documentation common to all components (such as design and install documentation). |
+| [tests](https://github.com/kata-containers/tests) | tests | Excludes unit tests which live with the main code. |
 
-### Code Repositories
+### Additional components
 
-#### Kata Containers-developed components
+The table below lists the remaining parts of the project:
 
-##### Agent
+| Component | Type | Description |
+|-|-|-|
+| [packaging](tools/packaging) | infrastructure | Scripts and metadata for producing packaged binaries<br/>(components, hypervisors, kernel and rootfs). |
+| [kernel](https://www.kernel.org) | kernel | Linux kernel used by the hypervisor to boot the guest image. Patches are stored [here](tools/packaging/kernel). |
+| [osbuilder](tools/osbuilder) | infrastructure | Tool to create "mini O/S" rootfs and initrd images and kernel for the hypervisor. |
+| [`agent-ctl`](tools/agent-ctl) | utility | Tool that provides low-level access for testing the agent. |
+| [`trace-forwarder`](src/trace-forwarder) | utility | Agent tracing helper. |
+| [`ci`](https://github.com/kata-containers/ci) | CI | Continuous Integration configuration files and scripts. |
+| [`katacontainers.io`](https://github.com/kata-containers/www.katacontainers.io) | Source for the [`katacontainers.io`](https://www.katacontainers.io) site. |
 
-The [`kata-agent`](src/agent/README.md) runs inside the
-virtual machine and sets up the container environment.
+### Packaging and releases
 
-##### KSM throttler
+Kata Containers is now
+[available natively for most distributions](docs/install/README.md#packaged-installation-methods).
+However, packaging scripts and metadata are still used to generate snap and GitHub releases. See
+the [components](#components) section for further details.
 
-The [`kata-ksm-throttler`](https://github.com/kata-containers/ksm-throttler)
-is an optional utility that monitors containers and deduplicates memory to
-maximize container density on a host.
+## Glossary of Terms
 
-##### Runtime
+See the [glossary of terms](Glossary.md) related to Kata Containers.
+---
 
-The [`kata-runtime`](src/runtime/README.md) is usually
-invoked by a container manager and provides high-level verbs to manage
-containers.
-
-##### Trace forwarder
-
-The [`kata-trace-forwarder`](src/trace-forwarder) is a component only used
-when tracing the [agent](#agent) process.
-
-#### Additional
-
-##### Hypervisor
-
-The [`qemu`](https://github.com/kata-containers/qemu) hypervisor is used to
-create virtual machines for hosting the containers.
-
-##### Kernel
-
-The hypervisor uses a [Linux\* kernel](https://github.com/kata-containers/linux) to boot the guest image.
-
-### Documentation
-
-The [docs](docs/README.md) directory holds documentation common to all code components.
-
-### Packaging
-
-We use the [packaging](tools/packaging/README.md) to create packages for the [system
-components](#kata-containers-developed-components) including
-[rootfs](#os-builder) and [kernel](#kernel) images.
-
-### Test code
-
-The [tests](https://github.com/kata-containers/tests) repository hosts all
-test code except the unit testing code (which is kept in the same repository
-as the component it tests).
-
-### Utilities
-
-#### OS builder
-
-The [osbuilder](tools/osbuilder/README.md) tool can create
-a rootfs and a "mini O/S" image. This image is used by the hypervisor to setup
-the environment before switching to the workload.
-
-#### `kata-agent-ctl`
-
-[`kata-agent-ctl`](tools/agent-ctl) is a low-level test tool for
-interacting with the agent.
-
-### Web content
-
-The
-[www.katacontainers.io](https://github.com/kata-containers/www.katacontainers.io)
-repository contains all sources for the https://www.katacontainers.io site.
-
-## Credits
-
-Kata Containers uses [packagecloud](https://packagecloud.io) for package
-hosting.
+[kernel]: https://www.kernel.org
+[github-katacontainers.io]: https://github.com/kata-containers/www.katacontainers.io

VERSION

@@ -1 +1 @@
-2.0.0-rc0
+2.2.0

ci/go-no-os-exit.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+# Copyright (c) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Check there are no os.Exit() calls creeping into the code
+# We don't use that exit path in the Kata codebase.
+
+# Allow the path to check to be over-ridden.
+# Default to the current directory.
+go_packages=${1:-.}
+
+echo "Checking for no os.Exit() calls for package [${go_packages}]"
+
+candidates=`go list -f '{{.Dir}}/*.go' $go_packages`
+for f in $candidates; do
+	filename=`basename $f`
+	# skip all go test files
+	[[ $filename == *_test.go ]] && continue
+	# skip exit.go where, the only file we should call os.Exit() from.
+	[[ $filename == "exit.go" ]] && continue
+	files="$f $files"
+done
+
+[ -z "$files" ] && echo "No files to check, skipping" && exit 0
+
+if egrep -n '\<os\.Exit\>' $files; then
+	echo "Direct calls to os.Exit() are forbidden, please use exit() so atexit() works"
+	exit 1
+fi

ci/install_musl.sh

@@ -12,10 +12,11 @@ install_aarch64_musl() {
 		local musl_tar="${arch}-linux-musl-native.tgz"
 		local musl_dir="${arch}-linux-musl-native"
 		pushd /tmp
-		curl -sLO https://musl.cc/${musl_tar}
-		tar -zxf ${musl_tar}
-		mkdir -p /usr/local/musl/
-		cp -r ${musl_dir}/* /usr/local/musl/
+		if curl -sLO --fail https://musl.cc/${musl_tar}; then
+			tar -zxf ${musl_tar}
+			mkdir -p /usr/local/musl/
+			cp -r ${musl_dir}/* /usr/local/musl/
+		fi
 		popd
 	fi
 }

ci/install_yq.sh

@@ -15,10 +15,18 @@ die() {
 # Install the yq yaml query package from the mikefarah github repo
 # Install via binary download, as we may not have golang installed at this point
 function install_yq() {
-	GOPATH=${GOPATH:-${HOME}/go}
-	local yq_path="${GOPATH}/bin/yq"
 	local yq_pkg="github.com/mikefarah/yq"
-	[ -x  "${GOPATH}/bin/yq" ] && return
+	local yq_version=3.4.1
+	INSTALL_IN_GOPATH=${INSTALL_IN_GOPATH:-true}
+
+	if [ "${INSTALL_IN_GOPATH}"  == "true" ];then
+		GOPATH=${GOPATH:-${HOME}/go}
+		mkdir -p "${GOPATH}/bin"
+		local yq_path="${GOPATH}/bin/yq"
+	else
+		yq_path="/usr/local/bin/yq"
+	fi
+	[ -x  "${yq_path}" ] && [ "`${yq_path} --version`"X == "yq version ${yq_version}"X ] && return
 
 	read -r -a sysInfo <<< "$(uname -sm)"
 
@@ -49,15 +57,12 @@ function install_yq() {
 		;;
 	esac
 
-	mkdir -p "${GOPATH}/bin"
 
 	# Check curl
 	if ! command -v "curl" >/dev/null; then
 		die "Please install curl"
 	fi
 
-	local yq_version=3.1.0
-
 	## NOTE: ${var,,} => gives lowercase value of var
 	local yq_url="https://${yq_pkg}/releases/download/${yq_version}/yq_${goos,,}_${goarch}"
 	curl -o "${yq_path}" -LSsf "${yq_url}"

ci/lib.sh

@@ -3,28 +3,39 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+set -o nounset
+
 export tests_repo="${tests_repo:-github.com/kata-containers/tests}"
 export tests_repo_dir="$GOPATH/src/$tests_repo"
+export branch="${target_branch:-main}"
 
+# Clones the tests repository and checkout to the branch pointed out by
+# the global $branch variable.
+# If the clone exists and `CI` is exported then it does nothing. Otherwise
+# it will clone the repository or `git pull` the latest code.
+#
 clone_tests_repo()
 {
-	# KATA_CI_NO_NETWORK is (has to be) ignored if there is
-	# no existing clone.
-	if [ -d "$tests_repo_dir" -a -n "$KATA_CI_NO_NETWORK" ]
-	then
-		return
-	fi
-
-	go get -d -u "$tests_repo" || true
-
-	if [ -n "${TRAVIS_BRANCH:-}" ]; then
-		( cd "${tests_repo_dir}" && git checkout "${TRAVIS_BRANCH}" )
+	if [ -d "$tests_repo_dir" ]; then
+		[ -n "${CI:-}" ] && return
+		pushd "${tests_repo_dir}"
+		git checkout "${branch}"
+		git pull
+		popd
+	else
+		git clone -q "https://${tests_repo}" "$tests_repo_dir"
+		pushd "${tests_repo_dir}"
+		git checkout "${branch}"
+		popd
 	fi
 }
 
 run_static_checks()
 {
 	clone_tests_repo
+	# Make sure we have the targeting branch
+	git remote set-branches --add origin "${branch}"
+	git fetch -a
 	bash "$tests_repo_dir/.ci/static-checks.sh" "github.com/kata-containers/kata-containers"
 }
 

ci/openshift-ci/images/Dockerfile.buildroot

@@ -0,0 +1,9 @@
+# Copyright (c) 2021 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# This is the build root image for Kata Containers on OpenShift CI.
+#
+FROM centos:8
+
+RUN yum -y update && yum -y install git sudo wget

ci/run.sh

@@ -13,4 +13,6 @@ clone_tests_repo
 
 pushd ${tests_repo_dir}
 .ci/run.sh
+# temporary fix, see https://github.com/kata-containers/tests/issues/3878
+[ "$(uname -m)" != "s390x" ] && tracing/test-agent-shutdown.sh
 popd

docs/CODE_OF_CONDUCT.md

@@ -1,3 +0,0 @@
-## Kata Containers Documentation Code of Conduct
-
-Kata Containers follows the [OpenStack Foundation Code of Conduct](https://www.openstack.org/legal/community-code-of-conduct/).

docs/CONTRIBUTING.md

@@ -1,5 +0,0 @@
-# Contributing
-
-## This repo is part of [Kata Containers](https://katacontainers.io)
-
-For details on how to contribute to the Kata Containers project, please see the main [contributing document](https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md).

docs/Developer-Guide.md

@@ -1,56 +1,3 @@
-* [Warning](#warning)
-* [Assumptions](#assumptions)
-* [Initial setup](#initial-setup)
-* [Requirements to build individual components](#requirements-to-build-individual-components)
-* [Build and install the Kata Containers runtime](#build-and-install-the-kata-containers-runtime)
-* [Check hardware requirements](#check-hardware-requirements)
-    * [Configure to use initrd or rootfs image](#configure-to-use-initrd-or-rootfs-image)
-    * [Enable full debug](#enable-full-debug)
-        * [debug logs and shimv2](#debug-logs-and-shimv2)
-            * [Enabling full `containerd` debug](#enabling-full-containerd-debug)
-            * [Enabling just `containerd shim` debug](#enabling-just-containerd-shim-debug)
-            * [Enabling `CRI-O` and `shimv2` debug](#enabling-cri-o-and-shimv2-debug)
-        * [journald rate limiting](#journald-rate-limiting)
-            * [`systemd-journald` suppressing messages](#systemd-journald-suppressing-messages)
-            * [Disabling `systemd-journald` rate limiting](#disabling-systemd-journald-rate-limiting)
-* [Build and install Kata shim](#build-and-install-kata-shim)
-* [Create and install rootfs and initrd image](#create-and-install-rootfs-and-initrd-image)
-    * [Build a custom Kata agent - OPTIONAL](#build-a-custom-kata-agent---optional)
-    * [Get the osbuilder](#get-the-osbuilder)
-    * [Create a rootfs image](#create-a-rootfs-image)
-        * [Create a local rootfs](#create-a-local-rootfs)
-        * [Add a custom agent to the image - OPTIONAL](#add-a-custom-agent-to-the-image---optional)
-        * [Build a rootfs image](#build-a-rootfs-image)
-        * [Install the rootfs image](#install-the-rootfs-image)
-    * [Create an initrd image - OPTIONAL](#create-an-initrd-image---optional)
-        * [Create a local rootfs for initrd image](#create-a-local-rootfs-for-initrd-image)
-        * [Build an initrd image](#build-an-initrd-image)
-        * [Install the initrd image](#install-the-initrd-image)
-* [Install guest kernel images](#install-guest-kernel-images)
-* [Install a hypervisor](#install-a-hypervisor)
-    * [Build a custom QEMU](#build-a-custom-qemu)
-        * [Build a custom QEMU for aarch64/arm64 - REQUIRED](#build-a-custom-qemu-for-aarch64arm64---required)
-* [Run Kata Containers with Docker](#run-kata-containers-with-docker)
-    * [Update the Docker systemd unit file](#update-the-docker-systemd-unit-file)
-    * [Create a container using Kata](#create-a-container-using-kata)
-* [Run Kata Containers with Kubernetes](#run-kata-containers-with-kubernetes)
-* [Troubleshoot Kata Containers](#troubleshoot-kata-containers)
-* [Appendices](#appendices)
-    * [Checking Docker default runtime](#checking-docker-default-runtime)
-    * [Set up a debug console](#set-up-a-debug-console)
-        * [Create a custom image containing a shell](#create-a-custom-image-containing-a-shell)
-        * [Create a debug systemd service](#create-a-debug-systemd-service)
-        * [Build the debug image](#build-the-debug-image)
-        * [Configure runtime for custom debug image](#configure-runtime-for-custom-debug-image)
-        * [Ensure debug options are valid](#ensure-debug-options-are-valid)
-        * [Create a container](#create-a-container)
-        * [Connect to the virtual machine using the debug console](#connect-to-the-virtual-machine-using-the-debug-console)
-        * [Obtain details of the image](#obtain-details-of-the-image)
-    * [Capturing kernel boot logs](#capturing-kernel-boot-logs)
-    * [Running standalone](#running-standalone)
-        * [Create an OCI bundle](#create-an-oci-bundle)
-        * [Launch the runtime to create a container](#launch-the-runtime-to-create-a-container)
-
 # Warning
 
 This document is written **specifically for developers**: it is not intended for end users.
@@ -66,7 +13,7 @@ The recommended way to create a development environment is to first
 to create a working system.
 
 The installation guide instructions will install all required Kata Containers
-components, plus Docker*, the hypervisor, and the Kata Containers image and
+components, plus *Docker*, the hypervisor, and the Kata Containers image and
 guest kernel.
 
 # Requirements to build individual components
@@ -76,7 +23,12 @@ You need to install the following to build Kata Containers components:
 - [golang](https://golang.org/dl)
 
   To view the versions of go known to work, see the `golang` entry in the
-  [versions database](https://github.com/kata-containers/runtime/blob/master/versions.yaml).
+  [versions database](../versions.yaml).
+
+- [rust](https://www.rust-lang.org/tools/install)
+
+  To view the versions of rust known to work, see the `rust` entry in the
+  [versions database](../versions.yaml).
 
 - `make`.
 - `gcc` (required for building the shim and runtime).
@@ -84,14 +36,14 @@ You need to install the following to build Kata Containers components:
 # Build and install the Kata Containers runtime
 
 ```
-$ go get -d -u github.com/kata-containers/runtime
-$ cd $GOPATH/src/github.com/kata-containers/runtime
+$ go get -d -u github.com/kata-containers/kata-containers
+$ cd $GOPATH/src/github.com/kata-containers/kata-containers/src/runtime
 $ make && sudo -E PATH=$PATH make install
 ```
 
 The build will create the following:
 
-- runtime binary: `/usr/local/bin/kata-runtime`
+- runtime binary: `/usr/local/bin/kata-runtime` and `/usr/local/bin/containerd-shim-kata-v2`
 - configuration file: `/usr/share/defaults/kata-containers/configuration.toml`
 
 # Check hardware requirements
@@ -99,7 +51,7 @@ The build will create the following:
 You can check if your system is capable of creating a Kata Container by running the following:
 
 ```
-$ sudo kata-runtime kata-check
+$ sudo kata-runtime check
 ```
 
 If your system is *not* able to run Kata Containers, the previous command will error out and explain why.
@@ -242,13 +194,6 @@ Restart `systemd-journald` for the changes to take effect:
 $ sudo systemctl restart systemd-journald
 ```
 
-# Build and install Kata shim
-
-```
-$ go get -d -u github.com/kata-containers/shim
-$ cd $GOPATH/src/github.com/kata-containers/shim && make && sudo make install
-```
-
 # Create and install rootfs and initrd image
 
 ## Build a custom Kata agent - OPTIONAL
@@ -257,15 +202,25 @@ $ cd $GOPATH/src/github.com/kata-containers/shim && make && sudo make install
 >
 > - You should only do this step if you are testing with the latest version of the agent.
 
+The rust-agent is built with a static linked `musl.` To configure this:
+
 ```
-$ go get -d -u github.com/kata-containers/agent
-$ cd $GOPATH/src/github.com/kata-containers/agent && make
+rustup target add x86_64-unknown-linux-musl
+sudo ln -s /usr/bin/g++ /bin/musl-g++
+```
+
+To build the agent:
+
+```
+$ go get -d -u github.com/kata-containers/kata-containers
+$ cd $GOPATH/src/github.com/kata-containers/kata-containers/src/agent && make
 ```
 
 ## Get the osbuilder
 
 ```
-$ go get -d -u github.com/kata-containers/osbuilder
+$ go get -d -u github.com/kata-containers/kata-containers
+$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder
 ```
 
 ## Create a rootfs image
@@ -276,16 +231,16 @@ able to run the `rootfs.sh` script with `USE_DOCKER=true` as expected in
 the following example.
 
 ```
-$ export ROOTFS_DIR=${GOPATH}/src/github.com/kata-containers/osbuilder/rootfs-builder/rootfs
+$ export ROOTFS_DIR=${GOPATH}/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder/rootfs
 $ sudo rm -rf ${ROOTFS_DIR}
-$ cd $GOPATH/src/github.com/kata-containers/osbuilder/rootfs-builder
+$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder
 $ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
 ```
 You MUST choose one of `alpine`, `centos`, `clearlinux`, `debian`, `euleros`, `fedora`, `suse`, and `ubuntu` for `${distro}`. By default `seccomp` packages are not included in the rootfs image. Set `SECCOMP` to `yes` to include them.
 
 > **Note:**
 >
-> - Check the [compatibility matrix](https://github.com/kata-containers/osbuilder#platform-distro-compatibility-matrix) before creating rootfs.
+> - Check the [compatibility matrix](../tools/osbuilder/README.md#platform-distro-compatibility-matrix) before creating rootfs.
 > - You must ensure that the *default Docker runtime* is `runc` to make use of
 >   the `USE_DOCKER` variable. If that is not the case, remove the variable
 >   from the previous command. See [Checking Docker default runtime](#checking-docker-default-runtime).
@@ -297,15 +252,15 @@ You MUST choose one of `alpine`, `centos`, `clearlinux`, `debian`, `euleros`, `f
 > - You should only do this step if you are testing with the latest version of the agent.
 
 ```
-$ sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/bin ../../agent/kata-agent
-$ sudo install -o root -g root -m 0440 ../../agent/kata-agent.service ${ROOTFS_DIR}/usr/lib/systemd/system/
-$ sudo install -o root -g root -m 0440 ../../agent/kata-containers.target ${ROOTFS_DIR}/usr/lib/systemd/system/
+$ sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/usr/bin ../../../src/agent/target/x86_64-unknown-linux-musl/release/kata-agent
+$ sudo install -o root -g root -m 0440 ../../../src/agent/kata-agent.service ${ROOTFS_DIR}/usr/lib/systemd/system/
+$ sudo install -o root -g root -m 0440 ../../../src/agent/kata-containers.target ${ROOTFS_DIR}/usr/lib/systemd/system/
 ```
 
 ### Build a rootfs image
 
 ```
-$ cd $GOPATH/src/github.com/kata-containers/osbuilder/image-builder
+$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/image-builder
 $ script -fec 'sudo -E USE_DOCKER=true ./image_builder.sh ${ROOTFS_DIR}'
 ```
 
@@ -332,9 +287,9 @@ $ (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
 ## Create an initrd image - OPTIONAL
 ### Create a local rootfs for initrd image
 ```
-$ export ROOTFS_DIR="${GOPATH}/src/github.com/kata-containers/osbuilder/rootfs-builder/rootfs"
+$ export ROOTFS_DIR="${GOPATH}/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder/rootfs"
 $ sudo rm -rf ${ROOTFS_DIR}
-$ cd $GOPATH/src/github.com/kata-containers/osbuilder/rootfs-builder
+$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder
 $ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
 ```
 `AGENT_INIT` controls if the guest image uses the Kata agent as the guest `init` process. When you create an initrd image,
@@ -344,17 +299,21 @@ You MUST choose one of `alpine`, `centos`, `clearlinux`, `euleros`, and `fedora`
 
 > **Note:**
 >
-> - Check the [compatibility matrix](https://github.com/kata-containers/osbuilder#platform-distro-compatibility-matrix) before creating rootfs.
+> - Check the [compatibility matrix](../tools/osbuilder/README.md#platform-distro-compatibility-matrix) before creating rootfs.
 
-Optionally, add your custom agent binary to the rootfs with the following:
+Optionally, add your custom agent binary to the rootfs with the following commands. The default `$LIBC` used
+is `musl`, but on ppc64le and s390x, `gnu` should be used. Also, Rust refers to ppc64le as `powerpc64le`:
 ```
-$ sudo install -o root -g root -m 0550 -T ../../agent/kata-agent ${ROOTFS_DIR}/sbin/init
+$ export ARCH=$(uname -m)
+$ [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
+$ [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
+$ sudo install -o root -g root -m 0550 -T ../../../src/agent/target/${ARCH}-unknown-linux-${LIBC}/release/kata-agent ${ROOTFS_DIR}/sbin/init
 ```
 
 ### Build an initrd image
 
 ```
-$ cd $GOPATH/src/github.com/kata-containers/osbuilder/initrd-builder
+$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/initrd-builder
 $ script -fec 'sudo -E AGENT_INIT=yes USE_DOCKER=true ./initrd_builder.sh ${ROOTFS_DIR}'
 ```
 
@@ -374,31 +333,56 @@ You can build and install the guest kernel image as shown [here](../tools/packag
 
 # Install a hypervisor
 
-When setting up Kata using a [packaged installation method](install/README.md#installing-on-a-linux-system), the `qemu-lite` hypervisor is installed automatically. For other installation methods, you will need to manually install a suitable hypervisor.
+When setting up Kata using a [packaged installation method](install/README.md#installing-on-a-linux-system), the
+`QEMU` VMM is installed automatically. Cloud-Hypervisor and Firecracker VMMs are available from the [release tarballs](https://github.com/kata-containers/kata-containers/releases), as well as through [`kata-deploy`](../tools/packaging/kata-deploy/README.md).
+You may choose to manually build your VMM/hypervisor.
 
 ## Build a custom QEMU
 
-Your QEMU directory need to be prepared with source code. Alternatively, you can use the [Kata containers QEMU](https://github.com/kata-containers/qemu/tree/master) and checkout the recommended branch:
+Kata Containers makes use of upstream QEMU branch. The exact version
+and repository utilized can be found by looking at the [versions file](../versions.yaml).
 
+Find the correct version of QEMU from the versions file:
 ```
-$ go get -d github.com/kata-containers/qemu
-$ qemu_branch=$(grep qemu-lite- ${GOPATH}/src/github.com/kata-containers/runtime/versions.yaml | cut -d '"' -f2)
-$ cd ${GOPATH}/src/github.com/kata-containers/qemu
-$ git checkout -b $qemu_branch remotes/origin/$qemu_branch
-$ your_qemu_directory=${GOPATH}/src/github.com/kata-containers/qemu
+$ source ${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging/scripts/lib.sh
+$ qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version")
+$ echo ${qemu_version}
+```
+Get source from the matching branch of QEMU:
+```
+$ go get -d github.com/qemu/qemu
+$ cd ${GOPATH}/src/github.com/qemu/qemu
+$ git checkout ${qemu_version}
+$ your_qemu_directory=${GOPATH}/src/github.com/qemu/qemu
 ```
 
-To build a version of QEMU using the same options as the default `qemu-lite` version , you could use the `configure-hypervisor.sh` script:
-
+There are scripts to manage the build and packaging of QEMU. For the examples below, set your
+environment as:
+```
+$ go get -d github.com/kata-containers/kata-containers
+$ packaging_dir="${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging"
+```
+
+Kata often utilizes patches for not-yet-upstream and/or backported fixes for components,
+including QEMU. These can be found in the [packaging/QEMU directory](../tools/packaging/qemu/patches),
+and it's *recommended* that you apply them. For example, suppose that you are going to build QEMU
+version 5.2.0, do:
 ```
-$ go get -d github.com/kata-containers/packaging
 $ cd $your_qemu_directory
-$ ${GOPATH}/src/github.com/kata-containers/packaging/scripts/configure-hypervisor.sh qemu > kata.cfg
+$ $packaging_dir/scripts/apply_patches.sh $packaging_dir/qemu/patches/5.2.x/
+```
+
+To build utilizing the same options as Kata, you should make use of the `configure-hypervisor.sh` script. For example:
+```
+$ cd $your_qemu_directory
+$ $packaging_dir/scripts/configure-hypervisor.sh kata-qemu > kata.cfg
 $ eval ./configure "$(cat kata.cfg)"
 $ make -j $(nproc)
 $ sudo -E make install
 ```
 
+See the [static-build script for QEMU](../tools/packaging/static-build/qemu/build-static-qemu.sh) for a reference on how to get, setup, configure and build QEMU for Kata.
+
 ### Build a custom QEMU for aarch64/arm64 - REQUIRED
 > **Note:**
 >
@@ -412,27 +396,11 @@ $ go get -d github.com/kata-containers/tests
 $ script -fec 'sudo -E ${GOPATH}/src/github.com/kata-containers/tests/.ci/install_qemu.sh'
 ```
 
-# Run Kata Containers with Docker
-
-## Update the Docker systemd unit file
-
-```
-$ dockerUnit=$(systemctl show -p FragmentPath docker.service | cut -d "=" -f 2)
-$ unitFile=${dockerUnit:-/etc/systemd/system/docker.service.d/kata-containers.conf}
-$ test -e "$unitFile" || { sudo mkdir -p "$(dirname $unitFile)"; echo -e "[Service]\nType=simple\nExecStart=\nExecStart=/usr/bin/dockerd -D --default-runtime runc" | sudo tee "$unitFile"; }
-$ grep -q "kata-runtime=" $unitFile || sudo sed -i 's!^\(ExecStart=[^$].*$\)!\1 --add-runtime kata-runtime=/usr/local/bin/kata-runtime!g' "$unitFile"
-$ sudo systemctl daemon-reload
-$ sudo systemctl restart docker
-```
-
-## Create a container using Kata
-
-```
-$ sudo docker run -ti --runtime kata-runtime busybox sh
-```
+# Run Kata Containers with Containerd
+Refer to the [How to use Kata Containers and Containerd](how-to/containerd-kata.md) how-to guide.
 
 # Run Kata Containers with Kubernetes
-Refer to to the [Run Kata Containers with Kubernetes](how-to/run-kata-with-k8s.md) how-to guide.
+Refer to the [Run Kata Containers with Kubernetes](how-to/run-kata-with-k8s.md) how-to guide.
 
 # Troubleshoot Kata Containers
 
@@ -449,21 +417,9 @@ script and paste its output directly into a
 > [runtime](../src/runtime) repository.
 
 To perform analysis on Kata logs, use the
-[`kata-log-parser`](https://github.com/kata-containers/tests/tree/master/cmd/log-parser)
+[`kata-log-parser`](https://github.com/kata-containers/tests/tree/main/cmd/log-parser)
 tool, which can convert the logs into formats (e.g. JSON, TOML, XML, and YAML).
 
-To obtain a full backtrace for the agent, proxy, runtime, or shim send the
-`SIGUSR1` signal to the process ID of the component. The component will send a
-backtrace to the system log on the host system and continue to run without
-interruption.
-
-For example, to obtain a backtrace for `kata-proxy`:
-
-```
-$ sudo kill -USR1 $kata_proxy_pid
-$ sudo journalctl -t kata-proxy
-```
-
 See [Set up a debug console](#set-up-a-debug-console).
 
 # Appendices
@@ -473,9 +429,59 @@ See [Set up a debug console](#set-up-a-debug-console).
 ```
 $ sudo docker info 2>/dev/null | grep -i "default runtime" | cut -d: -f2- | grep -q runc  && echo "SUCCESS" || echo "ERROR: Incorrect default Docker runtime"
 ```
-
 ## Set up a debug console
 
+Kata containers provides two ways to connect to the guest. One is using traditional login service, which needs additional works. In contrast the simple debug console is easy to setup.
+
+### Simple debug console setup
+
+Kata Containers 2.0 supports a shell simulated *console* for quick debug purpose. This approach uses VSOCK to
+connect to the shell running inside the guest which the agent starts. This method only requires the guest image to
+contain either `/bin/sh` or `/bin/bash`.
+
+#### Enable agent debug console
+
+Enable debug_console_enabled in the `configuration.toml` configuration file:
+
+```
+[agent.kata]
+debug_console_enabled = true
+```
+
+This will pass `agent.debug_console agent.debug_console_vport=1026` to agent as kernel parameters, and sandboxes created using this parameters will start a shell in guest if new connection is accept from VSOCK.
+
+#### Start `kata-monitor` - ONLY NEEDED FOR 2.0.x
+
+For Kata Containers `2.0.x` releases, the `kata-runtime exec` command depends on the`kata-monitor` running, in order to get the sandbox's `vsock` address to connect to. Thus, first start the `kata-monitor` process.
+
+```
+$ sudo kata-monitor
+```
+
+`kata-monitor` will serve at `localhost:8090` by default.
+
+#### Connect to debug console
+
+Command `kata-runtime exec` is used to connect to the debug console.
+
+```
+$ kata-runtime exec 1a9ab65be63b8b03dfd0c75036d27f0ed09eab38abb45337fea83acd3cd7bacd
+bash-4.2# id
+uid=0(root) gid=0(root) groups=0(root)
+bash-4.2# pwd
+/
+bash-4.2# exit
+exit
+```
+
+`kata-runtime exec` has a command-line option `runtime-namespace`, which is used to specify under which [runtime namespace](https://github.com/containerd/containerd/blob/master/docs/namespaces.md) the particular pod was created. By default, it is set to `k8s.io` and works for containerd when configured
+ with Kubernetes. For CRI-O, the namespace should set to `default` explicitly. This should not be confused with [Kubernetes namespaces](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/).
+For other CRI-runtimes and configurations, you may need to set the namespace utilizing the `runtime-namespace` option.
+
+If you want to access guest OS through a traditional way, see [Traditional debug console setup)](#traditional-debug-console-setup).
+
+### Traditional debug console setup
+
 By default you cannot login to a virtual machine, since this can be sensitive
 from a security perspective. Also, allowing logins would require additional
 packages in the rootfs, which would increase the size of the image used to
@@ -486,12 +492,12 @@ the following steps (using rootfs or initrd image).
 
 > **Note:** The following debug console instructions assume a systemd-based guest
 > O/S image. This means you must create a rootfs for a distro that supports systemd.
-> Currently, all distros supported by [osbuilder](https://github.com/kata-containers/osbuilder) support systemd
+> Currently, all distros supported by [osbuilder](../tools/osbuilder) support systemd
 > except for Alpine Linux.
 >
 > Look for `INIT_PROCESS=systemd` in the `config.sh` osbuilder rootfs config file
 > to verify an osbuilder distro supports systemd for the distro you want to build rootfs for.
-> For an example, see the [Clear Linux config.sh file](https://github.com/kata-containers/osbuilder/blob/master/rootfs-builder/clearlinux/config.sh).
+> For an example, see the [Clear Linux config.sh file](../tools/osbuilder/rootfs-builder/clearlinux/config.sh).
 >
 > For a non-systemd-based distro, create an equivalent system
 > service using that distro’s init system syntax. Alternatively, you can build a distro
@@ -501,7 +507,7 @@ the following steps (using rootfs or initrd image).
 >
 > Once these steps are taken you can connect to the virtual machine using the [debug console](Developer-Guide.md#connect-to-the-virtual-machine-using-the-debug-console).
 
-### Create a custom image containing a shell
+#### Create a custom image containing a shell
 
 To login to a virtual machine, you must
 [create a custom rootfs](#create-a-rootfs-image) or [custom initrd](#create-an-initrd-image---optional)
@@ -511,46 +517,17 @@ an additional `coreutils` package.
 For example using CentOS:
 
 ```
-$ cd $GOPATH/src/github.com/kata-containers/osbuilder/rootfs-builder
-$ export ROOTFS_DIR=${GOPATH}/src/github.com/kata-containers/osbuilder/rootfs-builder/rootfs
+$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder
+$ export ROOTFS_DIR=${GOPATH}/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder/rootfs
 $ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true EXTRA_PKGS="bash coreutils" ./rootfs.sh centos'
 ```
 
-### Create a debug systemd service
-
-Create the service file that starts the shell in the rootfs directory:
-
-```
-$ cat <<EOT | sudo tee ${ROOTFS_DIR}/lib/systemd/system/kata-debug.service
-[Unit]
-Description=Kata Containers debug console
-
-[Service]
-Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-StandardInput=tty
-StandardOutput=tty
-# Must be disabled to allow the job to access the real console
-PrivateDevices=no
-Type=simple
-ExecStart=/bin/bash
-Restart=always
-EOT
-```
-
-**Note**: You might need to adjust the `ExecStart=` path.
-
-Add a dependency to start the debug console:
-
-```
-$ sudo sed -i '$a Requires=kata-debug.service' ${ROOTFS_DIR}/lib/systemd/system/kata-containers.target
-```
-
-### Build the debug image
+#### Build the debug image
 
 Follow the instructions in the [Build a rootfs image](#build-a-rootfs-image)
 section when using rootfs, or when using initrd, complete the steps in the [Build an initrd image](#build-an-initrd-image) section.
 
-### Configure runtime for custom debug image
+#### Configure runtime for custom debug image
 
 Install the image:
 
@@ -575,44 +552,85 @@ $ (cd /usr/share/kata-containers && sudo ln -sf "$name" kata-containers.img)
 **Note**: You should take care to undo this change after you finish debugging
 to avoid all subsequently created containers from using the debug image.
 
-### Ensure debug options are valid
+#### Create a container
 
-For the debug console to work, you **must** ensure that proxy debug is
-**disabled** in the configuration file. If proxy debug is enabled, you will
-not see any output when you connect to the virtual machine:
+Create a container as normal. For example using `crictl`:
 
 ```
-$ sudo mkdir -p /etc/kata-containers/
-$ sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers
-$ sudo awk '{if (/^\[proxy\.kata\]/) {got=1}; if (got == 1 && /^.*enable_debug/) {print "#enable_debug = true"; got=0; next; } else {print}}' /etc/kata-containers/configuration.toml > /tmp/configuration.toml
-$ sudo install -o root -g root -m 0640 /tmp/configuration.toml /etc/kata-containers/
+$ sudo crictl run -r kata container.yaml pod.yaml
 ```
 
-### Create a container
+#### Connect to the virtual machine using the debug console
 
-Create a container as normal. For example using Docker:
+The steps required to enable debug console for QEMU slightly differ with
+those for firecracker / cloud-hypervisor.
+ 
+##### Enabling debug console for QEMU
+
+Add `agent.debug_console` to the guest kernel command line to allow the agent process to start a debug console. 
 
 ```
-$ sudo docker run -ti busybox sh
+$ sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.debug_console"/g' "${kata_configuration_file}"
 ```
 
-### Connect to the virtual machine using the debug console
+Here `kata_configuration_file` could point to `/etc/kata-containers/configuration.toml` 
+or `/usr/share/defaults/kata-containers/configuration.toml`
+or `/opt/kata/share/defaults/kata-containers/configuration-{hypervisor}.toml`, if
+you installed Kata Containers using `kata-deploy`.
 
+##### Enabling debug console for cloud-hypervisor / firecracker
+
+Slightly different configuration is required in case of firecracker and cloud hypervisor. 
+Firecracker and cloud-hypervisor don't have a UNIX socket connected to `/dev/console`. 
+Hence, the kernel command line option `agent.debug_console` will not work for them. 
+These hypervisors support `hybrid vsocks`,  which can be used for communication
+between the host and the guest. The kernel command line option `agent.debug_console_vport`
+ was added to allow developers specify on which `vsock` port the debugging console should be connected.
+
+
+Add the parameter `agent.debug_console_vport=1026` to the kernel command line
+as shown below:
 ```
-$ id=$(sudo docker ps -q --no-trunc)
-$ console="/var/run/vc/vm/${id}/console.sock"
-$ sudo socat "stdin,raw,echo=0,escape=0x11" "unix-connect:${console}"
+sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.debug_console_vport=1026"/g' "${kata_configuration_file}"
+```
+
+> **Note** Ports 1024 and 1025 are reserved for communication with the agent
+> and gathering of agent logs respectively. 
+
+##### Connecting to the debug console
+
+Next, connect to the debug console. The VSOCKS paths vary slightly between each
+VMM solution.
+
+In case of cloud-hypervisor, connect to the `vsock` as shown:
+```
+$ sudo su -c 'cd /var/run/vc/vm/${sandbox_id}/root/ && socat stdin unix-connect:clh.sock'
+CONNECT 1026
+```
+
+**Note**: You need to type `CONNECT 1026` and press `RETURN` key after entering the `socat` command.
+
+For firecracker, connect to the `hvsock` as shown:
+```
+$ sudo su -c 'cd /var/run/vc/firecracker/${sandbox_id}/root/ && socat stdin unix-connect:kata.hvsock'
+CONNECT 1026
 ```
 
 **Note**: You need to press the `RETURN` key to see the shell prompt.
 
+
+For QEMU, connect to the `vsock` as shown:
+```
+$ sudo su -c 'cd /var/run/vc/vm/${sandbox_id} && socat "stdin,raw,echo=0,escape=0x11" "unix-connect:console.sock"'
+```
+
 To disconnect from the virtual machine, type `CONTROL+q` (hold down the
 `CONTROL` key and press `q`).
 
-### Obtain details of the image
+## Obtain details of the image
 
 If the image is created using
-[osbuilder](https://github.com/kata-containers/osbuilder), the following YAML
+[osbuilder](../tools/osbuilder), the following YAML
 file exists and contains details of the image and how it was created:
 
 ```
@@ -629,54 +647,22 @@ command inside the container to view the kernel boot logs.
 If however you are unable to `exec` into the container, you can enable some debug
 options to have the kernel boot messages logged into the system journal.
 
-Which debug options you enable depends on if you are using the hypervisor `vsock` mode
-or not, as defined by the `use_vsock` setting in the `[hypervisor.qemu]` section of
-the configuration file. The following details the settings:
-
-- For `use_vsock = false`:
-    - Set `enable_debug = true` in both the `[hypervisor.qemu]` and `[proxy.kata]` sections
-- For `use_vsock = true`:
-    - Set `enable_debug = true` in both the `[hypervisor.qemu]` and `[shim.kata]` sections
+- Set `enable_debug = true` in the `[hypervisor.qemu]` and `[runtime]` sections
 
 For generic information on enabling debug in the configuration file, see the
 [Enable full debug](#enable-full-debug) section.
 
-The kernel boot messages will appear in the `kata-proxy` or `kata-shim` log appropriately,
+The kernel boot messages will appear in the `containerd` or `CRI-O` log appropriately,
 such as:
 
 ```bash
-$ sudo journalctl -t kata-proxy
+$ sudo journalctl -t containerd
 -- Logs begin at Thu 2020-02-13 16:20:40 UTC, end at Thu 2020-02-13 16:30:23 UTC. --
 ...
-Feb 13 16:20:56 minikube kata-proxy[17371]: time="2020-02-13T16:20:56.608714324Z" level=info msg="[    1.418768] brd: module loaded\n" name=kata-proxy pid=17371 sandbox=a13ffb2b9b5a66f7787bdae9a427fa954a4d21ec4031d0179eee2573986a8a6e source=agent
-Feb 13 16:20:56 minikube kata-proxy[17371]: time="2020-02-13T16:20:56.628493231Z" level=info msg="[    1.438612] loop: module loaded\n" name=kata-proxy pid=17371 sandbox=a13ffb2b9b5a66f7787bdae9a427fa954a4d21ec4031d0179eee2573986a8a6e source=agent
-Feb 13 16:20:56 minikube kata-proxy[17371]: time="2020-02-13T16:20:56.67707956Z" level=info msg="[    1.487165]  pmem0: p1\n" name=kata-proxy pid=17371 sandbox=a13ffb2b9b5a66f7787bdae9a427fa954a4d21ec4031d0179eee2573986a8a6e source=agent
+time="2020-09-15T14:56:23.095113803+08:00" level=debug msg="reading guest console" console-protocol=unix console-url=/run/vc/vm/ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791/console.sock pid=107642 sandbox=ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791 source=virtcontainers subsystem=sandbox vmconsole="[    0.395399] brd: module loaded"
+time="2020-09-15T14:56:23.102633107+08:00" level=debug msg="reading guest console" console-protocol=unix console-url=/run/vc/vm/ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791/console.sock pid=107642 sandbox=ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791 source=virtcontainers subsystem=sandbox vmconsole="[    0.402845] random: fast init done"
+time="2020-09-15T14:56:23.103125469+08:00" level=debug msg="reading guest console" console-protocol=unix console-url=/run/vc/vm/ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791/console.sock pid=107642 sandbox=ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791 source=virtcontainers subsystem=sandbox vmconsole="[    0.403544] random: crng init done"
+time="2020-09-15T14:56:23.105268162+08:00" level=debug msg="reading guest console" console-protocol=unix console-url=/run/vc/vm/ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791/console.sock pid=107642 sandbox=ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791 source=virtcontainers subsystem=sandbox vmconsole="[    0.405599] loop: module loaded"
+time="2020-09-15T14:56:23.121121598+08:00" level=debug msg="reading guest console" console-protocol=unix console-url=/run/vc/vm/ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791/console.sock pid=107642 sandbox=ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791 source=virtcontainers subsystem=sandbox vmconsole="[    0.421324] memmap_init_zone_device initialised 32768 pages in 12ms"
 ...
 ```
-
-## Running standalone
-
-It is possible to start the runtime without a container manager. This is
-mostly useful for testing and debugging purposes.
-
-### Create an OCI bundle
-
-To build an
-[OCI bundle](https://github.com/opencontainers/runtime-spec/blob/master/bundle.md),
-required by the runtime:
-
-```
-$ bundle="/tmp/bundle"
-$ rootfs="$bundle/rootfs"
-$ mkdir -p "$rootfs" && (cd "$bundle" && kata-runtime spec)
-$ sudo docker export $(sudo docker create busybox) | tar -C "$rootfs" -xvf -
-```
-
-### Launch the runtime to create a container
-
-Run the runtime standalone by providing it with the path to the
-previously-created [OCI bundle](#create-an-oci-bundle):
-
-```
-$ sudo kata-runtime --log=/dev/stdout run --bundle "$bundle" foo
-```

docs/Documentation-Requirements.md

@@ -1,16 +1,3 @@
-* [Introduction](#introduction)
-* [General requirements](#general-requirements)
-* [Linking advice](#linking-advice)
-* [Notes](#notes)
-* [Warnings and other admonitions](#warnings-and-other-admonitions)
-* [Files and command names](#files-and-command-names)
-* [Code blocks](#code-blocks)
-* [Images](#images)
-* [Spelling](#spelling)
-* [Names](#names)
-* [Version numbers](#version-numbers)
-* [The apostrophe](#the-apostrophe)
-
 # Introduction
 
 This document outlines the requirements for all documentation in the [Kata
@@ -23,10 +10,6 @@ All documents must:
 - Be written in simple English.
 - Be written in [GitHub Flavored Markdown](https://github.github.com/gfm) format.
 - Have a `.md` file extension.
-- Include a TOC (table of contents) at the top of the document with links to
-  all heading sections. We recommend using the
-  [`kata-check-markdown`](https://github.com/kata-containers/tests/tree/master/cmd/check-markdown)
-  tool to generate the TOC.
 - Be linked to from another document in the same repository.
 
   Although GitHub allows navigation of the entire repository, it should be
@@ -43,6 +26,10 @@ All documents must:
   which can then execute the commands specified to ensure the instructions are
   correct. This avoids documents becoming out of date over time.
 
+> **Note:**
+>
+> Do not add a table of contents (TOC) since GitHub will auto-generate one.
+
 # Linking advice
 
 Linking between documents is strongly encouraged to help users and developers
@@ -118,7 +105,7 @@ This section lists requirements for displaying commands and command output.
 The requirements must be adhered to since documentation containing code blocks
 is validated by the CI system, which executes the command blocks with the help
 of the
-[doc-to-script](https://github.com/kata-containers/tests/tree/master/.ci/kata-doc-to-script.sh)
+[doc-to-script](https://github.com/kata-containers/tests/tree/main/.ci/kata-doc-to-script.sh)
 utility.
 
 - If a document includes commands the user should run, they **MUST** be shown
@@ -202,7 +189,7 @@ and compare them with standard tools (e.g. `diff(1)`).
 
 Since this project uses a number of terms not found in conventional
 dictionaries, we have a
-[spell checking tool](https://github.com/kata-containers/tests/tree/master/cmd/check-spelling)
+[spell checking tool](https://github.com/kata-containers/tests/tree/main/cmd/check-spelling)
 that checks both dictionary words and the additional terms we use.
 
 Run the spell checking tool on your document before raising a PR to ensure it

docs/Licensing-strategy.md

@@ -1,9 +1,5 @@
 # Licensing strategy
 
-* [Project License](#project-license)
-* [License file](#license-file)
-* [License for individual files](#license-for-individual-files)      
-
 ## Project License
 
 The license for the [Kata Containers](https://github.com/kata-containers)
@@ -22,4 +18,4 @@ licensing and allows automated tooling to check the license of individual
 files.
 
 This SPDX licence identifier requirement is enforced by the
-[CI (Continuous Integration) system](https://github.com/kata-containers/tests/blob/master/.ci/static-checks.sh).
+[CI (Continuous Integration) system](https://github.com/kata-containers/tests/blob/main/.ci/static-checks.sh).

docs/Limitations.md

@@ -1,33 +1,3 @@
-* [Overview](#overview)
-* [Definition of a limitation](#definition-of-a-limitation)
-* [Scope](#scope)
-* [Contributing](#contributing)
-* [Pending items](#pending-items)
-    * [Runtime commands](#runtime-commands)
-        * [checkpoint and restore](#checkpoint-and-restore)
-        * [events command](#events-command)
-        * [update command](#update-command)
-    * [Networking](#networking)
-        * [Docker swarm and compose support](#docker-swarm-and-compose-support)
-    * [Resource management](#resource-management)
-        * [docker run and shared memory](#docker-run-and-shared-memory)
-        * [docker run and sysctl](#docker-run-and-sysctl)
-    * [Docker daemon features](#docker-daemon-features)
-        * [SELinux support](#selinux-support)
-* [Architectural limitations](#architectural-limitations)
-    * [Networking limitations](#networking-limitations)
-        * [Support for joining an existing VM network](#support-for-joining-an-existing-vm-network)
-        * [docker --net=host](#docker---nethost)
-        * [docker run --link](#docker-run---link)
-    * [Host resource sharing](#host-resource-sharing)
-        * [docker run --privileged](#docker-run---privileged)
-* [Miscellaneous](#miscellaneous)
-    * [Docker --security-opt option partially supported](#docker---security-opt-option-partially-supported)
-* [Appendices](#appendices)
-    * [The constraints challenge](#the-constraints-challenge)
-
----
-
 # Overview
 
 A [Kata Container](https://github.com/kata-containers) utilizes a Virtual Machine (VM) to enhance security and
@@ -92,7 +62,9 @@ This section lists items that might be possible to fix.
 ### checkpoint and restore
 
 The runtime does not provide `checkpoint` and `restore` commands. There
-are discussions about using VM save and restore to give [`criu`](https://github.com/checkpoint-restore/criu)-like functionality, which might provide a solution.
+are discussions about using VM save and restore to give us a
+`[criu](https://github.com/checkpoint-restore/criu)`-like functionality,
+which might provide a solution.
 
 Note that the OCI standard does not specify `checkpoint` and `restore`
 commands.
@@ -216,6 +188,17 @@ Equivalent functionality can be achieved with the newer docker networking comman
 See more documentation at
 [docs.docker.com](https://docs.docker.com/engine/userguide/networking/default_network/dockerlinks/).
 
+## Storage limitations
+
+### Kubernetes `volumeMounts.subPaths`
+
+Kubernetes `volumeMount.subPath` is not supported by Kata Containers at the
+moment.
+
+See [this issue](https://github.com/kata-containers/runtime/issues/2812) for more details.
+[Another issue](https://github.com/kata-containers/kata-containers/issues/1728) focuses on the case of `emptyDir`.
+
+
 ## Host resource sharing
 
 ### docker run --privileged
@@ -224,7 +207,7 @@ Privileged support in Kata is essentially different from `runc` containers.
 Kata does support `docker run --privileged` command, but in this case full access
 to the guest VM is provided in addition to some host access.
 
-The container runs with elevated capabilities within the guest and is granted 
+The container runs with elevated capabilities within the guest and is granted
 access to guest devices instead of the host devices.
 This is also true with using `securityContext privileged=true` with Kubernetes.
 

docs/README.md

@@ -1,16 +1,5 @@
 # Documentation
 
-* [Getting Started](#getting-started)
-* [More User Guides](#more-user-guides)
-* [Kata Use-Cases](#kata-use-cases)
-* [Developer Guide](#developer-guide)
-    * [Design and Implementations](#design-and-implementations)
-    * [How to Contribute](#how-to-contribute)
-    * [Code Licensing](#code-licensing)
-    * [The Release Process](#the-release-process)
-* [Help Improving the Documents](#help-improving-the-documents)
-* [Website Changes](#website-changes)
-
 The [Kata Containers](https://github.com/kata-containers)
 documentation repository hosts overall system documentation, with information
 common to multiple components.
@@ -40,6 +29,7 @@ See the [howto documentation](how-to).
 * [Intel QAT with Kata](./use-cases/using-Intel-QAT-and-kata.md)
 * [VPP with Kata](./use-cases/using-vpp-and-kata.md)
 * [SPDK vhost-user with Kata](./use-cases/using-SPDK-vhostuser-and-kata.md)
+* [Intel SGX with Kata](./use-cases/using-Intel-SGX-and-kata.md)
 
 ## Developer Guide
 
@@ -48,13 +38,14 @@ Documents that help to understand and contribute to Kata Containers.
 ### Design and Implementations
 
 * [Kata Containers Architecture](design/architecture.md): Architectural overview of Kata Containers
+* [Kata Containers E2E Flow](design/end-to-end-flow.md): The entire end-to-end flow of Kata Containers
 * [Kata Containers design](./design/README.md): More Kata Containers design documents
 
 ### How to Contribute
 
 * [Developer Guide](Developer-Guide.md): Setup the Kata Containers developing environments
 * [How to contribute to Kata Containers](https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md)
-* [Code of Conduct](CODE_OF_CONDUCT.md)
+* [Code of Conduct](../CODE_OF_CONDUCT.md)
 
 ### Code Licensing
 

docs/Release-Process.md

@@ -1,26 +1,10 @@
-
 # How to do a Kata Containers Release
   This document lists the tasks required to create a Kata Release.
 
-<!-- TOC START min:1 max:3 link:true asterisk:false update:true -->
-- [How to do a Kata Containers Release](#how-to-do-a-kata-containers-release)
-  - [Requirements](#requirements)
-  - [Release Process](#release-process)
-    - [Bump all Kata repositories](#bump-all-kata-repositories)
-    - [Merge all bump version Pull requests](#merge-all-bump-version-pull-requests)
-    - [Tag all Kata repositories](#tag-all-kata-repositories)
-    - [Check Git-hub Actions](#check-git-hub-actions)
-    - [Create OBS Packages](#create-obs-packages)
-    - [Create release notes](#create-release-notes)
-    - [Announce the release](#announce-the-release)
-<!-- TOC END -->
-
-
 ## Requirements
 
 - [hub](https://github.com/github/hub)
-
-- OBS account with permissions on [`/home:katacontainers`](https://build.opensuse.org/project/subprojects/home:katacontainers)
+  * Using an [application token](https://github.com/settings/tokens) is required for hub.
 
 - GitHub permissions to push tags and create releases in Kata repositories.
 
@@ -31,23 +15,36 @@
 
 ## Release Process
 
+
 ### Bump all Kata repositories
 
-  - We have set up a Jenkins job to bump the version in the `VERSION` file in all Kata repositories. Go to the [Jenkins bump-job page](http://jenkins.katacontainers.io/job/release/build) to trigger a new job.
-  - Start a new job with variables for the job passed as:
-     - `BRANCH=<the-branch-you-want-to-bump>`
-     - `NEW_VERSION=<the-new-kata-version>`
-
-     For example, in the case where you want to make a patch release `1.10.2`, the variable `NEW_VERSION` should be `1.10.2` and `BRANCH` should point to  `stable-1.10`. In case of an alpha or release candidate release, `BRANCH` should point to `master` branch.
-
-  Alternatively, you can also bump the repositories using a script in the Kata packaging repo
+  Bump the repositories using a script in the Kata packaging repo, where:
+  - `BRANCH=<the-branch-you-want-to-bump>`
+  - `NEW_VERSION=<the-new-kata-version>`
   ```
-  $ cd ${GOPATH}/src/github.com/kata-containers/packaging/release
+  $ cd ${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging/release
   $ export NEW_VERSION=<the-new-kata-version>
   $ export BRANCH=<the-branch-you-want-to-bump>
   $ ./update-repository-version.sh -p "$NEW_VERSION" "$BRANCH"
   ```
 
+### Point tests repository to stable branch
+
+  If you create a new stable branch, i.e. if your release changes a major or minor version number (not a patch release), then
+  you should modify the `tests` repository to point to that newly created stable branch and not the `main` branch.
+  The objective is that changes in the CI on the main branch will not impact the stable branch.
+
+  In the test directory, change references the main branch in:
+  * `README.md`
+  * `versions.yaml`
+  * `cmd/github-labels/labels.yaml.in`
+  * `cmd/pmemctl/pmemctl.sh`
+  * `.ci/lib.sh`
+  * `.ci/static-checks.sh`
+
+  See the commits in [the corresponding PR for stable-2.1](https://github.com/kata-containers/tests/pull/3504) for an example of the changes.
+
+
 ### Merge all bump version Pull requests
 
   - The above step will create a GitHub pull request in the Kata projects. Trigger the CI using `/test` command on each bump Pull request.
@@ -57,9 +54,9 @@
 ### Tag all Kata repositories
 
   Once all the pull requests to bump versions in all Kata repositories are merged,
-  tag all the repositories as shown below.  
+  tag all the repositories as shown below.
   ```
-  $ cd ${GOPATH}/src/github.com/kata-containers/packaging/release
+  $ cd ${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging/release
   $ git checkout  <kata-branch-to-release>
   $ git pull
   $ ./tag_repos.sh -p -b "$BRANCH" tag
@@ -67,37 +64,10 @@
 
 ### Check Git-hub Actions
 
-  We make use of [GitHub actions](https://github.com/features/actions) in this [file](https://github.com/kata-containers/kata-containers/blob/master/.github/workflows/main.yaml) in the `kata-containers/kata-containers` repository to build and upload release artifacts. This action is auto triggered with the above step when a new tag is pushed to the `kata-containers/kata-conatiners` repository.
+  We make use of [GitHub actions](https://github.com/features/actions) in this [file](https://github.com/kata-containers/kata-containers/blob/main/.github/workflows/main.yaml) in the `kata-containers/kata-containers` repository to build and upload release artifacts. This action is auto triggered with the above step when a new tag is pushed to the `kata-containers/kata-containers` repository.
 
   Check the [actions status page](https://github.com/kata-containers/kata-containers/actions) to verify all steps in the actions workflow have completed successfully. On success, a static tarball containing Kata release artifacts will be uploaded to the [Release page](https://github.com/kata-containers/kata-containers/releases).
 
-### Create OBS Packages
-
-  - We have set up an [Azure Pipelines](https://azure.microsoft.com/en-us/services/devops/pipelines/) job
-  to trigger generation of Kata packages in [OBS](https://build.opensuse.org/).
-  Go to the [Azure Pipelines job that creates OBS packages](https://dev.azure.com/kata-containers/release-process/_release?_a=releases&view=mine&definitionId=1).
-  - Click on "Create release" (blue button, at top right corner).
-    It should prompt you for variables to be passed to the release job. They should look like:
-
-    ```
-    BRANCH="the-kata-branch-that-is-release"
-    BUILD_HEAD=false
-    OBS_BRANCH="the-kata-branch-that-is-release"
-    ```
-    Note: If the release is `Alpha` , `Beta` , or `RC` (that is part of a `master` release), please use `OBS_BRANCH=master`.
-
-    The above step shall create OBS packages for Kata for various distributions that Kata supports and test them as well.
-  - Verify that the packages have built successfully by checking the [Kata OBS  project page](https://build.opensuse.org/project/subprojects/home:katacontainers).
-  - Make sure packages work correctly. This can be done manually or via the [package testing pipeline](http://jenkins.katacontainers.io/job/package-release-testing).
-    You have to make sure the packages are already published by OBS before this step.
-    It should prompt you for variables to be passed to the pipeline:
-
-    ```
-    BRANCH="<kata-branch-to-release>"
-    NEW_VERSION=<the-version-you-expect-to-be-packaged|latest>
-    ```
-    Note: `latest` will verify that a package provides the latest Kata tag in that branch.
-
 ### Create release notes
 
   We have a script in place in the packaging repository to create release notes that include a short-log of the commits across Kata components.
@@ -105,12 +75,12 @@
   Run the script as shown below:
 
   ```
-  $ cd ${GOPATH}/src/github.com/kata-containers/packaging/release
+  $ cd ${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging/release
   # Note: OLD_VERSION is where the script should start to get changes.
-  $ ./runtime-release-notes.sh ${OLD_VERSION} ${NEW_VERSION} > notes.md
+  $ ./release-notes.sh ${OLD_VERSION} ${NEW_VERSION} > notes.md
   # Edit the `notes.md` file to review and make any changes to the release notes.
-  # Add the release notes in GitHub runtime.
-  $ hub -C "${GOPATH}/src/github.com/kata-containers/runtime" release edit -F notes.md "${NEW_VERSION}"
+  # Add the release notes in the project's GitHub.
+  $ hub release edit -F notes.md "${NEW_VERSION}"
   ```
 
 ### Announce the release

docs/Stable-Branch-Strategy.md

@@ -32,16 +32,16 @@ provides additional information regarding release `99.123.77` in the previous ex
   changing the existing behavior*.
 
 - When `MAJOR` increases, the new release adds **new features, bug fixes, or
-  both** and which *changes the behavior from the previous release* (incompatible with previous releases).
+  both** and which **changes the behavior from the previous release** (incompatible with previous releases).
 
   A major release will also likely require a change of the container manager version used, 
-for example Docker\*. Please refer to the release notes for further details.
+for example Containerd or CRI-O. Please refer to the release notes for further details.
 
 ## Release Strategy
 
 Any new features added since the last release will be available in the next minor
 release. These will include bug fixes as well. To facilitate a stable user environment, 
-Kata provides stable branch-based releases and a master branch release.
+Kata provides stable branch-based releases and a main branch release.
 
 ## Stable branch patch criteria
 
@@ -49,9 +49,10 @@ No new features should be introduced to stable branches.  This is intended to li
 providing only bug and security fixes.
 
 ## Branch Management
-Kata Containers will maintain two stable release branches in addition to the master branch.
-Once a new MAJOR or MINOR release is created from master, a new stable branch is created for
-the prior MAJOR or MINOR release and the older stable branch is no longer maintained. End of
+Kata Containers will maintain **one** stable release branch, in addition to the main branch, for
+each active major release.
+Once a new MAJOR or MINOR release is created from main, a new stable branch is created for
+the prior MAJOR or MINOR release and the previous stable branch is no longer maintained. End of
 maintenance for a branch is announced on the Kata Containers mailing list.  Users can determine
 the version currently installed by running `kata-runtime kata-env`. It is recommended to use the
 latest stable branch available.
@@ -61,59 +62,59 @@ A couple of examples follow to help clarify this process.
 ### New bug fix introduced
 
 A bug fix is submitted against the runtime which does not introduce new inter-component dependencies.
-This fix is applied to both the master and stable branches, and there is no need to create a new
+This fix is applied to both the main and stable branches, and there is no need to create a new
 stable branch.
 
 | Branch | Original version | New version |
 |--|--|--|
-| `master` | `1.3.0-rc0` | `1.3.0-rc1` |
-| `stable-1.2` | `1.2.0` | `1.2.1` |
-| `stable-1.1` | `1.1.2` | `1.1.3` |
+| `main` | `2.3.0-rc0` | `2.3.0-rc1` |
+| `stable-2.2` | `2.2.0` | `2.2.1` |
+| `stable-2.1` | (unmaintained) | (unmaintained) |
 
 
 ### New release made feature or change adding new inter-component dependency
 
 A new feature is introduced, which adds a new inter-component dependency. In this case a new stable
-branch is created (stable-1.3) starting from master and the older stable branch (stable-1.1)
+branch is created (stable-2.3) starting from main and the previous stable branch (stable-2.2)
 is dropped from maintenance.
 
 
 | Branch | Original version | New version |
 |--|--|--|
-| `master` | `1.3.0-rc1` | `1.3.0` |
-| `stable-1.3` | N/A| `1.3.0` |
-| `stable-1.2` | `1.2.1` | `1.2.2` |
-| `stable-1.1` | `1.1.3` | (unmaintained) |
+| `main` | `2.3.0-rc1` | `2.3.0` |
+| `stable-2.3` | N/A| `2.3.0` |
+| `stable-2.2` | `2.2.1` | (unmaintained) |
+| `stable-2.1` | (unmaintained) | (unmaintained) |
 
-Note, the stable-1.1 branch will still exist with tag 1.1.3, but under current plans it is
-not maintained further. The next tag applied to master will be 1.4.0-alpha0. We would then
+Note, the stable-2.2 branch will still exist with tag 2.2.1, but under current plans it is
+not maintained further. The next tag applied to main will be 2.4.0-alpha0. We would then
 create a couple of alpha releases gathering features targeted for that particular release (in
-this case 1.4.0), followed by a release candidate. The release candidate marks a feature freeze.
+this case 2.4.0), followed by a release candidate. The release candidate marks a feature freeze.
 A new stable branch is created for the release candidate. Only bug fixes and any security issues
-are added to the branch going forward until release 1.4.0 is made.
+are added to the branch going forward until release 2.4.0 is made.
    
 ## Backporting Process 
 
-Development that occurs against the master branch and applicable code commits should also be submitted
+Development that occurs against the main branch and applicable code commits should also be submitted
 against the stable branches. Some guidelines for this process follow::
   1. Only bug and security fixes which do not introduce inter-component dependencies are
  candidates for stable branches. These PRs should be marked with "bug" in GitHub.
-  2. Once a PR is created against master which meets requirement of (1), a comparable one
+  2. Once a PR is created against main which meets requirement of (1), a comparable one
  should also be submitted against the stable branches. It is the responsibility of the submitter
  to apply their pull request against stable, and it is the responsibility of the
  reviewers to help identify stable-candidate pull requests.
  
 ## Continuous Integration Testing
 
-The test repository is forked to create stable branches from master. Full CI
-runs on each stable and master PR using its respective tests repository branch.
+The test repository is forked to create stable branches from main. Full CI
+runs on each stable and main PR using its respective tests repository branch.
 
 ### An alternative method for CI testing:
 
-Ideally, the continuous integration infrastructure will run the same test suite on both master
+Ideally, the continuous integration infrastructure will run the same test suite on both main
 and the stable branches.  When tests are modified or new feature tests are introduced, explicit
 logic should exist within the testing CI to make sure only applicable tests are executed against
-stable and master. While this is not in place currently, it should be considered in the long term.
+stable and main. While this is not in place currently, it should be considered in the long term.
 
 ## Release Management
 
@@ -121,7 +122,7 @@ stable and master. While this is not in place currently, it should be considered
 
 Releases are made every three weeks, which include a GitHub release as
 well as binary packages. These patch releases are made for both stable branches, and a "release candidate"
-for the next `MAJOR` or `MINOR` is created from master. If there are no changes across all the repositories, no
+for the next `MAJOR` or `MINOR` is created from main. If there are no changes across all the repositories, no
 release is created and an announcement is made on the developer mailing list to highlight this.
 If a release is being made, each repository is tagged for this release, regardless
 of whether changes are introduced. The release schedule can be seen on the
@@ -142,10 +143,10 @@ maturity, we have increased the cadence from six weeks to twelve weeks. The rele
 ### Compatibility
 Kata guarantees compatibility between components that are within one minor release of each other. 
  
-This is critical for dependencies which cross between host (runtime, shim, proxy) and
+This is critical for dependencies which cross between host (shimv2 runtime) and
 the guest (hypervisor, rootfs and agent).  For example, consider a cluster with a long-running
-deployment, workload-never-dies, all on Kata version 1.1.3 components. If the operator updates
-the Kata components to the next new minor release (i.e. 1.2.0), we need to guarantee that the 1.2.0
-runtime still communicates with 1.1.3 agent within workload-never-dies.
+deployment, workload-never-dies, all on Kata version 2.1.3 components. If the operator updates
+the Kata components to the next new minor release (i.e. 2.2.0), we need to guarantee that the 2.2.0
+shimv2 runtime still communicates with 2.1.3 agent within workload-never-dies.
 
 Handling live-update is out of the scope of this document. See this [`kata-runtime` issue](https://github.com/kata-containers/runtime/issues/492) for details.

docs/Upgrading.md

@@ -1,185 +1,127 @@
-* [Introduction](#introduction)
-* [Unsupported scenarios](#unsupported-scenarios)
-* [Maintenance Warning](#maintenance-warning)
-* [Upgrade from Clear Containers](#upgrade-from-clear-containers)
-    * [Stop all running Clear Container instances](#stop-all-running-clear-container-instances)
-    * [Configuration migration](#configuration-migration)
-    * [Remove Clear Containers packages](#remove-clear-containers-packages)
-        * [Fedora](#fedora)
-        * [Ubuntu](#ubuntu)
-    * [Disable old container manager configuration](#disable-old-container-manager-configuration)
-    * [Install Kata Containers](#install-kata-containers)
-    * [Create a Kata Container](#create-a-kata-container)
-* [Upgrade from runV](#upgrade-from-runv)
-* [Upgrade Kata Containers](#upgrade-kata-containers)
-* [Appendices](#appendices)
-    * [Assets](#assets)
-        * [Guest kernel](#guest-kernel)
-        * [Image](#image)
-        * [Determining asset versions](#determining-asset-versions)
-
 # Introduction
 
-This document explains how to upgrade from
-[Clear Containers](https://github.com/clearcontainers) and [runV](https://github.com/hyperhq/runv) to
-[Kata Containers](https://github.com/kata-containers) and how to upgrade an existing
-Kata Containers system to the latest version.
+This document outlines the options for upgrading from a
+[Kata Containers 1.x release](https://github.com/kata-containers/runtime/releases) to a
+[Kata Containers 2.x release](https://github.com/kata-containers/kata-containers/releases).
 
-# Unsupported scenarios
+# Maintenance warning
 
-Upgrading a Clear Containers system on the following distributions is **not**
-supported since the installation process for these distributions makes use of
-unpackaged components:
+Kata Containers 2.x is the new focus for the Kata Containers development
+community.
 
-- [CentOS](https://github.com/clearcontainers/runtime/blob/master/docs/centos-installation-guide.md)
-- [BCLinux](https://github.com/clearcontainers/runtime/blob/master/docs/bclinux-installation-guide.md)
-- [RHEL](https://github.com/clearcontainers/runtime/blob/master/docs/rhel-installation-guide.md)
-- [SLES](https://github.com/clearcontainers/runtime/blob/master/docs/sles-installation-guide.md)
+Although Kata Containers 1.x releases will continue to be published for a
+period of time, once a stable release for Kata Containers 2.x is published,
+Kata Containers 1.x stable users should consider switching to the Kata 2.x
+release.
 
-Additionally, upgrading
-[Clear Linux](https://github.com/clearcontainers/runtime/blob/master/docs/clearlinux-installation-guide.md)
-is not supported as Kata Containers packages do not yet exist.
+See the [stable branch strategy documentation](Stable-Branch-Strategy.md) for
+further details.
 
-# Maintenance Warning
+# Determine current version
 
-The Clear Containers codebase is no longer being developed. Only new releases
-will be considered for significant bug fixes.
+To display the current Kata Containers version, run one of the following:
 
-The main development focus is now on Kata Containers. All Clear Containers
-users are encouraged to switch to Kata Containers.
-
-# Upgrade from Clear Containers
-
-Since Kata Containers can co-exist on the same system as Clear Containers, if
-you already have Clear Containers installed, the upgrade process is simply to
-install Kata Containers. However, since Clear Containers is
-[no longer being actively developed](#maintenance-warning),
-you are encouraged to remove Clear Containers from your systems.
-
-## Stop all running Clear Container instances
-
-Assuming a Docker\* system, to stop all currently running Clear Containers:
-
-```
-$ for container in $(sudo docker ps -q); do sudo docker stop $container; done
+```bash
+$ kata-runtime --version
+$ containerd-shim-kata-v2 --version
 ```
 
-## Configuration migration
+# Determine latest version
 
-The automatic migration of
-[Clear Containers configuration](https://github.com/clearcontainers/runtime#configuration) to
-[Kata Containers configuration](../src/runtime/README.md#configuration) is
-not supported.
+Kata Containers 2.x releases are published on the
+[Kata Containers GitHub releases page](https://github.com/kata-containers/kata-containers/releases).
 
-If you have made changes to your Clear Containers configuration, you should
-review those changes and decide whether to manually apply those changes to the
-Kata Containers configuration.
+Alternatively, if you are using Kata Containers version 1.12.0 or newer, you
+can check for newer releases using the command line:
 
-> **Note**: This step must be completed before continuing to
-> [remove the Clear Containers packages](#remove-clear-containers-packages) since doing so will
-> *delete the default Clear Containers configuration file from your system*.
-
-## Remove Clear Containers packages
-
-> **Warning**: If you have modified your
-> [Clear Containers configuration](https://github.com/clearcontainers/runtime#configuration),
-> you might want to make a safe copy of the configuration file before removing the
-> packages since doing so will *delete the default configuration file*
-
-### Fedora
-
-```
-$ sudo -E dnf remove cc-runtime\* cc-proxy\* cc-shim\* linux-container clear-containers-image qemu-lite cc-ksm-throttler
-$ sudo rm /etc/yum.repos.d/home:clearcontainers:clear-containers-3.repo
+```bash
+$ kata-runtime check --check-version-only
 ```
 
-### Ubuntu
+There are various other related options. Run `kata-runtime check --help`
+for further details.
 
-```
-$ sudo apt-get purge cc-runtime\* cc-proxy\* cc-shim\* linux-container clear-containers-image qemu-lite cc-ksm-throttler
-$ sudo rm /etc/apt/sources.list.d/clear-containers.list
-```
+# Configuration changes
 
-## Disable old container manager configuration
+The [Kata Containers 2.x configuration file](/src/runtime/README.md#configuration)
+is compatible with the
+[Kata Containers 1.x configuration file](https://github.com/kata-containers/runtime/blob/master/README.md#configuration).
 
-Assuming a Docker installation, remove the docker configuration for Clear
-Containers:
+However, if you have created a local configuration file
+(`/etc/kata-containers/configuration.toml`), this will mask the newer Kata
+Containers 2.x configuration file.
 
-```
-$ sudo rm /etc/systemd/system/docker.service.d/clear-containers.conf
-```
-
-## Install Kata Containers
-
-Follow one of the [installation guides](install).
-
-## Create a Kata Container
-
-```
-$ sudo docker run -ti busybox sh
-```
-
-# Upgrade from runV
-
-runV and Kata Containers can run together on the same system without affecting each other, as long as they are
-not configured to use the same container root storage. Currently, runV defaults to `/run/runv` and Kata Containers
-defaults to `/var/run/kata-containers`.
-
-Now, to upgrade from runV you need to fresh install Kata Containers by following one of
-the [installation guides](install).
+Since Kata Containers 2.x introduces a number of new options and changes
+some default values, we recommend that you disable the local configuration
+file (by moving or renaming it) until you have reviewed the changes to the
+official configuration file and applied them to your local file if required.
 
 # Upgrade Kata Containers
 
+## Upgrade native distribution packaged version
+
 As shown in the
 [installation instructions](install),
 Kata Containers provide binaries for popular distributions in their native
 packaging formats. This allows Kata Containers to be upgraded using the
 standard package management tools for your distribution.
 
-# Appendices
+> **Note:**
+>
+> Users should prefer the distribution packaged version of Kata Containers
+> unless they understand the implications of a manual installation.
 
-## Assets
+## Static installation
 
-Kata Containers requires additional resources to create a virtual machine
-container. These resources are called
-[Kata Containers assets](./design/architecture.md#assets),
-which comprise a guest kernel and a root filesystem or initrd image. This
-section describes when these components are updated.
+> **Note:**
+>
+> Unless you are an advanced user, if you are using a static installation of
+> Kata Containers, we recommend you remove it and install a
+> [native distribution packaged version](#upgrade-native-distribution-packaged-version)
+> instead.
 
-Since the official assets are packaged, they are automatically upgraded when
-new package versions are published.
+### Determine if you are using a static installation
 
-> **Warning**: Note that if you use custom assets (by modifying the
-> [Kata Runtime configuration > file](../src/runtime/README.md#configuration)),
-> it is your responsibility to ensure they are updated as necessary.
-
-### Guest kernel
-
-The `kata-linux-container` package contains a Linux\* kernel based on the
-latest vanilla version of the
-[long-term kernel](https://www.kernel.org/)
-plus a small number of
-[patches](../tools/packaging/kernel).
-
-The `Longterm` branch is only updated with
-[important bug fixes](https://www.kernel.org/category/releases.html)
-meaning this package is only updated when necessary.
-
-The guest kernel package is updated when a new long-term kernel is released
-and when any patch updates are required.
-
-### Image
-
-The `kata-containers-image` package is updated only when critical updates are
-available for the packages used to create it, such as:
-
-- systemd
-- [Kata Containers Agent](../src/agent)
-
-### Determining asset versions
-
-To see which versions of the assets being used:
+If the following command displays the output "static", you are using a static
+version of Kata Containers:
 
+```bash
+$ ls /opt/kata/bin/kata-runtime &>/dev/null && echo static
 ```
-$ kata-runtime kata-env
-```
+
+### Remove a static installation
+
+Static installations are installed in `/opt/kata/`, so to uninstall simply
+remove this directory.
+
+### Upgrade a static installation
+
+If you understand the implications of using a static installation, to upgrade
+first
+[remove the existing static installation](#remove-a-static-installation), then
+[install the latest release](#determine-latest-version).
+
+See the
+[manual installation installation documentation](install/README.md#manual-installation)
+for details on how to automatically install and configuration a static release
+with containerd.
+
+# Custom assets
+
+> **Note:**
+>
+> This section only applies to advanced users who have built their own guest
+> kernel or image.
+
+If you are using custom
+[guest assets](design/architecture.md#guest-assets),
+you must upgrade them to work with Kata Containers 2.x since Kata
+Containers 1.x assets will **not** work.
+
+See the following for further details:
+
+- [Guest kernel documentation](/tools/packaging/kernel)
+- [Guest image and initrd documentation](/tools/osbuilder)
+
+The official assets are packaged meaning they are automatically included in
+new releases.

docs/design/README.md

@@ -8,4 +8,9 @@ Kata Containers design documents:
 - [VSocks](VSocks.md)
 - [VCPU handling](vcpu-handling.md)
 - [Host cgroups](host-cgroups.md)
+- [`Inotify` support](inotify.md)
 - [Metrics(Kata 2.0)](kata-2-0-metrics.md)
+
+---
+
+- [Design proposals](proposals)

docs/design/VSocks.md

@@ -1,12 +1,5 @@
 # Kata Containers and VSOCKs
 
-- [Introduction](#introduction)
-    - [VSOCK communication diagram](#vsock-communication-diagram)
-- [System requirements](#system-requirements)
-- [Advantages of using VSOCKs](#advantages-of-using-vsocks)
-    - [High density](#high-density)
-    - [Reliability](#reliability)
-
 ## Introduction
 
 There are two different ways processes in the virtual machine can communicate

docs/design/architecture.md

@@ -1,72 +1,35 @@
 # Kata Containers Architecture
 
-
-* [Overview](#overview)
-* [Virtualization](#virtualization)
-* [Guest assets](#guest-assets)
-    * [Guest kernel](#guest-kernel)
-    * [Guest Image](#guest-image)
-      * [Root filesystem image](#root-filesystem-image)
-      * [Initrd image](#initrd-image)
-* [Agent](#agent)
-* [Runtime](#runtime)
-    * [Configuration](#configuration)
-    * [Significant OCI commands](#significant-oci-commands)
-        * [create](#create)
-        * [start](#start)
-        * [exec](#exec)
-        * [kill](#kill)
-        * [delete](#delete)
-* [Networking](#networking)
-* [Storage](#storage)
-* [Kubernetes Support](#kubernetes-support)
-    * [Problem Statement](#problem-statement)
-    * [Containerd](#containerd)
-    * [CRI-O](#cri-o)
-        * [OCI Annotations](#oci-annotations)
-        * [Mixing VM based and namespace based runtimes](#mixing-vm-based-and-namespace-based-runtimes)
-* [Appendices](#appendices)
-    * [DAX](#dax)
-
 ## Overview
 
-This is an architectural overview of Kata Containers, based on the 1.5.0 release.
+This is an architectural overview of Kata Containers, based on the 2.0 release.
 
-The two primary deliverables of the Kata Containers project are a container runtime
-and a CRI friendly shim. There is also a CRI friendly library API behind them.
+The primary deliverable of the Kata Containers project is a CRI friendly shim. There is also a CRI friendly library API behind them.
 
-The [Kata Containers runtime (`kata-runtime`)](../../src/runtime)
+The [Kata Containers runtime](../../src/runtime)
 is compatible with the [OCI](https://github.com/opencontainers) [runtime specification](https://github.com/opencontainers/runtime-spec)
-and therefore works seamlessly with the
-[Docker\* Engine](https://www.docker.com/products/docker-engine) pluggable runtime
-architecture. It also supports the [Kubernetes\* Container Runtime Interface (CRI)](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-node/container-runtime-interface.md)
+and therefore works seamlessly with the [Kubernetes\* Container Runtime Interface (CRI)](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-node/container-runtime-interface.md)
 through the [CRI-O\*](https://github.com/kubernetes-incubator/cri-o) and
-[Containerd CRI Plugin\*](https://github.com/containerd/cri) implementation. In other words, you can transparently
-select between the [default Docker and CRI shim runtime (runc)](https://github.com/opencontainers/runc)
-and `kata-runtime`.
+[Containerd\*](https://github.com/containerd/containerd) implementation.
 
-`kata-runtime` creates a QEMU\*/KVM virtual machine for each container or pod,
-the Docker engine or `kubelet` (Kubernetes) creates respectively.
-
-![Docker and Kata Containers](arch-images/docker-kata.png)
+Kata Containers creates a QEMU\*/KVM virtual machine for pod that `kubelet` (Kubernetes) creates respectively.
 
 The [`containerd-shim-kata-v2` (shown as `shimv2` from this point onwards)](../../src/runtime/containerd-shim-v2) 
-is another Kata Containers entrypoint, which 
+is the Kata Containers entrypoint, which
 implements the [Containerd Runtime V2 (Shim API)](https://github.com/containerd/containerd/tree/master/runtime/v2) for Kata.
-With `shimv2`, Kubernetes can launch Pod and OCI compatible containers with one shim (the `shimv2`) per Pod instead
-of `2N+1` shims (a `containerd-shim` and a `kata-shim` for each container and the Pod sandbox itself), and no standalone
-`kata-proxy` process even if no VSOCK is available.
+
+Before `shimv2` (as done in [Kata Containers 1.x releases](https://github.com/kata-containers/runtime/releases)), we need to create a `containerd-shim` and a [`kata-shim`](https://github.com/kata-containers/shim) for each container and the Pod sandbox itself, plus an optional [`kata-proxy`](https://github.com/kata-containers/proxy) when VSOCK is not available. With `shimv2`, Kubernetes can launch Pod and OCI compatible containers with one shim (the `shimv2`) per Pod instead of `2N+1` shims, and no standalone `kata-proxy` process even if no VSOCK is available.
 
 ![Kubernetes integration with shimv2](arch-images/shimv2.svg)
 
 The container process is then spawned by
-[agent](../../src/agent), an agent process running
-as a daemon inside the virtual machine. `kata-agent` runs a gRPC server in
+[`kata-agent`](../../src/agent), an agent process running
+as a daemon inside the virtual machine. `kata-agent` runs a [`ttRPC`](https://github.com/containerd/ttrpc-rust) server in
 the guest using a VIRTIO serial or VSOCK interface which QEMU exposes as a socket
-file on the host. `kata-runtime` uses a gRPC protocol to communicate with
+file on the host. `shimv2` uses a `ttRPC` protocol to communicate with
 the agent. This protocol allows the runtime to send container management
 commands to the agent. The protocol is also used to carry the I/O streams (stdout,
-stderr, stdin) between the containers and the manage engines (e.g. Docker Engine).
+stderr, stdin) between the containers and the manage engines (e.g. CRI-O or containerd).
 
 For any given container, both the init process and all potentially executed
 commands within that container, together with their related I/O streams, need
@@ -74,7 +37,7 @@ to go through the VSOCK interface exported by QEMU.
 
 The container workload, that is, the actual OCI bundle rootfs, is exported from the
 host to the virtual machine.  In the case where a block-based graph driver is
-configured, `virtio-scsi` will be used. In all other cases a 9pfs VIRTIO mount point
+configured, `virtio-scsi` will be used. In all other cases a `virtio-fs` VIRTIO mount point
 will be used. `kata-agent` uses this mount point as the root filesystem for the
 container processes.
 
@@ -111,7 +74,7 @@ The only services running in the context of the mini O/S are the init daemon
 is created using libcontainer, creating a container in the same manner that is done
 by `runc`.
 
-For example, when `docker run -ti ubuntu date` is run:
+For example, when `ctr run -ti ubuntu date` is run:
 
 - The hypervisor will boot the mini-OS image using the guest kernel.
 - `systemd`, running inside the mini-OS context, will launch the `kata-agent` in
@@ -130,170 +93,37 @@ The only service running in the context of the initrd is the [Agent](#agent) as
 
 ## Agent
 
-[`kata-agent`](../../src/agent) is a process running in the
-guest as a supervisor for managing containers and processes running within
-those containers.
+[`kata-agent`](../../src/agent) is a process running in the guest as a supervisor for managing containers and processes running within those containers.
 
-The `kata-agent` execution unit is the sandbox. A `kata-agent` sandbox is a container sandbox defined by a set of namespaces (NS, UTS, IPC and PID). `kata-runtime` can
+For the 2.0 release, the `kata-agent` is rewritten in the [RUST programming language](https://www.rust-lang.org/) so that we can minimize its memory footprint while keeping the memory safety of the original GO version of [`kata-agent` used in Kata Container 1.x](https://github.com/kata-containers/agent). This memory footprint reduction is pretty impressive, from tens of megabytes down to less than 100 kilobytes, enabling Kata Containers in more use cases like functional computing and edge computing.
+
+The `kata-agent` execution unit is the sandbox. A `kata-agent` sandbox is a container sandbox defined by a set of namespaces (NS, UTS, IPC and PID). `shimv2` can
 run several containers per VM to support container engines that require multiple
-containers running inside a pod. In the case of docker, `kata-runtime` creates a
-single container per pod.
+containers running inside a pod.
 
-`kata-agent` communicates with the other Kata components over ttRPC.
-
-### Agent gRPC protocol
-
-placeholder
+`kata-agent` communicates with the other Kata components over `ttRPC`.
 
 ## Runtime
 
-`kata-runtime` is an OCI compatible container runtime and is responsible for handling
-all commands specified by
-[the OCI runtime specification](https://github.com/opencontainers/runtime-spec)
-and launching `kata-shim` instances.
+`containerd-shim-kata-v2` is a [containerd runtime shimv2](https://github.com/containerd/containerd/blob/v1.4.1/runtime/v2/README.md) implementation and is responsible for handling the `runtime v2 shim APIs`, which is similar to [the OCI runtime specification](https://github.com/opencontainers/runtime-spec) but simplifies the architecture by loading the runtime once and making RPC calls to handle the various container lifecycle commands. This refinement is an improvement on the OCI specification which requires the container manager call the runtime binary multiple times, at least once for each lifecycle command.
 
-`kata-runtime` heavily utilizes the
-[virtcontainers project](https://github.com/containers/virtcontainers), which
-provides a generic, runtime-specification agnostic, hardware-virtualized containers
-library.
+`containerd-shim-kata-v2` heavily utilizes the
+[virtcontainers package](../../src/runtime/virtcontainers/), which provides a generic, runtime-specification agnostic, hardware-virtualized containers library.
 
 ### Configuration
 
-The runtime uses a TOML format configuration file called `configuration.toml`. By
-default this file is installed in the `/usr/share/defaults/kata-containers`
-directory and contains various settings such as the paths to the hypervisor,
-the guest kernel and the mini-OS image.
+The runtime uses a TOML format configuration file called `configuration.toml`. By default this file is installed in the `/usr/share/defaults/kata-containers` directory and contains various settings such as the paths to the hypervisor, the guest kernel and the mini-OS image.
 
+The actual configuration file paths can be determined by running:
+```
+$ kata-runtime --show-default-config-paths
+```
 Most users will not need to modify the configuration file.
 
-The file is well commented and provides a few "knobs" that can be used to modify
-the behavior of the runtime.
+The file is well commented and provides a few "knobs" that can be used to modify the behavior of the runtime and your chosen hypervisor.
 
 The configuration file is also used to enable runtime [debug output](../Developer-Guide.md#enable-full-debug).
 
-### Significant OCI commands
-
-Here we describe how `kata-runtime` handles the most important OCI commands.
-
-#### `create`
-
-When handling the OCI
-[`create`](https://github.com/kata-containers/runtime/blob/master/cli/create.go)
-command, `kata-runtime` goes through the following steps:
-
-1. Create the network namespace where we will spawn VM and shims processes.
-2. Call into the pre-start hooks. One of them should be responsible for creating
-the `veth` network pair between the host network namespace and the network namespace
-freshly created.
-3. Scan the network from the new network namespace, and create a MACVTAP connection
- between the `veth` interface and a `tap` interface into the VM.
-4. Start the VM inside the network namespace by providing the `tap` interface
- previously created.
-5. Wait for the VM to be ready.
-6. Start `kata-proxy`, which will connect to the created VM. The `kata-proxy` process
-will take care of proxying all communications with the VM. Kata has a single proxy
-per VM.
-7. Communicate with `kata-agent` (through the proxy) to configure the sandbox
- inside the VM.
-8. Communicate with `kata-agent` to create the container, relying on the OCI
-configuration file `config.json` initially provided to `kata-runtime`. This
-spawns the container process inside the VM, leveraging the `libcontainer` package.
-9. Start `kata-shim`, which will connect to the gRPC server socket provided by the `kata-proxy`. `kata-shim`  will spawn a few Go routines to parallelize blocking calls `ReadStdout()` , `ReadStderr()` and `WaitProcess()`. Both `ReadStdout()` and `ReadStderr()` are run through infinite loops since `kata-shim` wants the output of those until the container process terminates. `WaitProcess()` is a unique call which returns the exit code of the container process when it terminates inside the VM. Note that `kata-shim` is started inside the network namespace, to allow upper layers to determine which network namespace has been created and by checking the `kata-shim` process. It also creates a new PID namespace by entering into it. This ensures that all `kata-shim` processes belonging to the same container will get killed when the `kata-shim` representing the container process terminates.
-
-At this point the container process is running inside of the VM, and it is represented
-on the host system by the `kata-shim` process.
-
-![`kata-oci-create`](arch-images/kata-oci-create.svg)
-
-#### `start`
-
-With traditional containers, [`start`](https://github.com/kata-containers/runtime/blob/master/cli/start.go) launches a container process in its own set of namespaces. With Kata Containers, the main task of `kata-runtime` is to ask [`kata-agent`](#agent) to start the container workload inside the virtual machine. `kata-runtime` will run through the following steps:
-
-1. Communicate with `kata-agent` (through the proxy) to start the container workload
- inside the VM. If, for example, the command to execute inside of the container is `top`,
- the `kata-shim`'s `ReadStdOut()` will start returning text output for top, and
-  `WaitProcess()` will continue to block as long as the `top` process runs.
-2. Call into the post-start hooks. Usually, this is a no-op since nothing is provided
-  (this needs clarification)
-
-![`kata-oci-start`](arch-images/kata-oci-start.svg)
-
-#### `exec`
-
-OCI [`exec`](https://github.com/kata-containers/runtime/blob/master/cli/exec.go) allows you to run an additional command within an already running
-container.  In Kata Containers, this is handled as follows:
-
-1. A request is sent to the `kata agent` (through the proxy) to start a new process
- inside an existing container running within the VM.
-2. A new `kata-shim` is created within the same network and PID namespaces as the
- original `kata-shim` representing the container process. This new `kata-shim` is
- used for the new exec process.
-
-Now the process started with `exec` is running within the VM, sharing `uts`, `pid`, `mnt` and `ipc` namespaces with the container process.
-
-![`kata-oci-exec`](arch-images/kata-oci-exec.svg)
-
-#### `kill`
-
-When sending the OCI [`kill`](https://github.com/kata-containers/runtime/blob/master/cli/kill.go) command, the container runtime should send a
-[UNIX signal](https://en.wikipedia.org/wiki/Unix_signal) to the container process.
-A `kill` sending a termination signal such as `SIGKILL` or `SIGTERM` is expected
-to terminate the container process.  In the context of a traditional container,
-this means stopping the container.  For `kata-runtime`, this translates to stopping
-the container and the VM associated with it.
-
-1. Send a request to kill the container process to the `kata-agent` (through the proxy).
-2. Wait for `kata-shim` process to exit.
-3. Force kill the container process if `kata-shim` process didn't return after a
- timeout. This is done by communicating with `kata-agent` (connecting the proxy),
- sending `SIGKILL` signal to the container process inside the VM.
-4. Wait for `kata-shim` process to exit, and return an error if we reach the
- timeout again.
-5. Communicate with `kata-agent` (through the proxy) to remove the container
- configuration from the VM.
-6. Communicate with `kata-agent` (through the proxy) to destroy the sandbox
- configuration from the VM.
-7. Stop the VM.
-8. Remove all network configurations inside the network namespace and delete the
- namespace.
-9. Execute post-stop hooks.
-
-If `kill` was invoked with a non-termination signal, this simply signals the container process. Otherwise, everything has been torn down, and the VM has been removed.
-
-#### `delete`
-
-[`delete`](https://github.com/kata-containers/runtime/blob/master/cli/delete.go) removes all internal resources related to a container. A running container
-cannot be deleted unless the OCI runtime is explicitly being asked to, by using
-`--force` flag.
-
-If the sandbox is not stopped, but the particular container process returned on
-its own already, the `kata-runtime` will first go through most of the steps a `kill`
-would go through for a termination signal. After this process, or if the `sandboxID` was already stopped to begin with, then `kata-runtime` will:
-
-1. Remove container resources. Every file kept under `/var/{lib,run}/virtcontainers/sandboxes/<sandboxID>/<containerID>`.
-2. Remove sandbox resources. Every file kept under `/var/{lib,run}/virtcontainers/sandboxes/<sandboxID>`.
-
-At this point, everything related to the container should have been removed from the host system, and no related process should be running.
-
-#### `state`
-
-[`state`](https://github.com/kata-containers/runtime/blob/master/cli/state.go)
-returns the status of the container. For `kata-runtime`, this means being
-able to detect if the container is still running by looking at the state of `kata-shim`
-process representing this container process.
-
-1. Ask the container status by checking information stored on disk. (clarification needed)
-2. Check `kata-shim` process representing the container.
-3. In case the container status on disk was supposed to be `ready` or `running`,
- and the `kata-shim` process no longer exists, this involves the detection of a
- stopped container. This means that before returning the container status,
- the container has to be properly stopped. Here are the steps involved in this detection:
-	1. Wait for `kata-shim` process to exit.
-	2. Force kill the container process if `kata-shim` process didn't return after a timeout. This is done by communicating with `kata-agent` (connecting the proxy), sending `SIGKILL` signal to the container process inside the VM.
-	3. Wait for `kata-shim` process to exit, and return an error if we reach the timeout again.
-	4. Communicate with `kata-agent` (connecting the proxy) to remove the container configuration from the VM.
-4. Return container status.
-
 ## Networking
 
 Containers will typically live in their own, possibly shared, networking namespace.
@@ -305,66 +135,31 @@ In order to do so, container engines will usually add one end of a virtual
 ethernet (`veth`) pair into the container networking namespace. The other end of
 the `veth` pair is added to the host networking namespace.
 
-This is a very namespace-centric approach as many hypervisors (in particular QEMU)
-cannot handle `veth` interfaces. Typically, `TAP` interfaces are created for VM
-connectivity.
+This is a very namespace-centric approach as many hypervisors/VMMs cannot handle `veth` 
+interfaces. Typically, `TAP` interfaces are created for VM connectivity.
 
 To overcome incompatibility between typical container engines expectations
-and virtual machines, `kata-runtime` networking transparently connects `veth`
-interfaces with `TAP` ones using MACVTAP:
+and virtual machines, Kata Containers networking transparently connects `veth`
+interfaces with `TAP` ones using Traffic Control:
 
 ![Kata Containers networking](arch-images/network.png)
 
+With a TC filter in place, a redirection is created between the container network and the
+virtual machine. As an example, the CNI may create a device, `eth0`, in the container's network
+namespace, which is a VETH device. Kata Containers will create a tap device for the VM, `tap0_kata`,
+and setup a TC redirection filter to mirror traffic from `eth0`'s ingress to `tap0_kata`'s egress,
+and a second to mirror traffic from `tap0_kata`'s ingress to `eth0`'s egress.
+
+Kata Containers maintains support for MACVTAP, which was an earlier implementation used in Kata. TC-filter
+is the default because it allows for simpler configuration, better CNI plugin compatibility, and performance
+on par with MACVTAP.
+
+Kata Containers has deprecated support for bridge due to lacking performance relative to TC-filter and MACVTAP.
+
  Kata Containers supports both
 [CNM](https://github.com/docker/libnetwork/blob/master/docs/design.md#the-container-network-model)
 and [CNI](https://github.com/containernetworking/cni) for networking management.
 
-### CNM
-
-![High-level CNM Diagram](arch-images/CNM_overall_diagram.png)
-
-__CNM lifecycle__
-
-1.  `RequestPool`
-
-2.  `CreateNetwork`
-
-3.  `RequestAddress`
-
-4.  `CreateEndPoint`
-
-5.  `CreateContainer`
-
-6.  Create `config.json`
-
-7.  Create PID and network namespace
-
-8.  `ProcessExternalKey`
-
-9.  `JoinEndPoint`
-
-10. `LaunchContainer`
-
-11. Launch
-
-12. Run container
-
-![Detailed CNM Diagram](arch-images/CNM_detailed_diagram.png)
-
-__Runtime network setup with CNM__
-
-1. Read `config.json`
-
-2. Create the network namespace
-
-3. Call the `prestart` hook (from inside the netns)
-
-4. Scan network interfaces inside netns and get the name of the interface
-  created by prestart hook
-
-5. Create bridge, TAP, and link all together with network interface previously
-  created
-
 ### Network Hotplug
 
 Kata Containers has developed a set of network sub-commands and APIs to add, list and
@@ -375,35 +170,14 @@ The following diagram illustrates the Kata Containers network hotplug workflow.
 ![Network Hotplug](arch-images/kata-containers-network-hotplug.png)
 
 ## Storage
-Container workloads are shared with the virtualized environment through [9pfs](https://www.kernel.org/doc/Documentation/filesystems/9p.txt).
-The devicemapper storage driver is a special case. The driver uses dedicated block
-devices rather than formatted filesystems, and operates at the block level rather
-than the file level. This knowledge is used to directly use the underlying block
-device instead of the overlay file system for the container root file system. The
-block device maps to the top read-write layer for the overlay. This approach gives
-much better I/O performance compared to using 9pfs to share the container file system.
+Container workloads are shared with the virtualized environment through [virtio-fs](https://virtio-fs.gitlab.io/).
 
-The approach above does introduce a limitation in terms of dynamic file copy
-in/out of the container using the `docker cp` operations. The copy operation from
-host to container accesses the mounted file system on the host-side. This is
-not expected to work and may lead to inconsistencies as the block device will
-be simultaneously written to from two different mounts. The copy operation from
-container to host will work, provided the user calls `sync(1)` from within the
-container prior to the copy to make sure any outstanding cached data is written
-to the block device.
+The [devicemapper `snapshotter`](https://github.com/containerd/containerd/tree/master/snapshots/devmapper) is a special case. The `snapshotter` uses dedicated block devices rather than formatted filesystems, and operates at the block level rather than the file level. This knowledge is used to directly use the underlying block device instead of the overlay file system for the container root file system. The block device maps to the top read-write layer for the overlay. This approach gives much better I/O performance compared to using `virtio-fs` to share the container file system.
 
-```
-docker cp [OPTIONS] CONTAINER:SRC_PATH HOST:DEST_PATH
-docker cp [OPTIONS] HOST:SRC_PATH CONTAINER:DEST_PATH
-```
+Kata Containers has the ability to hotplug and remove block devices, which makes it possible to use block devices for containers started after the VM has been launched.
 
-Kata Containers has the ability to hotplug and remove block devices, which makes it
-possible to use block devices for containers started after the VM has been launched.
-
-Users can check to see if the container uses the devicemapper block device as its
-rootfs by calling `mount(8)` within the container.  If the devicemapper block device
-is used, `/` will be mounted on `/dev/vda`.  Users can disable direct mounting
-of the underlying block device through the runtime configuration.
+Users can check to see if the container uses the devicemapper block device as its rootfs by calling `mount(8)` within the container.  If the devicemapper block device
+is used, `/` will be mounted on `/dev/vda`.  Users can disable direct mounting of the underlying block device through the runtime configuration.
 
 ## Kubernetes support
 
@@ -424,44 +198,13 @@ lifecycle management from container execution through the dedicated
 
 In other words, a Kubelet is a CRI client and expects a CRI implementation to
 handle the server side of the interface.
-[CRI-O\*](https://github.com/kubernetes-incubator/cri-o) and [Containerd CRI Plugin\*](https://github.com/containerd/cri) are CRI implementations that rely on [OCI](https://github.com/opencontainers/runtime-spec)
+[CRI-O\*](https://github.com/kubernetes-incubator/cri-o) and [Containerd\*](https://github.com/containerd/containerd/) are CRI implementations that rely on [OCI](https://github.com/opencontainers/runtime-spec)
 compatible runtimes for managing container instances.
 
-Kata Containers is an officially supported CRI-O and Containerd CRI Plugin runtime. It is OCI compatible and therefore aligns with project's architecture and requirements.
-However, due to the fact that Kubernetes execution units are sets of containers (also
-known as pods) rather than single containers, the Kata Containers runtime needs to
-get extra information to seamlessly integrate with Kubernetes.
-
-### Problem statement
-
-The Kubernetes\* execution unit is a pod that has specifications detailing constraints
-such as namespaces, groups, hardware resources, security contents, *etc* shared by all
-the containers within that pod.
-By default the Kubelet will send a container creation request to its CRI runtime for
-each pod and container creation. Without additional metadata from the CRI runtime,
-the Kata Containers runtime will thus create one virtual machine for each pod and for
-each containers within a pod. However the task of providing the Kubernetes pod semantics
-when creating one virtual machine for each container within the same pod is complex given
-the resources of these virtual machines (such as networking or PID) need to be shared.
-
-The challenge with Kata Containers when working as a Kubernetes\* runtime is thus to know
-when to create a full virtual machine (for pods) and when to create a new container inside
-a previously created virtual machine. In both cases it will get called with very similar
-arguments, so it needs the help of the Kubernetes CRI runtime to be able to distinguish a
-pod creation request from a container one.
-
-### Containerd
-
-As of Kata Containers 1.5, using `shimv2` with containerd 1.2.0 or above is the preferred
-way to run Kata Containers with Kubernetes ([see the howto](../how-to/how-to-use-k8s-with-cri-containerd-and-kata.md#configure-containerd-to-use-kata-containers)).
-The CRI-O will catch up soon ([`kubernetes-sigs/cri-o#2024`](https://github.com/kubernetes-sigs/cri-o/issues/2024)).
-
-Refer to the following how-to guides:
+Kata Containers is an officially supported CRI-O and Containerd runtime. Refer to the following guides on how to set up Kata Containers with Kubernetes:
 
 - [How to use Kata Containers and Containerd](../how-to/containerd-kata.md)
-- [How to use Kata Containers and CRI (containerd plugin) with Kubernetes](../how-to/how-to-use-k8s-with-cri-containerd-and-kata.md)
-
-### CRI-O
+- [Run Kata Containers with Kubernetes](../how-to/run-kata-with-k8s.md)
 
 ####  OCI annotations
 
@@ -506,36 +249,10 @@ with a Kubernetes pod:
 
 #### Mixing VM based and namespace based runtimes
 
-> **Note:** Since Kubernetes 1.12, the [`Kubernetes RuntimeClass`](../how-to/containerd-kata.md#kubernetes-runtimeclass)
+> **Note:** Since Kubernetes 1.12, the [`Kubernetes RuntimeClass`](https://kubernetes.io/docs/concepts/containers/runtime-class/)
 > has been supported and the user can specify runtime without the non-standardized annotations.
 
-One interesting evolution of the CRI-O support for `kata-runtime` is the ability
-to run virtual machine based pods alongside namespace ones. With CRI-O and Kata
-Containers, one can introduce the concept of workload trust inside a Kubernetes
-cluster.
-
-A cluster operator can now tag (through Kubernetes annotations) container workloads
-as `trusted` or `untrusted`. The former labels known to be safe workloads while
-the latter describes potentially malicious or misbehaving workloads that need the
-highest degree of isolation. In a software development context, an example of a `trusted` workload would be a containerized continuous integration engine whereas all
-developers applications would be `untrusted` by default. Developers workloads can
-be buggy, unstable or even include malicious code and thus from a security perspective
-it makes sense to tag them as `untrusted`. A CRI-O and Kata Containers based
-Kubernetes cluster handles this use case transparently as long as the deployed
-containers are properly tagged. All `untrusted` containers will be handled by Kata Containers and thus run in a hardware virtualized secure sandbox while `runc`, for
-example, could  handle the `trusted` ones.
-
-CRI-O's default behavior is to trust all pods, except when they're annotated with
-`io.kubernetes.cri-o.TrustedSandbox` set to `false`. The default CRI-O trust level
-is set through its `configuration.toml` configuration file. Generally speaking,
-the CRI-O runtime selection between its trusted runtime (typically `runc`) and its untrusted one (`kata-runtime`) is a function of the pod `Privileged` setting, the `io.kubernetes.cri-o.TrustedSandbox` annotation value, and the default CRI-O trust
-level. When a pod is `Privileged`, the runtime will always be `runc`. However, when
-a pod is **not** `Privileged` the runtime selection is done as follows:
-
-|                                        | `io.kubernetes.cri-o.TrustedSandbox` not set   | `io.kubernetes.cri-o.TrustedSandbox` = `true` | `io.kubernetes.cri-o.TrustedSandbox` = `false` |
-| :---                                   |     :---:                                      |     :---:                                     |     :---:                                             |
-| Default CRI-O trust level: `trusted`   | runc                                           | runc                                          | Kata Containers |
-| Default CRI-O trust level: `untrusted` | Kata Containers                               | Kata Containers                              |  Kata Containers |
+With `RuntimeClass`, users can define Kata Containers as a `RuntimeClass` and then explicitly specify that a pod being created as a Kata Containers pod. For details, please refer to [How to use Kata Containers and Containerd](../../docs/how-to/containerd-kata.md).
 
 
 # Appendices

docs/design/data/metrics.yaml

@@ -220,6 +220,566 @@ components:
               fixed: false
               values: []
           since: 2.0.0
+    - prefix: kata_firecracker
+      title: Firecracker vmm metrics
+      desc: Metrics for Firecracker vmm
+      metrics:
+        - name: kata_firecracker_api_server
+          type: GAUGE
+          unit: ""
+          help: Metrics related to the internal API server.
+          labels:
+            - name: item
+              desc: ""
+              manually_edit: false
+              fixed: true
+              values:
+                - value: process_startup_time_cpu_us
+                  desc: ""
+                - value: process_startup_time_us
+                  desc: ""
+                - value: sync_response_fails
+                  desc: ""
+                - value: sync_vmm_send_timeout_count
+                  desc: ""
+            - name: sandbox_id
+              desc: ""
+              manually_edit: false
+              fixed: false
+              values: []
+          since: 2.0.0
+        - name: kata_firecracker_block
+          type: GAUGE
+          unit: ""
+          help: Block Device associated metrics.
+          labels:
+            - name: item
+              desc: ""
+              manually_edit: false
+              fixed: true
+              values:
+                - value: activate_fails
+                  desc: ""
+                - value: cfg_fails
+                  desc: ""
+                - value: event_fails
+                  desc: ""
+                - value: execute_fails
+                  desc: ""
+                - value: flush_count
+                  desc: ""
+                - value: invalid_reqs_count
+                  desc: ""
+                - value: no_avail_buffer
+                  desc: ""
+                - value: queue_event_count
+                  desc: ""
+                - value: rate_limiter_event_count
+                  desc: ""
+                - value: rate_limiter_throttled_events
+                  desc: ""
+                - value: read_bytes
+                  desc: ""
+                - value: read_count
+                  desc: ""
+                - value: update_count
+                  desc: ""
+                - value: update_fails
+                  desc: ""
+                - value: write_bytes
+                  desc: ""
+                - value: write_count
+                  desc: ""
+            - name: sandbox_id
+              desc: ""
+              manually_edit: false
+              fixed: false
+              values: []
+          since: 2.0.0
+        - name: kata_firecracker_get_api_requests
+          type: GAUGE
+          unit: ""
+          help: Metrics specific to GET API Requests for counting user triggered actions and/or failures.
+          labels:
+            - name: item
+              desc: ""
+              manually_edit: false
+              fixed: true
+              values:
+                - value: instance_info_count
+                  desc: ""
+                - value: instance_info_fails
+                  desc: ""
+                - value: machine_cfg_count
+                  desc: ""
+                - value: machine_cfg_fails
+                  desc: ""
+            - name: sandbox_id
+              desc: ""
+              manually_edit: false
+              fixed: false
+              values: []
+          since: 2.0.0
+        - name: kata_firecracker_i8042
+          type: GAUGE
+          unit: ""
+          help: Metrics specific to the i8042 device.
+          labels:
+            - name: item
+              desc: ""
+              manually_edit: false
+              fixed: true
+              values:
+                - value: error_count
+                  desc: ""
+                - value: missed_read_count
+                  desc: ""
+                - value: missed_write_count
+                  desc: ""
+                - value: read_count
+                  desc: ""
+                - value: reset_count
+                  desc: ""
+                - value: write_count
+                  desc: ""
+            - name: sandbox_id
+              desc: ""
+              manually_edit: false
+              fixed: false
+              values: []
+          since: 2.0.0
+        - name: kata_firecracker_latencies_us
+          type: GAUGE
+          unit: ""
+          help: Performance metrics related for the moment only to snapshots.
+          labels:
+            - name: item
+              desc: ""
+              manually_edit: false
+              fixed: true
+              values:
+                - value: diff_create_snapshot
+                  desc: ""
+                - value: full_create_snapshot
+                  desc: ""
+                - value: load_snapshot
+                  desc: ""
+                - value: pause_vm
+                  desc: ""
+                - value: resume_vm
+                  desc: ""
+                - value: vmm_diff_create_snapshot
+                  desc: ""
+                - value: vmm_full_create_snapshot
+                  desc: ""
+                - value: vmm_load_snapshot
+                  desc: ""
+                - value: vmm_pause_vm
+                  desc: ""
+                - value: vmm_resume_vm
+                  desc: ""
+            - name: sandbox_id
+              desc: ""
+              manually_edit: false
+              fixed: false
+              values: []
+          since: 2.0.0
+        - name: kata_firecracker_logger
+          type: GAUGE
+          unit: ""
+          help: Metrics for the logging subsystem.
+          labels:
+            - name: item
+              desc: ""
+              manually_edit: false
+              fixed: true
+              values:
+                - value: log_fails
+                  desc: ""
+                - value: metrics_fails
+                  desc: ""
+                - value: missed_log_count
+                  desc: ""
+                - value: missed_metrics_count
+                  desc: ""
+            - name: sandbox_id
+              desc: ""
+              manually_edit: false
+              fixed: false
+              values: []
+          since: 2.0.0
+        - name: kata_firecracker_mmds
+          type: GAUGE
+          unit: ""
+          help: Metrics for the MMDS functionality.
+          labels:
+            - name: item
+              desc: ""
+              manually_edit: false
+              fixed: true
+              values:
+                - value: connections_created
+                  desc: ""
+                - value: connections_destroyed
+                  desc: ""
+                - value: rx_accepted
+                  desc: ""
+                - value: rx_accepted_err
+                  desc: ""
+                - value: rx_accepted_unusual
+                  desc: ""
+                - value: rx_bad_eth
+                  desc: ""
+                - value: rx_count
+                  desc: ""
+                - value: tx_bytes
+                  desc: ""
+                - value: tx_count
+                  desc: ""
+                - value: tx_errors
+                  desc: ""
+                - value: tx_frames
+                  desc: ""
+            - name: sandbox_id
+              desc: ""
+              manually_edit: false
+              fixed: false
+              values: []
+          since: 2.0.0
+        - name: kata_firecracker_net
+          type: GAUGE
+          unit: ""
+          help: Network-related metrics.
+          labels:
+            - name: item
+              desc: ""
+              manually_edit: false
+              fixed: true
+              values:
+                - value: activate_fails
+                  desc: ""
+                - value: cfg_fails
+                  desc: ""
+                - value: event_fails
+                  desc: ""
+                - value: mac_address_updates
+                  desc: ""
+                - value: no_rx_avail_buffer
+                  desc: ""
+                - value: no_tx_avail_buffer
+                  desc: ""
+                - value: rx_bytes_count
+                  desc: ""
+                - value: rx_count
+                  desc: ""
+                - value: rx_event_rate_limiter_count
+                  desc: ""
+                - value: rx_fails
+                  desc: ""
+                - value: rx_packets_count
+                  desc: ""
+                - value: rx_partial_writes
+                  desc: ""
+                - value: rx_queue_event_count
+                  desc: ""
+                - value: rx_rate_limiter_throttled
+                  desc: ""
+                - value: rx_tap_event_count
+                  desc: ""
+                - value: tap_read_fails
+                  desc: ""
+                - value: tap_write_fails
+                  desc: ""
+                - value: tx_bytes_count
+                  desc: ""
+                - value: tx_count
+                  desc: ""
+                - value: tx_fails
+                  desc: ""
+                - value: tx_malformed_frames
+                  desc: ""
+                - value: tx_packets_count
+                  desc: ""
+                - value: tx_partial_reads
+                  desc: ""
+                - value: tx_queue_event_count
+                  desc: ""
+                - value: tx_rate_limiter_event_count
+                  desc: ""
+                - value: tx_rate_limiter_throttled
+                  desc: ""
+                - value: tx_spoofed_mac_count
+                  desc: ""
+            - name: sandbox_id
+              desc: ""
+              manually_edit: false
+              fixed: false
+              values: []
+          since: 2.0.0
+        - name: kata_firecracker_patch_api_requests
+          type: GAUGE
+          unit: ""
+          help: Metrics specific to PATCH API Requests for counting user triggered actions and/or failures.
+          labels:
+            - name: item
+              desc: ""
+              manually_edit: false
+              fixed: true
+              values:
+                - value: drive_count
+                  desc: ""
+                - value: drive_fails
+                  desc: ""
+                - value: machine_cfg_count
+                  desc: ""
+                - value: machine_cfg_fails
+                  desc: ""
+                - value: network_count
+                  desc: ""
+                - value: network_fails
+                  desc: ""
+            - name: sandbox_id
+              desc: ""
+              manually_edit: false
+              fixed: false
+              values: []
+          since: 2.0.0
+        - name: kata_firecracker_put_api_requests
+          type: GAUGE
+          unit: ""
+          help: Metrics specific to PUT API Requests for counting user triggered actions and/or failures.
+          labels:
+            - name: item
+              desc: ""
+              manually_edit: false
+              fixed: true
+              values:
+                - value: actions_count
+                  desc: ""
+                - value: actions_fails
+                  desc: ""
+                - value: boot_source_count
+                  desc: ""
+                - value: boot_source_fails
+                  desc: ""
+                - value: drive_count
+                  desc: ""
+                - value: drive_fails
+                  desc: ""
+                - value: logger_count
+                  desc: ""
+                - value: logger_fails
+                  desc: ""
+                - value: machine_cfg_count
+                  desc: ""
+                - value: machine_cfg_fails
+                  desc: ""
+                - value: metrics_count
+                  desc: ""
+                - value: metrics_fails
+                  desc: ""
+                - value: network_count
+                  desc: ""
+                - value: network_fails
+                  desc: ""
+            - name: sandbox_id
+              desc: ""
+              manually_edit: false
+              fixed: false
+              values: []
+          since: 2.0.0
+        - name: kata_firecracker_rtc
+          type: GAUGE
+          unit: ""
+          help: Metrics specific to the RTC device.
+          labels:
+            - name: item
+              desc: ""
+              manually_edit: false
+              fixed: true
+              values:
+                - value: error_count
+                  desc: ""
+                - value: missed_read_count
+                  desc: ""
+                - value: missed_write_count
+                  desc: ""
+            - name: sandbox_id
+              desc: ""
+              manually_edit: false
+              fixed: false
+              values: []
+          since: 2.0.0
+        - name: kata_firecracker_seccomp
+          type: GAUGE
+          unit: ""
+          help: Metrics for the seccomp filtering.
+          labels:
+            - name: item
+              desc: ""
+              manually_edit: false
+              fixed: true
+              values:
+                - value: num_faults
+                  desc: ""
+            - name: sandbox_id
+              desc: ""
+              manually_edit: false
+              fixed: false
+              values: []
+          since: 2.0.0
+        - name: kata_firecracker_signals
+          type: GAUGE
+          unit: ""
+          help: Metrics related to signals.
+          labels:
+            - name: item
+              desc: ""
+              manually_edit: false
+              fixed: true
+              values:
+                - value: sigbus
+                  desc: ""
+                - value: sigsegv
+                  desc: ""
+            - name: sandbox_id
+              desc: ""
+              manually_edit: false
+              fixed: false
+              values: []
+          since: 2.0.0
+        - name: kata_firecracker_uart
+          type: GAUGE
+          unit: ""
+          help: Metrics specific to the UART device.
+          labels:
+            - name: item
+              desc: ""
+              manually_edit: false
+              fixed: true
+              values:
+                - value: error_count
+                  desc: ""
+                - value: flush_count
+                  desc: ""
+                - value: missed_read_count
+                  desc: ""
+                - value: missed_write_count
+                  desc: ""
+                - value: read_count
+                  desc: ""
+                - value: write_count
+                  desc: ""
+            - name: sandbox_id
+              desc: ""
+              manually_edit: false
+              fixed: false
+              values: []
+          since: 2.0.0
+        - name: kata_firecracker_vcpu
+          type: GAUGE
+          unit: ""
+          help: Metrics specific to VCPUs' mode of functioning.
+          labels:
+            - name: item
+              desc: ""
+              manually_edit: false
+              fixed: true
+              values:
+                - value: exit_io_in
+                  desc: ""
+                - value: exit_io_out
+                  desc: ""
+                - value: exit_mmio_read
+                  desc: ""
+                - value: exit_mmio_write
+                  desc: ""
+                - value: failures
+                  desc: ""
+                - value: filter_cpuid
+                  desc: ""
+            - name: sandbox_id
+              desc: ""
+              manually_edit: false
+              fixed: false
+              values: []
+          since: 2.0.0
+        - name: kata_firecracker_vmm
+          type: GAUGE
+          unit: ""
+          help: Metrics specific to the machine manager as a whole.
+          labels:
+            - name: item
+              desc: ""
+              manually_edit: false
+              fixed: true
+              values:
+                - value: device_events
+                  desc: ""
+                - value: panic_count
+                  desc: ""
+            - name: sandbox_id
+              desc: ""
+              manually_edit: false
+              fixed: false
+              values: []
+          since: 2.0.0
+        - name: kata_firecracker_vsock
+          type: GAUGE
+          unit: ""
+          help: Vsock-related metrics.
+          labels:
+            - name: item
+              desc: ""
+              manually_edit: false
+              fixed: true
+              values:
+                - value: activate_fails
+                  desc: ""
+                - value: cfg_fails
+                  desc: ""
+                - value: conn_event_fails
+                  desc: ""
+                - value: conns_added
+                  desc: ""
+                - value: conns_killed
+                  desc: ""
+                - value: conns_removed
+                  desc: ""
+                - value: ev_queue_event_fails
+                  desc: ""
+                - value: killq_resync
+                  desc: ""
+                - value: muxer_event_fails
+                  desc: ""
+                - value: rx_bytes_count
+                  desc: ""
+                - value: rx_packets_count
+                  desc: ""
+                - value: rx_queue_event_count
+                  desc: ""
+                - value: rx_queue_event_fails
+                  desc: ""
+                - value: rx_read_fails
+                  desc: ""
+                - value: tx_bytes_count
+                  desc: ""
+                - value: tx_flush_fails
+                  desc: ""
+                - value: tx_packets_count
+                  desc: ""
+                - value: tx_queue_event_count
+                  desc: ""
+                - value: tx_queue_event_fails
+                  desc: ""
+                - value: tx_write_fails
+                  desc: ""
+            - name: sandbox_id
+              desc: ""
+              manually_edit: false
+              fixed: false
+              values: []
+          since: 2.0.0
     - prefix: kata_guest
       title: Kata guest OS metrics
       desc: Guest OS's metrics in hypervisor.

docs/design/end-to-end-flow.md

@@ -0,0 +1,3 @@
+# Kata Containers E2E Flow
+
+![Kata containers e2e flow](arch-images/katacontainers-e2e-with-bg.jpg)

docs/design/host-cgroups.md

@@ -1,18 +1,3 @@
-- [Host cgroup management](#host-cgroup-management)
-  - [Introduction](#introduction)
-  - [`SandboxCgroupOnly` enabled](#sandboxcgrouponly-enabled)
-    - [What does Kata do in this configuration?](#what-does-kata-do-in-this-configuration)
-    - [Why create a Kata-cgroup under the parent cgroup?](#why-create-a-kata-cgroup-under-the-parent-cgroup)
-    - [Improvements](#improvements)
-  - [`SandboxCgroupOnly` disabled (default, legacy)](#sandboxcgrouponly-disabled-default-legacy)
-    - [What does this method do?](#what-does-this-method-do)
-      - [Impact](#impact)
-- [Supported cgroups](#supported-cgroups)
-  - [Cgroups V1](#cgroups-v1)
-  - [Cgroups V2](#cgroups-v2)
-    - [Distro Support](#distro-support)
-- [Summary](#summary)
-
 # Host cgroup management
 
 ## Introduction

docs/design/inotify.md

@@ -0,0 +1,30 @@
+# Kata Containers support for `inotify`
+
+## Background on `inotify` usage
+
+A common pattern in Kubernetes is to watch for changes to files/directories passed in as `ConfigMaps`
+or `Secrets`. Sidecar's normally use `inotify` to watch for changes and then signal the primary container to reload
+the updated configuration. Kata Containers typically will pass these host files into the guest using `virtiofs`, which
+does not support `inotify` today. While we work to enable this use case in `virtiofs`, we introduced a workaround in Kata Containers.
+This document describes how Kata Containers implements this workaround.
+
+### Detecting a `watchable` mount
+
+Kubernetes creates `secrets` and `ConfigMap` mounts at very specific locations on the host filesystem. For container mounts,
+the `Kata Containers` runtime will check the source of the mount to identify these special cases. For these use cases, only a single file
+or very few would typically need to be watched. To avoid excessive overheads in making a mount watchable,
+we enforce a limit of eight files per mount. If a `secret` or `ConfigMap` mount contains more than 8 files, it will not be
+considered watchable. We similarly enforce a limit of 1 MB per mount to be considered watchable. Non-watchable mounts will
+continue to propagate changes from the mount on the host to the container workload, but these updates will not trigger an
+`inotify` event.
+
+If at any point a mount grows beyond the eight file or 1MB limit, it will no longer be `watchable.`
+
+### Presenting a `watchable` mount to the workload
+
+For mounts that are considered `watchable`, inside the guest, the `kata-agent` will poll the mount presented from
+the host through `virtiofs` and copy any changed files to a `tmpfs` mount that is presented to the container. In this way,
+for `watchable` mounts, Kata will do the polling on behalf of the workload and existing workloads needn't change their usage
+of `inotify`.
+
+![drawing](arch-images/inotify-workaround.png)

docs/design/kata-2-0-metrics.md

@@ -1,19 +1,5 @@
 # Kata 2.0 Metrics Design
 
-* [Limitations of Kata 1.x and the target of Kata 2.0](#limitations-of-kata-1x-and-the-target-of-kata-20)
-* [Metrics architecture](#metrics-architecture)
-  * [Kata monitor](#kata-monitor)
-  * [Kata runtime](#kata-runtime)
-  * [Kata agent](#kata-agent)
-  * [Performance and overhead](#performance-and-overhead)
-* [Metrics list](#metrics-list)
-  * [Metric types](#metric-types)
-  * [Kata agent metrics](#kata-agent-metrics)
-  * [Kata guest OS metrics](#kata-guest-os-metrics)
-  * [Hypervisor metrics](#hypervisor-metrics)
-  * [Kata monitor metrics](#kata-monitor-metrics)
-  * [Kata containerd shim v2 metrics](#kata-containerd-shim-v2-metrics)
-
 Kata implement CRI's API and support [`ContainerStats`](https://github.com/kubernetes/kubernetes/blob/release-1.18/staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2/api.proto#L101) and [`ListContainerStats`](https://github.com/kubernetes/kubernetes/blob/release-1.18/staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2/api.proto#L103) interfaces to expose containers metrics. User can use these interface to get basic metrics about container.
 
 But unlike `runc`, Kata is a VM-based runtime and has a different architecture.
@@ -152,6 +138,7 @@ Metrics is categorized by component where metrics are collected from and for.
 
 * [Metric types](#metric-types)
 * [Kata agent metrics](#kata-agent-metrics)
+* [Firecracker metrics](#firecracker-metrics)
 * [Kata guest OS metrics](#kata-guest-os-metrics)
 * [Hypervisor metrics](#hypervisor-metrics)
 * [Kata monitor metrics](#kata-monitor-metrics)
@@ -198,6 +185,30 @@ Agent's metrics contains metrics about agent process.
 | `kata_agent_total_time`: <br> Agent process total time | `GAUGE` |  | <ul><li>`sandbox_id`</li></ul> | 2.0.0 |
 | `kata_agent_total_vm`: <br> Agent process total `vm` size | `GAUGE` |  | <ul><li>`sandbox_id`</li></ul> | 2.0.0 |
 
+### Firecracker metrics
+
+Metrics for Firecracker vmm.
+
+| Metric name | Type | Units | Labels | Introduced in Kata version |
+|---|---|---|---|---|
+| `kata_firecracker_api_server`: <br> Metrics related to the internal API server. | `GAUGE` |  | <ul><li>`item`<ul><li>`process_startup_time_cpu_us`</li><li>`process_startup_time_us`</li><li>`sync_response_fails`</li><li>`sync_vmm_send_timeout_count`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
+| `kata_firecracker_block`: <br> Block Device associated metrics. | `GAUGE` |  | <ul><li>`item`<ul><li>`activate_fails`</li><li>`cfg_fails`</li><li>`event_fails`</li><li>`execute_fails`</li><li>`flush_count`</li><li>`invalid_reqs_count`</li><li>`no_avail_buffer`</li><li>`queue_event_count`</li><li>`rate_limiter_event_count`</li><li>`rate_limiter_throttled_events`</li><li>`read_bytes`</li><li>`read_count`</li><li>`update_count`</li><li>`update_fails`</li><li>`write_bytes`</li><li>`write_count`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
+| `kata_firecracker_get_api_requests`: <br> Metrics specific to GET API Requests for counting user triggered actions and/or failures. | `GAUGE` |  | <ul><li>`item`<ul><li>`instance_info_count`</li><li>`instance_info_fails`</li><li>`machine_cfg_count`</li><li>`machine_cfg_fails`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
+| `kata_firecracker_i8042`: <br> Metrics specific to the i8042 device. | `GAUGE` |  | <ul><li>`item`<ul><li>`error_count`</li><li>`missed_read_count`</li><li>`missed_write_count`</li><li>`read_count`</li><li>`reset_count`</li><li>`write_count`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
+| `kata_firecracker_latencies_us`: <br> Performance metrics related for the moment only to snapshots. | `GAUGE` |  | <ul><li>`item`<ul><li>`diff_create_snapshot`</li><li>`full_create_snapshot`</li><li>`load_snapshot`</li><li>`pause_vm`</li><li>`resume_vm`</li><li>`vmm_diff_create_snapshot`</li><li>`vmm_full_create_snapshot`</li><li>`vmm_load_snapshot`</li><li>`vmm_pause_vm`</li><li>`vmm_resume_vm`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
+| `kata_firecracker_logger`: <br> Metrics for the logging subsystem. | `GAUGE` |  | <ul><li>`item`<ul><li>`log_fails`</li><li>`metrics_fails`</li><li>`missed_log_count`</li><li>`missed_metrics_count`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
+| `kata_firecracker_mmds`: <br> Metrics for the MMDS functionality. | `GAUGE` |  | <ul><li>`item`<ul><li>`connections_created`</li><li>`connections_destroyed`</li><li>`rx_accepted`</li><li>`rx_accepted_err`</li><li>`rx_accepted_unusual`</li><li>`rx_bad_eth`</li><li>`rx_count`</li><li>`tx_bytes`</li><li>`tx_count`</li><li>`tx_errors`</li><li>`tx_frames`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
+| `kata_firecracker_net`: <br> Network-related metrics. | `GAUGE` |  | <ul><li>`item`<ul><li>`activate_fails`</li><li>`cfg_fails`</li><li>`event_fails`</li><li>`mac_address_updates`</li><li>`no_rx_avail_buffer`</li><li>`no_tx_avail_buffer`</li><li>`rx_bytes_count`</li><li>`rx_count`</li><li>`rx_event_rate_limiter_count`</li><li>`rx_fails`</li><li>`rx_packets_count`</li><li>`rx_partial_writes`</li><li>`rx_queue_event_count`</li><li>`rx_rate_limiter_throttled`</li><li>`rx_tap_event_count`</li><li>`tap_read_fails`</li><li>`tap_write_fails`</li><li>`tx_bytes_count`</li><li>`tx_count`</li><li>`tx_fails`</li><li>`tx_malformed_frames`</li><li>`tx_packets_count`</li><li>`tx_partial_reads`</li><li>`tx_queue_event_count`</li><li>`tx_rate_limiter_event_count`</li><li>`tx_rate_limiter_throttled`</li><li>`tx_spoofed_mac_count`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
+| `kata_firecracker_patch_api_requests`: <br> Metrics specific to PATCH API Requests for counting user triggered actions and/or failures. | `GAUGE` |  | <ul><li>`item`<ul><li>`drive_count`</li><li>`drive_fails`</li><li>`machine_cfg_count`</li><li>`machine_cfg_fails`</li><li>`network_count`</li><li>`network_fails`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
+| `kata_firecracker_put_api_requests`: <br> Metrics specific to PUT API Requests for counting user triggered actions and/or failures. | `GAUGE` |  | <ul><li>`item`<ul><li>`actions_count`</li><li>`actions_fails`</li><li>`boot_source_count`</li><li>`boot_source_fails`</li><li>`drive_count`</li><li>`drive_fails`</li><li>`logger_count`</li><li>`logger_fails`</li><li>`machine_cfg_count`</li><li>`machine_cfg_fails`</li><li>`metrics_count`</li><li>`metrics_fails`</li><li>`network_count`</li><li>`network_fails`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
+| `kata_firecracker_rtc`: <br> Metrics specific to the RTC device. | `GAUGE` |  | <ul><li>`item`<ul><li>`error_count`</li><li>`missed_read_count`</li><li>`missed_write_count`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
+| `kata_firecracker_seccomp`: <br> Metrics for the seccomp filtering. | `GAUGE` |  | <ul><li>`item`<ul><li>`num_faults`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
+| `kata_firecracker_signals`: <br> Metrics related to signals. | `GAUGE` |  | <ul><li>`item`<ul><li>`sigbus`</li><li>`sigsegv`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
+| `kata_firecracker_uart`: <br> Metrics specific to the UART device. | `GAUGE` |  | <ul><li>`item`<ul><li>`error_count`</li><li>`flush_count`</li><li>`missed_read_count`</li><li>`missed_write_count`</li><li>`read_count`</li><li>`write_count`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
+| `kata_firecracker_vcpu`: <br> Metrics specific to VCPUs' mode of functioning. | `GAUGE` |  | <ul><li>`item`<ul><li>`exit_io_in`</li><li>`exit_io_out`</li><li>`exit_mmio_read`</li><li>`exit_mmio_write`</li><li>`failures`</li><li>`filter_cpuid`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
+| `kata_firecracker_vmm`: <br> Metrics specific to the machine manager as a whole. | `GAUGE` |  | <ul><li>`item`<ul><li>`device_events`</li><li>`panic_count`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
+| `kata_firecracker_vsock`: <br> Vsock-related metrics. | `GAUGE` |  | <ul><li>`item`<ul><li>`activate_fails`</li><li>`cfg_fails`</li><li>`conn_event_fails`</li><li>`conns_added`</li><li>`conns_killed`</li><li>`conns_removed`</li><li>`ev_queue_event_fails`</li><li>`killq_resync`</li><li>`muxer_event_fails`</li><li>`rx_bytes_count`</li><li>`rx_packets_count`</li><li>`rx_queue_event_count`</li><li>`rx_queue_event_fails`</li><li>`rx_read_fails`</li><li>`tx_bytes_count`</li><li>`tx_flush_fails`</li><li>`tx_packets_count`</li><li>`tx_queue_event_count`</li><li>`tx_queue_event_fails`</li><li>`tx_write_fails`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
+
 ### Kata guest OS metrics
 
 Guest OS's metrics in hypervisor.

docs/design/kata-api-design.md

@@ -1,9 +1,9 @@
 # Kata API Design
+
 To fulfill the [Kata design requirements](kata-design-requirements.md), and based on the discussion on [Virtcontainers API extensions](https://docs.google.com/presentation/d/1dbGrD1h9cpuqAPooiEgtiwWDGCYhVPdatq7owsKHDEQ), the Kata runtime library features the following APIs:
 -  Sandbox based top API
 -  Storage and network hotplug API
 -  Plugin frameworks for external proprietary Kata runtime extensions
--  Built-in shim and proxy types and capabilities
 
 ## Sandbox Based API
 ### Sandbox Management API
@@ -23,17 +23,17 @@ To fulfill the [Kata design requirements](kata-design-requirements.md), and base
 |`sandbox.Stats()`| Get the stats of a running sandbox, return a `SandboxStats` structure.|
 |`sandbox.Status()`| Get the status of the sandbox and containers, return a `SandboxStatus` structure.|
 |`sandbox.Stop(force)`| Stop a sandbox and Destroy the containers in the sandbox. When force is true, ignore guest related stop failures.|
-|`sandbox.CreateContainer(contConfig)`| Create new container in the sandbox with the `ContainerConfig` param. It will add new container config to `sandbox.config.Containers`.|
-|`sandbox.DeleteContainer(containerID)`| Delete a container from the sandbox by containerID, return a `Container` structure.|
+|`sandbox.CreateContainer(contConfig)`| Create new container in the sandbox with the `ContainerConfig` parameter. It will add new container config to `sandbox.config.Containers`.|
+|`sandbox.DeleteContainer(containerID)`| Delete a container from the sandbox by `containerID`, return a `Container` structure.|
 |`sandbox.EnterContainer(containerID, cmd)`| Run a new process in a container, executing customer's `types.Cmd` command.|
-|`sandbox.KillContainer(containerID, signal, all)`| Signal a container in the sandbox by the containerID.|
-|`sandbox.PauseContainer(containerID)`| Pause a running container in the sandbox by the containerID.|
+|`sandbox.KillContainer(containerID, signal, all)`| Signal a container in the sandbox by the `containerID`.|
+|`sandbox.PauseContainer(containerID)`| Pause a running container in the sandbox by the `containerID`.|
 |`sandbox.ProcessListContainer(containerID, options)`| List every process running inside a specific container in the sandbox, return a `ProcessList` structure.|
-|`sandbox.ResumeContainer(containerID)`| Resume a paused container in the sandbox by the containerID.|
-|`sandbox.StartContainer(containerID)`| Start a container in the sandbox by the containerID.|
+|`sandbox.ResumeContainer(containerID)`| Resume a paused container in the sandbox by the `containerID`.|
+|`sandbox.StartContainer(containerID)`| Start a container in the sandbox by the `containerID`.|
 |`sandbox.StatsContainer(containerID)`| Get the stats of a running container, return a `ContainerStats` structure.|
 |`sandbox.StatusContainer(containerID)`| Get the status of a container in the sandbox, return a `ContainerStatus` structure.|
-|`sandbox.StopContainer(containerID, force)`| Stop a container in the sandbox by the containerID.|
+|`sandbox.StopContainer(containerID, force)`| Stop a container in the sandbox by the `containerID`.|
 |`sandbox.UpdateContainer(containerID, resources)`| Update a running container in the sandbox.|
 |`sandbox.WaitProcess(containerID, processID)`| Wait on a process to terminate.|
 ### Sandbox Hotplug API
@@ -57,7 +57,7 @@ To fulfill the [Kata design requirements](kata-design-requirements.md), and base
 |Name|Description|
 |---|---|
 |`sandbox.GetOOMEvent()`| Monitor the OOM events that occur in the sandbox..|
-|`sandbox.UpdateRuntimeMetrics()`| Update the shim/hypervisor's metrics of the running sandbox.|
+|`sandbox.UpdateRuntimeMetrics()`| Update the `shim/hypervisor` metrics of the running sandbox.|
 |`sandbox.GetAgentMetrics()`| Get metrics of the agent and the guest in the running sandbox.|
 
 ## Plugin framework for external proprietary Kata runtime extensions
@@ -99,32 +99,3 @@ Built-in implementations include:
 
 ### Sandbox Connection Plugin Workflow
 ![Sandbox Connection Plugin Workflow](https://raw.githubusercontent.com/bergwolf/raw-contents/master/kata/Sandbox-Connection.png "Sandbox Connection Plugin Workflow")
-
-## Built-in Shim and Proxy Types and Capabilities
-### Built-in shim/proxy sandbox configurations
--  Supported shim configurations:
-
-|Name|Description|
-|---|---|
-|`noopshim`|Do not start any shim process.|
-|`ccshim`| Start the cc-shim binary.|
-|`katashim`| Start the `kata-shim` binary.|
-|`katashimbuiltin`|No standalone shim process but shim functionality APIs are exported.|
--  Supported proxy configurations:
-
-|Name|Description|
-|---|---|
-|`noopProxy`| a dummy proxy implementation of the proxy interface, only used for testing purpose.|
-|`noProxy`|generic implementation for any case where no actual proxy is needed.|
-|`ccProxy`|run `ccProxy` to proxy between runtime and agent.|
-|`kataProxy`|run `kata-proxy` to translate Yamux connections between runtime and Kata agent. |
-|`kataProxyBuiltin`| no standalone proxy process and connect to Kata agent with internal Yamux translation.|
-
-### Built-in Shim Capability
-Built-in shim capability is implemented by removing standalone shim process, and
-supporting the shim related APIs.
-
-### Built-in Proxy Capability
-Built-in proxy capability is achieved by removing standalone proxy process, and
-connecting to Kata agent with a custom gRPC dialer that is internal Yamux translation.
-The behavior is enabled when proxy is configured as `kataProxyBuiltin`.

docs/design/proposals/README.md

@@ -0,0 +1,5 @@
+# Design proposals
+
+Kata Containers design proposal documents:
+
+- [Kata Containers tracing](tracing-proposals.md)

docs/design/proposals/tracing-proposals.md

@@ -0,0 +1,213 @@
+# Kata Tracing proposals
+
+## Overview
+
+This document summarises a set of proposals triggered by the
+[tracing documentation PR][tracing-doc-pr].
+
+## Required context
+
+This section explains some terminology required to understand the proposals.
+Further details can be found in the
+[tracing documentation PR][tracing-doc-pr].
+
+### Agent trace mode terminology
+
+| Trace mode | Description | Use-case |
+|-|-|-|
+| Static |  Trace agent from startup to shutdown | Entire lifespan |
+| Dynamic | Toggle tracing on/off as desired | On-demand "snapshot" |
+
+### Agent trace type terminology
+
+| Trace type | Description | Use-case |
+|-|-|-|
+| isolated | traces all relate to single component | Observing lifespan |
+| collated | traces "grouped" (runtime+agent) | Understanding component interaction |
+
+### Container lifespan
+
+| Lifespan | trace mode | trace type |
+|-|-|-|
+| short-lived | static | collated if possible, else isolated? |
+| long-running | dynamic | collated? (to see interactions) |
+
+## Original plan for agent
+
+- Implement all trace types and trace modes for agent.
+
+- Why?
+  - Maximum flexibility.
+
+    > **Counterargument:**
+    >
+    > Due to the intrusive nature of adding tracing, we have
+    > learnt that landing small incremental changes is simpler and quicker!
+
+  - Compatibility with [Kata 1.x tracing][kata-1x-tracing].
+
+    > **Counterargument:**
+    >
+    > Agent tracing in Kata 1.x was extremely awkward to setup (to the extent
+    > that it's unclear how many users actually used it!)
+    >
+    > This point, coupled with the new architecture for Kata 2.x, suggests
+    > that we may not need to supply the same set of tracing features (in fact
+    > they may not make sense)).
+
+## Agent tracing proposals
+
+### Agent tracing proposal 1: Don't implement dynamic trace mode
+
+- All tracing will be static.
+
+- Why?
+  - Because dynamic tracing will always be "partial"
+
+    > In fact, not only would it be only a "snapshot" of activity, it may not
+    > even be possible to create a complete "trace transaction". If this is
+    > true, the trace output would be partial and would appear "unstructured".
+
+### Agent tracing proposal 2: Simplify handling of trace type
+
+- Agent tracing will be "isolated" by default.
+- Agent tracing will be "collated" if runtime tracing is also enabled.
+
+- Why?
+  - Offers a graceful fallback for agent tracing if runtime tracing disabled.
+  - Simpler code!
+
+## Questions to ask yourself (part 1)
+
+- Are your containers long-running or short-lived?
+
+- Would you ever need to turn on tracing "briefly"?
+  - If "yes", is a "partial trace" useful or useless?
+
+    > Likely to be considered useless as it is a partial snapshot.
+    > Alternative tracing methods may be more appropriate to dynamic
+    > OpenTelemetry tracing.
+
+## Questions to ask yourself (part 2)
+
+- Are you happy to stop a container to enable tracing?
+  If "no", dynamic tracing may be required.
+
+- Would you ever want to trace the agent and the runtime "in isolation" at the
+  same time?
+  - If "yes", we need to fully implement `trace_mode=isolated`
+
+    > This seems unlikely though.
+
+## Trace collection
+
+The second set of proposals affect the way traces are collected.
+
+### Motivation
+
+Currently:
+
+- The runtime sends trace spans to Jaeger directly.
+- The agent will send trace spans to the [`trace-forwarder`][trace-forwarder] component.
+- The trace forwarder will send trace spans to Jaeger.
+
+Kata agent tracing overview:
+
+```
++-------------------------------------------+
+| Host                                      |
+|                                           |
+| +-----------+                             |
+| | Trace     |                             |
+| | Collector |                             |
+| +-----+-----+                             |
+|       ^                  +--------------+ |
+|       | spans            | Kata VM      | |
+| +-----+-----+            |              | |
+| | Kata      |    spans   |     +-----+  | |
+| | Trace     |<-----------------|Kata |  | |
+| | Forwarder |    VSOCK   |     |Agent|  | |
+| +-----------+    Channel |     +-----+  | |
+|                          +--------------+ |
++-------------------------------------------+
+```
+
+Currently:
+
+- If agent tracing is enabled but the trace forwarder is not running,
+  the agent will error.
+
+- If the trace forwarder is started but Jaeger is not running,
+  the trace forwarder will error.
+
+### Goals
+
+- The runtime and agent should:
+  - Use the same trace collection implementation.
+  - Use the most the common configuration items.
+
+- Kata should should support more trace collection software or `SaaS`
+  (for example `Zipkin`, `datadog`).
+
+- Trace collection should not block normal runtime/agent operations
+  (for example if `vsock-exporter`/Jaeger is not running, Kata Containers should work normally).
+
+### Trace collection proposals
+
+#### Trace collection proposal 1: Send all spans to the trace forwarder as a span proxy
+
+Kata runtime/agent all send spans to trace forwarder, and the trace forwarder,
+acting as a tracing proxy, sends all spans to a tracing back-end, such as Jaeger or `datadog`.
+
+**Pros:**
+
+- Runtime/agent will be simple.
+- Could update trace collection target while Kata Containers are running.
+
+**Cons:**
+
+- Requires the trace forwarder component to be running (that is a pressure to operation).
+
+#### Trace collection proposal 2: Send spans to collector directly from runtime/agent
+
+Send spans to collector directly from runtime/agent, this proposal need
+network accessible to the collector.
+
+**Pros:**
+
+- No additional trace forwarder component needed.
+
+**Cons:**
+
+- Need more code/configuration to support all trace collectors.
+
+## Future work
+
+- We could add dynamic and fully isolated tracing at a later stage,
+  if required.
+
+## Further details
+
+- See the new [GitHub project](https://github.com/orgs/kata-containers/projects/28).
+- [kata-containers-tracing-status](https://gist.github.com/jodh-intel/0ee54d41d2a803ba761e166136b42277) gist.
+- [tracing documentation PR][tracing-doc-pr].
+
+## Summary
+
+### Time line
+
+- 2021-07-01: A summary of the discussion was
+  [posted to the mail list](http://lists.katacontainers.io/pipermail/kata-dev/2021-July/001996.html).
+- 2021-06-22: These proposals were
+  [discussed in the Kata Architecture Committee meeting](https://etherpad.opendev.org/p/Kata_Containers_2021_Architecture_Committee_Mtgs).
+- 2021-06-18: These proposals where
+  [announced on the mailing list](http://lists.katacontainers.io/pipermail/kata-dev/2021-June/001980.html).
+
+### Outcome
+
+- Nobody opposed the agent proposals, so they are being implemented.
+- The trace collection proposals are still being considered.
+
+[kata-1x-tracing]: https://github.com/kata-containers/agent/blob/master/TRACING.md
+[trace-forwarder]: /src/trace-forwarder
+[tracing-doc-pr]: https://github.com/kata-containers/kata-containers/pull/1937

docs/design/vcpu-handling.md

@@ -1,11 +1,3 @@
-- [Virtual machine vCPU sizing in Kata Containers](#virtual-machine-vcpu-sizing-in-kata-containers)
-  * [Default number of virtual CPUs](#default-number-of-virtual-cpus)
-  * [Virtual CPUs and Kubernetes pods](#virtual-cpus-and-kubernetes-pods)
-  * [Container lifecycle](#container-lifecycle)
-  * [Container without CPU constraint](#container-without-cpu-constraint)
-  * [Container with CPU constraint](#container-with-cpu-constraint)
-  * [Do not waste resources](#do-not-waste-resources)
-
 # Virtual machine vCPU sizing in Kata Containers
 
 ## Default number of virtual CPUs

docs/design/virtualization.md

@@ -1,16 +1,5 @@
 # Virtualization in Kata Containers
 
-- [Virtualization in Kata Containers](#virtualization-in-kata-containers)
-  - [Mapping container concepts to virtual machine technologies](#mapping-container-concepts-to-virtual-machine-technologies)
-  - [Kata Containers Hypervisor and VMM support](#kata-containers-hypervisor-and-vmm-support)
-    - [QEMU/KVM](#qemukvm)
-      - [Machine accelerators](#machine-accelerators)
-      - [Hotplug devices](#hotplug-devices)
-    - [Firecracker/KVM](#firecrackerkvm)
-    - [Cloud Hypervisor/KVM](#cloud-hypervisorkvm)
-    - [Summary](#summary)
-
-
 Kata Containers, a second layer of isolation is created on top of those provided by traditional namespace-containers. The
 hardware virtualization interface is the basis of this additional layer. Kata will launch a lightweight virtual machine,
 and use the guest’s Linux kernel to create a container workload, or workloads in the case of multi-container pods. In Kubernetes
@@ -22,10 +11,10 @@ the multiple hypervisors and virtual machine monitors that Kata supports.
 ## Mapping container concepts to virtual machine technologies
 
 A typical deployment of Kata Containers will be in Kubernetes by way of a Container Runtime Interface (CRI) implementation. On every node,
-Kubelet will interact with a CRI implementor (such as containerd or CRI-O), which will in turn interface with Kata Containers (an OCI based runtime).
+Kubelet will interact with a CRI implementer (such as containerd or CRI-O), which will in turn interface with Kata Containers (an OCI based runtime).
 
 The CRI API, as defined at the [Kubernetes CRI-API repo](https://github.com/kubernetes/cri-api/), implies a few constructs being supported by the
-CRI implementation, and ultimately in Kata Containers. In order to support the full [API](https://github.com/kubernetes/cri-api/blob/a6f63f369f6d50e9d0886f2eda63d585fbd1ab6a/pkg/apis/runtime/v1alpha2/api.proto#L34-L110) with the CRI-implementor, Kata must provide the following constructs:
+CRI implementation, and ultimately in Kata Containers. In order to support the full [API](https://github.com/kubernetes/cri-api/blob/a6f63f369f6d50e9d0886f2eda63d585fbd1ab6a/pkg/apis/runtime/v1alpha2/api.proto#L34-L110) with the CRI-implementer, Kata must provide the following constructs:
 
 ![API to construct](./arch-images/api-to-construct.png)
 
@@ -41,14 +30,9 @@ Each hypervisor or VMM varies on how or if it handles each of these.
 
 ## Kata Containers Hypervisor and VMM support
 
-Kata Containers is designed to support multiple virtual machine monitors (VMMs) and hypervisors.
-Kata Containers supports:
- - [ACRN hypervisor](https://projectacrn.org/)
- - [Cloud Hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor)/[KVM](https://www.linux-kvm.org/page/Main_Page)
- - [Firecracker](https://github.com/firecracker-microvm/firecracker)/KVM
- - [QEMU](http://www.qemu-project.org/)/KVM
+Kata Containers [supports multiple hypervisors](../hypervisors.md).
 
-Which configuration to use will depend on the end user's requirements. Details of each solution and a summary are provided below.
+Details of each solution and a summary are provided below.
 
 ### QEMU/KVM
 
@@ -62,7 +46,7 @@ be changed by editing the runtime [`configuration`](./architecture.md/#configura
 Devices and features used:
 - virtio VSOCK or virtio serial
 - virtio block or virtio SCSI
-- virtio net
+- [virtio net](https://www.redhat.com/en/virtio-networking-series)
 - virtio fs or virtio 9p (recommend: virtio fs)
 - VFIO
 - hotplug
@@ -105,25 +89,34 @@ Devices used:
 
 ### Cloud Hypervisor/KVM
 
-Cloud Hypervisor, based on [rust-VMM](https://github.com/rust-vmm), is designed to have a lighter footprint and attack surface. For Kata Containers,
-relative to Firecracker, the Cloud Hypervisor configuration provides better compatibility at the expense of exposing additional devices: file system
-sharing and direct device assignment.  As of the 1.10 release of Kata Containers, Cloud Hypervisor does not support device hotplug, and as a result
-does not support updating container resources after boot, or utilizing block based volumes. While Cloud Hypervisor does support VFIO, Kata is still adding
-this support. As of 1.10, Kata does not support block based volumes or direct device assignment. See [Cloud Hypervisor device support documentation](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/device_model.md)
-for more details on Cloud Hypervisor.
+[Cloud Hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor), based
+on [rust-vmm](https://github.com/rust-vmm), is designed to have a
+lighter footprint and smaller attack surface for running modern cloud
+workloads. Kata Containers with Cloud
+Hypervisor provides mostly complete compatibility with Kubernetes
+comparable to the QEMU configuration. As of the 1.12 and 2.0.0 release
+of Kata Containers, the Cloud Hypervisor configuration supports both CPU
+and memory resize, device hotplug (disk and VFIO), file-system sharing through virtio-fs,
+block-based volumes, booting from VM images backed by pmem device, and
+fine-grained seccomp filters for each VMM threads (e.g. all virtio
+device worker threads). Please check [this GitHub Project](https://github.com/orgs/kata-containers/projects/21)
+for details of ongoing integration efforts.
 
-Devices used:
-- virtio VSOCK
+Devices and features used:
+- virtio VSOCK or virtio serial
 - virtio block
 - virtio net
 - virtio fs
+- virtio pmem
+- VFIO
+- hotplug
+- seccomp filters
+- [HTTP OpenAPI](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/vmm/src/api/openapi/cloud-hypervisor.yaml)
 
 ### Summary
 
 | Solution | release introduced | brief summary |
 |-|-|-|
-| QEMU | 1.0 | upstream QEMU, with support for hotplug and filesystem sharing |
-| NEMU | 1.4 | Deprecated, removed as of 1.10 release. Slimmed down fork of QEMU, with experimental support of virtio-fs |
+| Cloud Hypervisor | 1.10 | upstream Cloud Hypervisor with rich feature support, e.g. hotplug, VFIO and FS sharing|
 | Firecracker | 1.5 | upstream Firecracker, rust-VMM based, no VFIO, no FS sharing, no memory/CPU hotplug |
-| QEMU-virtio-fs | 1.7 | upstream QEMU with support for virtio-fs. Will be removed once virtio-fs lands in upstream QEMU |
-| Cloud Hypervisor | 1.10 |  rust-VMM based, includes VFIO and FS sharing through virtio-fs, no hotplug |
+| QEMU | 1.0 | upstream QEMU, with support for hotplug and filesystem sharing |

docs/how-to/README.md

@@ -1,11 +1,8 @@
 # Howto Guides
 
-* [Howto Guides](#howto-guides)
-  * [Kubernetes Integration](#kubernetes-integration)
-  * [Hypervisors Integration](#hypervisors-integration)
-  * [Advanced Topics](#advanced-topics)
-
 ## Kubernetes Integration
+
+- [Run Kata containers with `crictl`](run-kata-with-crictl.md)
 - [Run Kata Containers with Kubernetes](run-kata-with-k8s.md)
 - [How to use Kata Containers and Containerd](containerd-kata.md)
 - [How to use Kata Containers and CRI (containerd plugin) with Kubernetes](how-to-use-k8s-with-cri-containerd-and-kata.md)
@@ -13,11 +10,21 @@
 - [How to import Kata Containers logs into Fluentd](how-to-import-kata-logs-with-fluentd.md)
 
 ## Hypervisors Integration
+
+  Currently supported hypervisors with Kata Containers include:
+- `qemu`
+- `cloud-hypervisor`
+- `firecracker`
+- `ACRN`
+
+  While `qemu` and `cloud-hypervisor` work out of the box with installation of Kata,
+  some additional configuration is needed in case of `firecracker` and `ACRN`.
+  Refer to the following guides for additional configuration steps:
 - [Kata Containers with Firecracker](https://github.com/kata-containers/documentation/wiki/Initial-release-of-Kata-Containers-with-Firecracker-support)
-- [Kata Containers with NEMU](how-to-use-kata-containers-with-nemu.md)
 - [Kata Containers with ACRN Hypervisor](how-to-use-kata-containers-with-acrn.md)
 
 ## Advanced Topics
+
 - [How to use Kata Containers with virtio-fs](how-to-use-virtio-fs-with-kata.md)
 - [Setting Sysctls with Kata](how-to-use-sysctls-with-kata.md)
 - [What Is VMCache and How To Enable It](what-is-vm-cache-and-how-do-I-use-it.md)
@@ -27,3 +34,4 @@
 - [How to use Kata Containers with `virtio-mem`](how-to-use-virtio-mem-with-kata.md)
 - [How to set sandbox Kata Containers configurations with pod annotations](how-to-set-sandbox-config-kata.md)
 - [How to monitor Kata Containers in K8s](how-to-set-prometheus-in-k8s.md)
+- [How to use hotplug memory on arm64 in Kata Containers](how-to-hotplug-memory-arm64.md)

docs/how-to/containerd-kata.md

@@ -1,23 +1,5 @@
 # How to use Kata Containers and Containerd
 
-- [Concepts](#concepts)
-    - [Kubernetes `RuntimeClass`](#kubernetes-runtimeclass)
-    - [Containerd Runtime V2 API: Shim V2 API](#containerd-runtime-v2-api-shim-v2-api)
-- [Install](#install)
-    - [Install Kata Containers](#install-kata-containers)
-    - [Install containerd with CRI plugin](#install-containerd-with-cri-plugin)
-    - [Install CNI plugins](#install-cni-plugins)
-    - [Install `cri-tools`](#install-cri-tools)
-- [Configuration](#configuration)
-    - [Configure containerd to use Kata Containers](#configure-containerd-to-use-kata-containers)
-        - [Kata Containers as a `RuntimeClass`](#kata-containers-as-a-runtimeclass)
-        - [Kata Containers as the runtime for untrusted workload](#kata-containers-as-the-runtime-for-untrusted-workload)
-    - [Kata Containers as the default runtime](#kata-containers-as-the-default-runtime)
-    - [Configuration for `cri-tools`](#configuration-for-cri-tools)
-- [Run](#run)
-    - [Launch containers with `ctr` command line](#launch-containers-with-ctr-command-line)
-    - [Launch Pods with `crictl` command line](#launch-pods-with-crictl-command-line)
-
 This document covers the installation and configuration of [containerd](https://containerd.io/) 
 and [Kata Containers](https://katacontainers.io). The containerd provides not only the `ctr`
 command line tool, but also the [CRI](https://kubernetes.io/blog/2016/12/container-runtime-interface-cri-in-kubernetes/) 
@@ -193,13 +175,16 @@ From Containerd v1.2.4 and Kata v1.6.0, there is a new runtime option supported,
 ```toml
       [plugins.cri.containerd.runtimes.kata]
          runtime_type = "io.containerd.kata.v2"
+	 privileged_without_host_devices = true
 	 [plugins.cri.containerd.runtimes.kata.options]
 	   ConfigPath = "/etc/kata-containers/config.toml"
 ```
 
+`privileged_without_host_devices` tells containerd that a privileged Kata container should not have direct access to all host devices. If unset, containerd will pass all host devices to Kata container, which may cause security issues.
+
 This `ConfigPath` option is optional. If you do not specify it, shimv2 first tries to get the configuration file from the environment variable `KATA_CONF_FILE`. If neither are set, shimv2 will use the default Kata configuration file paths (`/etc/kata-containers/configuration.toml` and `/usr/share/defaults/kata-containers/configuration.toml`).
 
-If you use Containerd older than v1.2.4 or a version of Kata older than v1.6.0  and also want to specify a configuration file, you can use the following workaround, since the shimv2 accepts an environment variable, `KATA_CONF_FILE` for the configuration file path. Then, you can create a 
+If you use Containerd older than v1.2.4 or a version of Kata older than v1.6.0  and also want to specify a configuration file, you can use the following workaround, since the shimv2 accepts an environment variable, `KATA_CONF_FILE` for the configuration file path. Then, you can create a
 shell script with the following:
 
 ```bash

docs/how-to/data/crictl/busybox/container_config.json

@@ -0,0 +1,19 @@
+{
+	"metadata": {
+		"name": "busybox-container",
+		"namespace": "test.kata"
+	},
+	"image": {
+		"image": "docker.io/library/busybox:latest"
+	},
+	"command": [
+		"sleep",
+		"9999"
+	],
+	"args": [],
+	"working_dir": "/",
+	"log_path": "",
+	"stdin": false,
+	"stdin_once": false,
+	"tty": false
+}

docs/how-to/data/crictl/busybox/sandbox_config.json

@@ -0,0 +1,20 @@
+{
+	"metadata": {
+		"name": "busybox-pod",
+		"uid": "busybox-pod",
+		"namespace": "test.kata"
+	},
+	"hostname": "busybox_host",
+	"log_directory": "",
+	"dns_config": {
+	},
+	"port_mappings": [],
+	"resources": {
+	},
+	"labels": {
+	},
+	"annotations": {
+	},
+	"linux": {
+	}
+}

docs/how-to/data/crictl/redis/redis_client_container_config.json

@@ -0,0 +1,39 @@
+{
+	"metadata": {
+		"name": "redis-client",
+		"namespace": "test.kata"
+	},
+	"image": {
+		"image": "docker.io/library/redis:6.0.8-alpine"
+	},
+	"command": [
+		  "tail", "-f", "/dev/null"
+	],
+	"envs": [
+		{
+			"key": "PATH",
+			"value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+		},
+		{
+			"key": "TERM",
+			"value": "xterm"
+		}
+	],
+	"labels": {
+		"tier": "backend"
+	},
+	"annotations": {
+		"pod": "redis-client-pod"
+	},
+	"log_path": "",
+	"stdin": false,
+	"stdin_once": false,
+	"tty": false,
+	"linux": {
+		"resources": {
+			"memory_limit_in_bytes": 524288000
+		},
+		"security_context": {
+		}
+	}
+}

docs/how-to/data/crictl/redis/redis_client_sandbox_config.json

@@ -0,0 +1,28 @@
+{
+	"metadata": {
+		"name": "redis-client-pod",
+		"uid": "test-redis-client-pod",
+		"namespace": "test.kata"
+	},
+	"hostname": "redis-client",
+	"log_directory": "",
+	"dns_config": {
+		"searches": [
+			"8.8.8.8"
+		]
+	},
+	"port_mappings": [],
+	"resources": {
+		"cpu": {
+			"limits": 1,
+			"requests": 1
+		}
+	},
+	"labels": {
+		"tier": "backend"
+	},
+	"annotations": {
+	},
+	"linux": {
+	}
+}

docs/how-to/data/crictl/redis/redis_server_container_config.json

@@ -0,0 +1,36 @@
+{
+	"metadata": {
+		"name": "redis-server",
+		"namespace": "test.kata"
+	},
+	"image": {
+		"image": "docker.io/library/redis:6.0.8-alpine"
+	},
+	"envs": [
+		{
+			"key": "PATH",
+			"value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+		},
+		{
+			"key": "TERM",
+			"value": "xterm"
+		}
+	],
+	"labels": {
+		"tier": "backend"
+	},
+	"annotations": {
+		"pod": "redis-server-pod"
+	},
+	"log_path": "",
+	"stdin": false,
+	"stdin_once": false,
+	"tty": false,
+	"linux": {
+		"resources": {
+			"memory_limit_in_bytes": 524288000
+		},
+		"security_context": {
+		}
+	}
+}

docs/how-to/data/crictl/redis/redis_server_sandbox_config.json

@@ -0,0 +1,28 @@
+{
+	"metadata": {
+		"name": "redis-server-pod",
+		"uid": "test-redis-server-pod",
+		"namespace": "test.kata"
+	},
+	"hostname": "redis-server",
+	"log_directory": "",
+	"dns_config": {
+		"searches": [
+			"8.8.8.8"
+		]
+	},
+	"port_mappings": [],
+	"resources": {
+		"cpu": {
+			"limits": 1,
+			"requests": 1
+		}
+	},
+	"labels": {
+		"tier": "backend"
+	},
+	"annotations": {
+	},
+	"linux": {
+	}
+}

docs/how-to/data/kata-monitor-daemonset.yml

@@ -26,7 +26,7 @@ spec:
       hostNetwork: true
       containers:
       - name: kata-monitor
-        image: docker.io/katadocker/kata-monitor:2.0.0
+        image: quay.io/kata-containers/kata-monitor:2.0.0
         args: 
           - -log-level=debug
         ports:

docs/how-to/how-to-hotplug-memory-arm64.md

@@ -0,0 +1,28 @@
+# How to use memory hotplug feature in Kata Containers on arm64
+
+## Introduction
+
+Memory hotplug is a key feature for containers to allocate memory dynamically in deployment.
+As Kata Container bases on VM, this feature needs support both from VMM and guest kernel. Luckily, it has been fully supported for the current default version of QEMU and guest kernel used by Kata on arm64. For other VMMs, e.g, Cloud Hypervisor, the enablement work is on the road. Apart from VMM and guest kernel, memory hotplug also depends on ACPI which depends on firmware either. On x86, you can boot a VM using QEMU with ACPI enabled directly, because it boots up with firmware implicitly. For arm64, however, you need specify firmware explicitly. That is to say, if you are ready to run a normal Kata Container on arm64, what you need extra to do is to install the UEFI ROM before use the memory hotplug feature.
+
+## Install UEFI ROM
+
+We have offered a helper script for you to install the UEFI ROM. If you have installed Kata normally on your host, you just need to run the script as fellows:
+
+```bash
+$ pushd $GOPATH/src/github.com/kata-containers/tests
+$ sudo .ci/aarch64/install_rom_aarch64.sh
+$ popd
+```
+
+## Run for test
+
+Let's test if the memory hotplug is ready for Kata after install the UEFI ROM. Make sure containerd is ready to run Kata before test.
+
+```bash
+$ sudo ctr image pull docker.io/library/ubuntu:latest
+$ sudo ctr run --runtime io.containerd.run.kata.v2 -t --rm docker.io/library/ubuntu:latest hello sh -c "free -h"
+$ sudo ctr run --runtime io.containerd.run.kata.v2 -t --memory-limit 536870912 --rm docker.io/library/ubuntu:latest hello sh -c "free -h"
+```
+
+Compare the results between the two tests. If the latter is 0.5G larger than the former, you have done what you want, and congratulation!

docs/how-to/how-to-import-kata-logs-with-fluentd.md

@@ -1,20 +1,5 @@
 # Importing Kata Containers logs with Fluentd
 
-* [Introduction](#introduction)
-* [Overview](#overview)
-    * [Test stack](#test-stack)
-    * [Importing the logs](#importing-the-logs)
-    * [Direct import `logfmt` from `systemd`](#direct-import-logfmt-from-systemd)
-        * [Configuring `minikube`](#configuring-minikube)
-        * [Pull from `systemd`](#pull-from-systemd)
-        * [Systemd Summary](#systemd-summary)
-    * [Directly importing JSON](#directly-importing-json)
-        * [JSON in files](#json-in-files)
-            * [Prefixing all keys](#prefixing-all-keys)
-* [Kata `shimv2`](#kata-shimv2)
-* [Caveats](#caveats)
-* [Summary](#summary)
-
 # Introduction
 
 This document describes how to import Kata Containers logs into [Fluentd](https://www.fluentd.org/),
@@ -143,7 +128,7 @@ YAML can be found
   tag kata-containers
   path /run/log/journal
   pos_file /run/log/journal/kata-journald.pos
-  filters [{"SYSLOG_IDENTIFIER": "kata-runtime"}, {"SYSLOG_IDENTIFIER": "kata-proxy"}, {"SYSLOG_IDENTIFIER": "kata-shim"}]
+  filters [{"SYSLOG_IDENTIFIER": "kata-runtime"}, {"SYSLOG_IDENTIFIER": "kata-shim"}]
   read_from_head true
 </source>
 ```
@@ -161,7 +146,7 @@ generate some Kata specific log entries:
 
 ```bash
 $ minikube addons open efk
-$ cd $GOPATH/src/github.com/kata-containers/packaging/kata-deploy
+$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/packaging/kata-deploy
 $ kubectl apply -f examples/nginx-deployment-qemu.yaml
 ```
 
@@ -178,14 +163,14 @@ sub-filter on, for instance, the `SYSLOG_IDENTIFIER` to differentiate the Kata c
 on the `PRIORITY` to filter out critical issues etc.
 
 Kata generates a significant amount of Kata specific information, which can be seen as
-[`logfmt`](https://github.com/kata-containers/tests/tree/master/cmd/log-parser#logfile-requirements).
+[`logfmt`](https://github.com/kata-containers/tests/tree/main/cmd/log-parser#logfile-requirements).
 data contained in the `MESSAGE` field. Imported as-is, there is no easy way to filter on that data
 in Kibana:
 
 ![Kata tags in EFK](./images/efk_syslog_entry_detail.png).
 
 We can however further sub-parse the Kata entries using the
-[Fluentd plugins](https://docs.fluentbit.io/manual/parser/logfmt) that will parse
+[Fluentd plugins](https://docs.fluentbit.io/manual/pipeline/parsers/logfmt) that will parse
 `logfmt` formatted data. We can utilise these to parse the sub-fields using a Fluentd filter
 section. At the same time, we will prefix the new fields with `kata_` to make it clear where
 they have come from:
@@ -222,7 +207,7 @@ test to check the parsing works. The resulting output from Fluentd is:
   "_COMM":"kata-runtime",
   "_EXE":"/opt/kata/bin/kata-runtime",
   "SYSLOG_TIMESTAMP":"Feb 21 10:31:27 ",
-  "_CMDLINE":"/opt/kata/bin/kata-runtime --kata-config /opt/kata/share/defaults/kata-containers/configuration-qemu.toml --root /run/runc state 7cdd31660d8705facdadeb8598d2c0bd008e8142c54e3b3069abd392c8d58997",
+  "_CMDLINE":"/opt/kata/bin/kata-runtime --config /opt/kata/share/defaults/kata-containers/configuration-qemu.toml --root /run/runc state 7cdd31660d8705facdadeb8598d2c0bd008e8142c54e3b3069abd392c8d58997",
   "SYSLOG_PID":"14314",
   "_PID":"14314",
   "MESSAGE":"time=\"2020-02-21T10:31:27.810781647Z\" level=info msg=\"release sandbox\" arch=amd64 command=state container=7cdd31660d8705facdadeb8598d2c0bd008e8142c54e3b3069abd392c8d58997 name=kata-runtime pid=14314 sandbox=1c3e77cad66aa2b6d8cc846f818370f79cb0104c0b840f67d0f502fd6562b68c source=virtcontainers subsystem=sandbox",
@@ -272,16 +257,15 @@ go directly to a full Kata specific JSON format logfile test.
 
 Kata runtime has the ability to generate JSON logs directly, rather than its default `logfmt` format. Passing
 the `--log-format=json` argument to the Kata runtime enables this. The easiest way to pass in this extra
-parameter from a [Kata deploy](https://github.com/kata-containers/packaging/tree/master/kata-deploy) installation
-is to edit the `/opt/kata/bin/kata-qemu` shell script (generated by the
-[Kata packaging release scripts](https://github.com/kata-containers/packaging/blob/master/release/kata-deploy-binaries.sh)).
+parameter from a [Kata deploy](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy) installation
+is to edit the `/opt/kata/bin/kata-qemu` shell script.
 
 At the same time, we will add the `--log=/var/log/kata-runtime.log` argument to store the Kata logs in their
 own file (rather than into the system journal).
 
 ```bash
 #!/bin/bash
-/opt/kata/bin/kata-runtime --kata-config "/opt/kata/share/defaults/kata-containers/configuration-qemu.toml" --log-format=json --log=/var/log/kata-runtime.log $@
+/opt/kata/bin/kata-runtime --config "/opt/kata/share/defaults/kata-containers/configuration-qemu.toml" --log-format=json --log=/var/log/kata-runtime.log $@
 ```
 
 And then we'll add the Fluentd config section to parse that file. Note, we inform the parser that Kata is

docs/how-to/how-to-load-kernel-modules-with-kata.md

@@ -56,8 +56,9 @@ There are some limitations with this approach:
 
 As was mentioned above, not all containers need the same modules, therefore using
 the configuration file for specifying the list of kernel modules per [POD][3] can
-be a pain. Unlike the configuration file, annotations provide a way to specify
-custom configurations per POD.
+be a pain.
+Unlike the configuration file, [annotations](how-to-set-sandbox-config-kata.md)
+provide a way to specify custom configurations per POD.
 
 The list of kernel modules and parameters can be set using the annotation
 `io.katacontainers.config.agent.kernel_modules` as a semicolon separated
@@ -101,7 +102,7 @@ spec:
     tty: true
 ```
 
-> **Note**: To pass annotations to Kata containers, [cri must to be configurated correctly](how-to-set-sandbox-config-kata.md#cri-configuration)
+> **Note**: To pass annotations to Kata containers, [CRI-O must be configured correctly](how-to-set-sandbox-config-kata.md#cri-o-configuration)
 
 [1]: ../../src/runtime
 [2]: ../../src/agent

docs/how-to/how-to-set-prometheus-in-k8s.md

@@ -2,14 +2,6 @@
 
 This document describes how to run `kata-monitor` in a Kubernetes cluster using Prometheus's service discovery to scrape metrics from `kata-agent`.
 
-- [Introduction](#introduction)
-- [Pre-requisites](#pre-requisites)
-- [Configure Prometheus](#configure-prometheus)
-- [Configure `kata-monitor`](#configure-kata-monitor)
-- [Setup Grafana](#setup-grafana)
-  * [Create `datasource`](#create-datasource)
-  * [Import dashboard](#import-dashboard)
-
 > **Warning**: This how-to is only for evaluation purpose, you **SHOULD NOT** running it in production using this configurations.
 
 ## Introduction
@@ -34,7 +26,7 @@ Also you should ensure that `kubectl` working correctly.
 Start Prometheus by utilizing our sample manifest:
 
 ```
-$ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/2.0-dev/docs/how-to/data/prometheus.yml
+$ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/docs/how-to/data/prometheus.yml
 ```
 
 This will create a new namespace, `prometheus`, and create the following resources:
@@ -60,7 +52,7 @@ go_gc_duration_seconds{quantile="0.75"} 0.000229911
 `kata-monitor` can be started on the cluster as follows:
 
 ```
-$ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/2.0-dev/docs/how-to/data/kata-monitor-daemonset.yml
+$ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/docs/how-to/data/kata-monitor-daemonset.yml
 ```
 
 This will create a new namespace `kata-system` and a `daemonset` in it.
@@ -73,7 +65,7 @@ Once the `daemonset` is running, Prometheus should discover `kata-monitor` as a
 Run this command to run Grafana in Kubernetes:
 
 ```
-$ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/2.0-dev/docs/how-to/data/grafana.yml
+$ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/docs/how-to/data/grafana.yml
 ```
 
 This will create deployment and service for Grafana under namespace `prometheus`.
@@ -99,7 +91,7 @@ You can import this dashboard using Grafana UI, or using `curl` command in conso
 $ curl -XPOST -i localhost:3000/api/dashboards/import \
     -u admin:admin \
     -H "Content-Type: application/json" \
-	-d "{\"dashboard\":$(curl -sL https://raw.githubusercontent.com/kata-containers/kata-containers/2.0-dev/docs/how-to/data/dashboard.json )}"
+	-d "{\"dashboard\":$(curl -sL https://raw.githubusercontent.com/kata-containers/kata-containers/main/docs/how-to/data/dashboard.json )}"
 ```
 
 ## References

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -3,6 +3,11 @@
 Kata Containers gives users freedom to customize at per-pod level, by setting
 a wide range of Kata specific annotations in the pod specification.
 
+Some annotations may be [restricted](#restricted-annotations) by the
+configuration file for security reasons, notably annotations that could lead the
+runtime to execute programs on the host. Such annotations are marked with _(R)_ in
+the tables below.
+
 # Kata Configuration Annotations
 There are several kinds of Kata configurations and they are listed below.
 
@@ -21,11 +26,13 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.runtime.disable_new_netns` | `boolean` | determines if a new netns is created for the hypervisor process |
 | `io.katacontainers.config.runtime.internetworking_model` | string| determines how the VM should be connected to the container network interface. Valid values are `macvtap`, `tcfilter` and `none` |
 | `io.katacontainers.config.runtime.sandbox_cgroup_only`| `boolean` | determines if Kata processes are managed only in sandbox cgroup |
+| `io.katacontainers.config.runtime.enable_pprof` | `boolean` | enables Golang `pprof` for `containerd-shim-kata-v2` process |
 
 ## Agent Options
 | Key | Value Type | Comments |
 |-------| ----- | ----- |
 | `io.katacontainers.config.agent.enable_tracing` | `boolean` | enable tracing for the agent |
+| `io.katacontainers.config.agent.container_pipe_size` | uint32 | specify the size of the std(in/out) pipes created for containers |
 | `io.katacontainers.config.agent.kernel_modules` | string | the list of kernel modules and their parameters that will be loaded in the guest kernel. Semicolon separated list of kernel modules and their parameters. These modules will be loaded in the guest kernel using `modprobe`(8). E.g., `e1000e InterruptThrottleRate=3000,3000,3000 EEE=1; i915 enable_ppgtt=0` |
 | `io.katacontainers.config.agent.trace_mode` | string | the trace mode for the agent |
 | `io.katacontainers.config.agent.trace_type` | string | the trace type for the agent |
@@ -38,17 +45,24 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.block_device_cache_noflush` | `boolean` | Denotes whether flush requests for the device are ignored |
 | `io.katacontainers.config.hypervisor.block_device_cache_set` | `boolean` | cache-related options will be set to block devices or not |
 | `io.katacontainers.config.hypervisor.block_device_driver` | string | the driver to be used for block device, valid values are `virtio-blk`, `virtio-scsi`, `nvdimm`|
+| `io.katacontainers.config.hypervisor.cpu_features` | `string` | Comma-separated list of CPU features to pass to the CPU (QEMU) |
+| `io.katacontainers.config.hypervisor.ctlpath` (R) | `string` | Path to the `acrnctl` binary for the ACRN hypervisor |
 | `io.katacontainers.config.hypervisor.default_max_vcpus` | uint32| the maximum number of vCPUs allocated for the VM by the hypervisor |
 | `io.katacontainers.config.hypervisor.default_memory` | uint32| the memory assigned for a VM by the hypervisor in `MiB` |
 | `io.katacontainers.config.hypervisor.default_vcpus` | uint32| the default vCPUs assigned for a VM by the hypervisor |
 | `io.katacontainers.config.hypervisor.disable_block_device_use` | `boolean` | disallow a block device from being used |
+| `io.katacontainers.config.hypervisor.disable_image_nvdimm` | `boolean` | specify if a `nvdimm` device should be used as rootfs for the guest (QEMU) |
 | `io.katacontainers.config.hypervisor.disable_vhost_net` | `boolean` | specify if `vhost-net` is not available on the host |
 | `io.katacontainers.config.hypervisor.enable_hugepages` | `boolean` | if the memory should be `pre-allocated` from huge pages |
+| `io.katacontainers.config.hypervisor.enable_iommu_platform` | `boolean` | enable `iommu` on CCW devices (QEMU s390x) |
+| `io.katacontainers.config.hypervisor.enable_iommu` | `boolean` | enable `iommu` on Q35 (QEMU x86_64) |
 | `io.katacontainers.config.hypervisor.enable_iothreads` | `boolean`| enable IO to be processed in a separate thread. Supported currently for virtio-`scsi` driver |
 | `io.katacontainers.config.hypervisor.enable_mem_prealloc` | `boolean` | the memory space used for `nvdimm` device by the hypervisor |
 | `io.katacontainers.config.hypervisor.enable_swap` | `boolean` | enable swap of VM memory |
-| `io.katacontainers.config.hypervisor.entropy_source` | string| the path to a host source of entropy (`/dev/random`, `/dev/urandom` or real hardware RNG device) |
-| `io.katacontainers.config.hypervisor.file_mem_backend` | string | file based memory backend root directory |
+| `io.katacontainers.config.hypervisor.enable_vhost_user_store` | `boolean` | enable vhost-user storage device (QEMU) |
+| `io.katacontainers.config.hypervisor.enable_virtio_mem` | `boolean` | enable virtio-mem (QEMU) |
+| `io.katacontainers.config.hypervisor.entropy_source` (R) | string| the path to a host source of entropy (`/dev/random`, `/dev/urandom` or real hardware RNG device) |
+| `io.katacontainers.config.hypervisor.file_mem_backend` (R) | string | file based memory backend root directory |
 | `io.katacontainers.config.hypervisor.firmware_hash` | string | container firmware SHA-512 hash value |
 | `io.katacontainers.config.hypervisor.firmware` | string | the guest firmware that will run the container VM |
 | `io.katacontainers.config.hypervisor.guest_hook_path` | string | the path within the VM that will be used for drop in hooks |
@@ -59,27 +73,31 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.initrd_hash` | string | container guest initrd SHA-512 hash value |
 | `io.katacontainers.config.hypervisor.initrd` | string | the guest initrd image that will run in the container VM |
 | `io.katacontainers.config.hypervisor.jailer_hash` | string | container jailer SHA-512 hash value |
-| `io.katacontainers.config.hypervisor.jailer_path` | string | the jailer that will constrain the container VM |
+| `io.katacontainers.config.hypervisor.jailer_path` (R) | string | the jailer that will constrain the container VM |
 | `io.katacontainers.config.hypervisor.kernel_hash` | string | container kernel image SHA-512 hash value |
 | `io.katacontainers.config.hypervisor.kernel_params` | string | additional guest kernel parameters |
 | `io.katacontainers.config.hypervisor.kernel` | string | the kernel used to boot the container VM |
 | `io.katacontainers.config.hypervisor.machine_accelerators` | string | machine specific accelerators for the hypervisor |
 | `io.katacontainers.config.hypervisor.machine_type` | string | the type of machine being emulated by the hypervisor |
-| `io.katacontainers.config.hypervisor.memory_offset` | uint32| the memory space used for `nvdimm` device by the hypervisor |
+| `io.katacontainers.config.hypervisor.memory_offset` | uint64| the memory space used for `nvdimm` device by the hypervisor |
 | `io.katacontainers.config.hypervisor.memory_slots` | uint32| the memory slots assigned to the VM by the hypervisor |
 | `io.katacontainers.config.hypervisor.msize_9p` | uint32 | the `msize` for 9p shares |
 | `io.katacontainers.config.hypervisor.path` | string | the hypervisor that will run the container VM |
+| `io.katacontainers.config.hypervisor.pcie_root_port` | specify the number of PCIe Root Port devices. The PCIe Root Port device is used to hot-plug a PCIe device (QEMU) |
 | `io.katacontainers.config.hypervisor.shared_fs` | string | the shared file system type, either `virtio-9p` or `virtio-fs` |
 | `io.katacontainers.config.hypervisor.use_vsock` | `boolean` | specify use of `vsock` for agent communication |
+| `io.katacontainers.config.hypervisor.vhost_user_store_path` (R) | `string` | specify the directory path where vhost-user devices related folders, sockets and device nodes should be (QEMU) |
 | `io.katacontainers.config.hypervisor.virtio_fs_cache_size` | uint32 | virtio-fs DAX cache size in `MiB` |
 | `io.katacontainers.config.hypervisor.virtio_fs_cache` | string | the cache mode for virtio-fs, valid values are `always`, `auto` and `none` |
 | `io.katacontainers.config.hypervisor.virtio_fs_daemon` | string | virtio-fs `vhost-user` daemon path |
 | `io.katacontainers.config.hypervisor.virtio_fs_extra_args` | string | extra options passed to `virtiofs` daemon |
 
-# CRI Configuration
+# CRI-O Configuration
 
 In case of CRI-O, all annotations specified in the pod spec are passed down to Kata.
 
+# containerd Configuration
+
 For containerd, annotations specified in the pod spec are passed down to Kata
 starting with version `1.3.0` of containerd. Additionally, extra configuration is
 needed for containerd, by providing a `pod_annotations` field in the containerd config
@@ -92,16 +110,14 @@ for passing annotations to Kata from containerd:
 $ cat /etc/containerd/config
 ....
 
-[plugins.cri.containerd.runtimes.kata]
-           runtime_type = "io.containerd.runc.v1"
+         [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
+           runtime_type = "io.containerd.kata.v2"
            pod_annotations = ["io.katacontainers.*"]
-           [plugins.cri.containerd.runtimes.kata.options]
-             BinaryName = "/usr/bin/kata-runtime"
 ....
 
 ```
 
-Additional documentation on the above configuration can be found in the 
+Additional documentation on the above configuration can be found in the
 [containerd docs](https://github.com/containerd/cri/blob/8d5a8355d07783ba2f8f451209f6bdcc7c412346/docs/config.md).
 
 # Example - Using annotations
@@ -159,3 +175,32 @@ spec:
     stdin: true
     tty: true
 ```
+
+# Restricted annotations
+
+Some annotations are _restricted_, meaning that the configuration file specifies
+the acceptable values. Currently, only hypervisor annotations are restricted,
+for security reason, with the intent to control which binaries the Kata
+Containers runtime will launch on your behalf.
+
+The configuration file validates the annotation _name_ as well as the annotation
+_value_.
+
+The acceptable annotation names are defined by the `enable_annotations` entry in
+the configuration file.
+
+For restricted annotations, an additional configuration entry provides a list of
+acceptable values. Since most restricted annotations are intended to control
+which binaries the runtime can execute, the valid value is generally provided by
+a shell pattern, as defined by `glob(3)`. The table below provides the name of
+the configuration entry:
+
+| Key | Config file entry | Comments |
+|-------| ----- | ----- |
+| `ctlpath`  | `valid_ctlpaths` | Valid paths for `acrnctl` binary |
+| `entropy_source` | `valid_entropy_sources` | Valid entropy sources, e.g. `/dev/random` |
+| `file_mem_backend`  | `valid_file_mem_backends` | Valid locations for the file-based memory backend root directory |
+| `jailer_path`  | `valid_jailer_paths`| Valid paths for the jailer constraining the container VM (Firecracker) |
+| `path`  | `valid_hypervisor_paths` | Valid hypervisors to run the container VM |
+| `vhost_user_store_path`  | `valid_vhost_user_store_paths` | Valid paths for vhost-user related files|
+| `virtio_fs_daemon`  | `valid_virtio_fs_daemon_paths` | Valid paths for the `virtiofsd` daemon |

docs/how-to/how-to-use-k8s-with-cri-containerd-and-kata.md

@@ -1,26 +1,11 @@
 # How to use Kata Containers and CRI (containerd plugin) with Kubernetes
 
-* [Requirements](#requirements)
-* [Install and configure containerd](#install-and-configure-containerd)
-* [Install and configure Kubernetes](#install-and-configure-kubernetes)
-    * [Install Kubernetes](#install-kubernetes)
-    * [Configure Kubelet to use containerd](#configure-kubelet-to-use-containerd)
-    * [Configure HTTP proxy - OPTIONAL](#configure-http-proxy---optional)
-* [Start Kubernetes](#start-kubernetes)
-* [Install a Pod Network](#install-a-pod-network)
-* [Allow pods to run in the master node](#allow-pods-to-run-in-the-master-node)
-* [Create an untrusted pod using Kata Containers](#create-an-untrusted-pod-using-kata-containers)
-* [Delete created pod](#delete-created-pod)
-
 This document describes how to set up a single-machine Kubernetes (k8s) cluster.
 
 The Kubernetes cluster will use the
 [CRI containerd plugin](https://github.com/containerd/cri) and
 [Kata Containers](https://katacontainers.io) to launch untrusted workloads.
 
-For Kata Containers 1.5.0-rc2 and above, we will use `containerd-shim-kata-v2` (short as `shimv2` in this documentation)
-to launch Kata Containers. For the previous version of Kata Containers, the Pods are launched with `kata-runtime`.
-
 ## Requirements
 
 - Kubernetes, Kubelet, `kubeadm`
@@ -29,7 +14,7 @@ to launch Kata Containers. For the previous version of Kata Containers, the Pods
 
 > **Note:** For information about the supported versions of these components,
 > see the  Kata Containers
-> [`versions.yaml`](https://github.com/kata-containers/runtime/blob/master/versions.yaml)
+> [`versions.yaml`](../../versions.yaml)
 > file.
 
 ## Install and configure containerd
@@ -125,43 +110,33 @@ $ sudo systemctl daemon-reload
   $ sudo -E kubectl get pods
   ```
 
-## Install a Pod Network
+## Configure Pod Network
 
 A pod network plugin is needed to allow pods to communicate with each other.
+You can find more about CNI plugins from the [Creating a cluster with `kubeadm`](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#instructions) guide.
 
-- Install the `flannel` plugin by following the
-  [Using `kubeadm` to Create a Cluster](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#instructions)
-  guide, starting from the **Installing a pod network** section.
-
-- Create a pod network using flannel
-
-  > **Note:** There is no known way to determine programmatically the best version (commit) to use.
-  > See https://github.com/coreos/flannel/issues/995.
+By default the CNI plugin binaries is installed under `/opt/cni/bin` (in package `kubernetes-cni`), you only need to create a configuration file for CNI plugin.
 
   ```bash
-  $ sudo -E kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
-  ```
+  $ sudo -E mkdir -p /etc/cni/net.d
 
-- Wait for the pod network to become available
-
-  ```bash
-  # number of seconds to wait for pod network to become available
-  $ timeout_dns=420
-
-  $ while [ "$timeout_dns" -gt 0 ]; do
-      if sudo -E kubectl get pods --all-namespaces | grep dns | grep Running; then
-          break
-      fi
-
-      sleep 1s
-      ((timeout_dns--))
-   done
-  ```
-
-- Check the pod network is running
-
-  ```bash
-  $ sudo -E kubectl get pods --all-namespaces | grep dns | grep Running && echo "OK" || ( echo "FAIL" && false )
+  $ sudo -E cat > /etc/cni/net.d/10-mynet.conf <<EOF
+  {
+    "cniVersion": "0.2.0",
+    "name": "mynet",
+    "type": "bridge",
+    "bridge": "cni0",
+    "isGateway": true,
+    "ipMasq": true,
+    "ipam": {
+      "type": "host-local",
+      "subnet": "172.19.0.0/24",
+      "routes": [
+        { "dst": "0.0.0.0/0" }
+      ]
+    }
+  }
+  EOF
   ```
 
 ## Allow pods to run in the master node
@@ -172,24 +147,38 @@ By default, the cluster will not schedule pods in the master node. To enable mas
 $ sudo -E kubectl taint nodes --all node-role.kubernetes.io/master-
 ```
 
-## Create an untrusted pod using Kata Containers
+## Create runtime class for Kata Containers
 
 By default, all pods are created with the default runtime configured in CRI containerd plugin.
+From Kubernetes v1.12, users can use [`RuntimeClass`](https://kubernetes.io/docs/concepts/containers/runtime-class/#runtime-class) to specify a different runtime for Pods.
 
-If a pod has the `io.kubernetes.cri.untrusted-workload` annotation set to `"true"`, the CRI plugin runs the pod with the
+```bash
+$ cat > runtime.yaml <<EOF
+apiVersion: node.k8s.io/v1beta1
+kind: RuntimeClass
+metadata:
+  name: kata
+handler: kata
+EOF
+
+$ sudo -E kubectl apply -f runtime.yaml
+```
+
+## Run pod in Kata Containers
+
+If a pod has the `runtimeClassName` set to `kata`, the CRI plugin runs the pod with the
 [Kata Containers runtime](../../src/runtime/README.md).
 
-- Create an untrusted pod configuration
+- Create an pod configuration that using Kata Containers runtime
 
   ```bash
-  $ cat << EOT | tee nginx-untrusted.yaml
+  $ cat << EOT | tee nginx-kata.yaml
   apiVersion: v1
   kind: Pod
   metadata:
-    name: nginx-untrusted
-    annotations:
-      io.kubernetes.cri.untrusted-workload: "true"
+    name: nginx-kata
   spec:
+    runtimeClassName: kata
     containers:
     - name: nginx
       image: nginx
@@ -197,9 +186,9 @@ If a pod has the `io.kubernetes.cri.untrusted-workload` annotation set to `"true
   EOT
   ```
 
-- Create an untrusted pod
+- Create the pod
   ```bash
-  $ sudo -E kubectl apply -f nginx-untrusted.yaml
+  $ sudo -E kubectl apply -f nginx-kata.yaml
   ```
 
 - Check pod is running
@@ -216,5 +205,5 @@ If a pod has the `io.kubernetes.cri.untrusted-workload` annotation set to `"true
 ## Delete created pod
 
 ```bash
-$ sudo -E kubectl delete -f nginx-untrusted.yaml
+$ sudo -E kubectl delete -f nginx-kata.yaml
 ```

docs/how-to/how-to-use-kata-containers-with-acrn.md

@@ -2,11 +2,6 @@
 
 This document provides an overview on how to run Kata containers with ACRN hypervisor and device model.
 
-- [Introduction](#introduction)
-- [Pre-requisites](#pre-requisites)
-- [Configure Docker](#configure-docker)
-- [Configure Kata Containers with ACRN](#configure-kata-containers-with-acrn)
-
 ## Introduction
 
 ACRN is a flexible, lightweight Type-1 reference hypervisor built with real-time and safety-criticality in mind. ACRN uses an open source platform making it optimized to streamline embedded development.
@@ -91,7 +86,7 @@ To configure Kata Containers with ACRN, copy the generated `configuration-acrn.t
 The following command shows full paths to the `configuration.toml` files that the runtime loads. It will use the first path that exists. (Please make sure the kernel and image paths are set correctly in the `configuration.toml` file)
 
 ```bash
-$ sudo kata-runtime --kata-show-default-config-paths
+$ sudo kata-runtime --show-default-config-paths
 ```
 
 >**Warning:** Please offline CPUs using [this](offline_cpu.sh) script, else VM launches will fail.

docs/how-to/how-to-use-kata-containers-with-nemu.md

@@ -1,115 +0,0 @@
-
-# Kata Containers with NEMU
-
-* [Introduction](#introduction)
-* [Pre-requisites](#pre-requisites)
-* [NEMU](#nemu)
-    * [Download and build](#download-and-build)
-        * [x86_64](#x86_64)
-        * [aarch64](#aarch64)
-* [Configure Kata Containers](#configure-kata-containers)
-
-Kata Containers relies by default on the QEMU hypervisor in order to spawn the virtual machines running containers. [NEMU](https://github.com/intel/nemu) is a fork of QEMU that:
-- Reduces the number of lines of code.
-- Removes all legacy devices.
-- Reduces the emulation as far as possible.
-
-## Introduction
-
-This document describes how to run Kata Containers with NEMU, first by explaining how to download, build and install it. Then it walks through the steps needed to update your Kata Containers configuration in order to run with NEMU. 
-
-## Pre-requisites
-This document requires Kata Containers to be [installed](../install/README.md) on your system.
-
-Also, it's worth noting that NEMU only supports `x86_64` and `aarch64` architecture.
-
-## NEMU
-
-### Download and build
-
-```bash
-$ git clone https://github.com/intel/nemu.git
-$ cd nemu
-$ git fetch origin
-$ git checkout origin/experiment/automatic-removal
-```
-#### x86_64
-```
-$ SRCDIR=$PWD ./tools/build_x86_64_virt.sh
-```
-#### aarch64
-```
-$ SRCDIR=$PWD ./tools/build_aarch64.sh
-```
-
-> **Note:** The branch `experiment/automatic-removal` is a branch published by Jenkins after it has applied the automatic removal script to the `topic/virt-x86` branch. The purpose of this code removal being to reduce the source tree by removing files not being used by NEMU.
-
-After those commands have successfully returned, you will find the NEMU binary at `$HOME/build-x86_64_virt/x86_64_virt-softmmu/qemu-system-x86_64_virt` (__x86__), or `$HOME/build-aarch64/aarch64-softmmu/qemu-system-aarch64` (__ARM__).
-
-You also need the `OVMF` firmware in order to boot the virtual machine's kernel. It can currently be found at this [location](https://github.com/intel/ovmf-virt/releases).
-```bash
-$ sudo mkdir -p /usr/share/nemu
-$ OVMF_URL=$(curl -sL https://api.github.com/repos/intel/ovmf-virt/releases/latest | jq -S '.assets[0].browser_download_url')
-$ curl -o OVMF.fd -L $(sed -e 's/^"//' -e 's/"$//' <<<"$OVMF_URL")
-$ sudo install -o root -g root -m 0640 OVMF.fd /usr/share/nemu/
-```
-> **Note:** The OVMF firmware will be located at this temporary location until the changes can be pushed upstream.
-
-
-## Configure Kata Containers
-All you need from this section is to modify the configuration file `/usr/share/defaults/kata-containers/configuration.toml` to specify the options related to the hypervisor.
-
-
-```diff
- [hypervisor.qemu]
--path = "/usr/bin/qemu-lite-system-x86_64"
-+path = "/home/foo/build-x86_64_virt/x86_64_virt-softmmu/qemu-system-x86_64_virt"
- kernel = "/usr/share/kata-containers/vmlinuz.container"
- initrd = "/usr/share/kata-containers/kata-containers-initrd.img"
- image = "/usr/share/kata-containers/kata-containers.img"
--machine_type = "pc"
-+machine_type = "virt"
- 
- # Optional space-separated list of options to pass to the guest kernel.
- # For example, use `kernel_params = "vsyscall=emulate"` if you are having
-@@ -31,7 +31,7 @@
- 
- # Path to the firmware.
- # If you want that qemu uses the default firmware leave this option empty
--firmware = ""
-+firmware = "/usr/share/nemu/OVMF.fd"
- 
- # Machine accelerators
- # comma-separated list of machine accelerators to pass to the hypervisor.
-```
-
-As you can see from this snippet above, all you need to change is:
-- The path to the hypervisor binary, `/home/foo/build-x86_64_virt/x86_64_virt-softmmu/qemu-system-x86_64_virt` in this example.
-- The machine type from `pc` to `virt`.
-- The path to the firmware binary, `/usr/share/nemu/OVMF.fd` in this example.
-
-Once you have saved those modifications, you can start a new container:
-```bash
-$ docker run --runtime=kata-runtime -it busybox
-```
-And you will be able to verify this new container is running with the NEMU hypervisor by looking for the hypervisor path and the machine type from the `qemu` process running on your system:
-```bash
-$ ps -aux | grep qemu
-root ... /home/foo/build-x86_64_virt/x86_64_virt-softmmu/qemu-system-x86_64_virt
-...  -machine virt,accel=kvm,kernel_irqchip,nvdimm ...
-```
-
-Also relying on `kata-runtime kata-env` is a reliable way to validate you are using the expected hypervisor:
-```bash
-$ kata-runtime kata-env | awk -v RS= '/\[Hypervisor\]/'
-[Hypervisor]
-  MachineType = "virt"
-  Version = "NEMU (like QEMU) version 3.0.0 (v3.0.0-179-gaf9a791)\nCopyright (c) 2003-2017 Fabrice Bellard and the QEMU Project developers"
-  Path = "/home/foo/build-x86_64_virt/x86_64_virt-softmmu/qemu-system-x86_64_virt"
-  BlockDeviceDriver = "virtio-scsi"
-  EntropySource = "/dev/urandom"
-  Msize9p = 8192
-  MemorySlots = 10
-  Debug = true
-  UseVSock = false
-```

docs/how-to/how-to-use-sysctls-with-kata.md

@@ -1,6 +1,7 @@
 # Setting Sysctls with Kata
 
 ## Sysctls
+
 In Linux, the sysctl interface allows an administrator to modify kernel 
 parameters at runtime. Parameters are available via the `/proc/sys/` virtual 
 process file system. 
@@ -16,11 +17,10 @@ To get a complete list of kernel parameters, run:
 $ sudo sysctl -a
 ```
 
-Both Docker and Kubernetes provide mechanisms for setting namespaced sysctls. 
-Namespaced sysctls can be set per pod in the case of Kubernetes or per container
-in case of Docker.
+Kubernetes provide mechanisms for setting namespaced sysctls. 
+Namespaced sysctls can be set per pod in the case of Kubernetes.
 The following sysctls are known to be namespaced and can be set with 
-Docker and Kubernetes:
+Kubernetes:
 
 - `kernel.shm*`
 - `kernel.msg*`
@@ -30,31 +30,10 @@ Docker and Kubernetes:
 
 ### Namespaced Sysctls:
 
-Kata Containers supports setting namespaced sysctls with Docker and Kubernetes.
+Kata Containers supports setting namespaced sysctls with Kubernetes.
 All namespaced sysctls can be set in the same way as regular Linux based
 containers, the difference being, in the case of Kata they are set inside the guest.
 
-#### Setting Namespaced Sysctls with Docker:
-
-```
-$ sudo docker run --runtime=kata-runtime -it alpine cat /proc/sys/fs/mqueue/queues_max
-256
-$ sudo docker run --runtime=kata-runtime --sysctl fs.mqueue.queues_max=512 -it alpine cat /proc/sys/fs/mqueue/queues_max
-512
-```
-
-... and:
-
-```
-$ sudo docker run --runtime=kata-runtime -it alpine cat /proc/sys/kernel/shmmax
-18446744073692774399
-$ sudo docker run --runtime=kata-runtime --sysctl kernel.shmmax=1024 -it alpine cat /proc/sys/kernel/shmmax
-1024
-```
-
-For additional documentation on setting sysctls with Docker please refer to [Docker-sysctl-doc](https://docs.docker.com/engine/reference/commandline/run/#configure-namespaced-kernel-parameters-sysctls-at-runtime).
-
-
 #### Setting Namespaced Sysctls with Kubernetes:
 
 Kubernetes considers certain sysctls as safe and others as unsafe. For detailed
@@ -100,7 +79,7 @@ spec:
 
 ### Non-Namespaced Sysctls:
 
-Docker and Kubernetes disallow sysctls without a namespace.
+Kubernetes disallow sysctls without a namespace.
 The recommendation is to set them directly on the host or use a privileged
 container in the case of Kubernetes.
 

docs/how-to/how-to-use-virtio-fs-with-kata.md

@@ -1,61 +1,9 @@
 # Kata Containers with virtio-fs
 
-- [Introduction](#introduction)
-- [Pre-requisites](#pre-requisites)
-- [Install Kata Containers with virtio-fs support](#install-kata-containers-with-virtio-fs-support)
-- [Run a Kata Container utilizing virtio-fs](#run-a-kata-container-utilizing-virtio-fs)
-
 ## Introduction
 
 Container deployments utilize explicit or implicit file sharing between host filesystem and containers. From a trust perspective, avoiding a shared file-system between the trusted host and untrusted container is recommended. This is not always feasible. In Kata Containers, block-based volumes are preferred as they allow usage of either device pass through or `virtio-blk` for access within the virtual machine.
 
-As of the 1.7 release of Kata Containers, [9pfs](https://www.kernel.org/doc/Documentation/filesystems/9p.txt) is the default filesystem sharing mechanism. While this does allow for workload compatibility, it does so with degraded performance and potential for POSIX compliance limitations.
+As of the 2.0 release of Kata Containers, [virtio-fs](https://virtio-fs.gitlab.io/) is the default filesystem sharing mechanism.
 
-To help address these limitations, [virtio-fs](https://virtio-fs.gitlab.io/) has been developed. virtio-fs is a shared file system that lets virtual machines access a directory tree on the host. In Kata Containers, virtio-fs can be used to share container volumes, secrets, config-maps, configuration files (hostname, hosts, `resolv.conf`) and the container rootfs on the host with the guest.  virtio-fs provides significant performance and POSIX compliance improvements compared to 9pfs.
-
-Enabling of virtio-fs requires changes in the guest kernel as well as the VMM. For Kata Containers, experimental virtio-fs support is enabled through `qemu` and `cloud-hypervisor` VMMs.
-
-**Note: virtio-fs support is experimental in the 1.7 release of Kata Containers. Work is underway to improve stability, performance and upstream integration. This is available for early preview - use at your own risk**
-
-This document describes how to get Kata Containers to work with virtio-fs.
-
-## Pre-requisites
-
-Before Kata 1.8 this feature required the host to have hugepages support enabled. Enable this with the `sysctl vm.nr_hugepages=1024` command on the host.In later versions of Kata, virtio-fs leverages `/dev/shm` as the shared memory backend. The default size of `/dev/shm` on a system is typically half of the total system memory. This can pose a physical limit to the maximum number of pods that can be launched with virtio-fs. This can be overcome by increasing the size of `/dev/shm` as shown below:
-
-```bash
-$ mount -o remount,size=${desired_shm_size} /dev/shm
-```
- 
-## Install Kata Containers with virtio-fs support
-
-The Kata Containers `qemu` configuration with virtio-fs and the `virtiofs` daemon are available in the [Kata Container release](https://github.com/kata-containers/runtime/releases) artifacts starting with the 1.9 release. Installation is available through [distribution packages](https://github.com/kata-containers/documentation/blob/master/install/README.md#supported-distributions) as well through [`kata-deploy`](https://github.com/kata-containers/packaging/tree/master/kata-deploy).
-
-**Note: Support for virtio-fs was first introduced in `NEMU` hypervisor in Kata 1.8 release. This hypervisor has been deprecated.**
-
-Install the latest release of Kata with `kata-deploy` as follows:
-```
-docker run --runtime=runc -v /opt/kata:/opt/kata -v /var/run/dbus:/var/run/dbus -v /run/systemd:/run/systemd -v /etc/docker:/etc/docker -it katadocker/kata-deploy kata-deploy-docker install
-```
-
-This will place the Kata release artifacts in `/opt/kata`, and update Docker's configuration to include a runtime target, `kata-qemu-virtiofs`. Learn more about `kata-deploy` and how to use `kata-deploy` in Kubernetes [here](https://github.com/kata-containers/packaging/tree/master/kata-deploy#kubernetes-quick-start).
-
-## Run a Kata Container utilizing virtio-fs
-
-Once installed, start a new container, utilizing `qemu` + `virtiofs`:
-```bash
-$ docker run --runtime=kata-qemu-virtiofs -it busybox
-```
-
-Verify the new container is running with the `qemu` hypervisor as well as using `virtiofsd`. To do this look for the hypervisor path and the `virtiofs` daemon process on the host:
-```bash
-$ ps -aux | grep virtiofs
-root ... /home/foo/build-x86_64_virt/x86_64_virt-softmmu/qemu-system-x86_64_virt
-...  -machine virt,accel=kvm,kernel_irqchip,nvdimm ...
-root ... /home/foo/build-x86_64_virt/virtiofsd-x86_64 ...
-```
-
-You can also try out virtio-fs using `cloud-hypervisor` VMM:
-```bash
-$ docker run --runtime=kata-clh -it busybox
-```
+virtio-fs support works out of the box for `cloud-hypervisor` and `qemu`, when Kata Containers is deployed using `kata-deploy`. Learn more about `kata-deploy` and how to use `kata-deploy` in Kubernetes [here](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy#kubernetes-quick-start).

docs/how-to/how-to-use-virtio-mem-with-kata.md

@@ -1,9 +1,5 @@
 # Kata Containers with `virtio-mem`
 
-- [Introduction](#introduction)
-- [Requisites](#requisites)
-- [Run a Kata Container utilizing `virtio-mem`](#run-a-kata-container-utilizing-virtio-mem)
-
 ## Introduction
 
 The basic idea of `virtio-mem` is to provide a flexible, cross-architecture memory hot plug and hot unplug solution that avoids many limitations imposed by existing technologies, architectures, and interfaces.
@@ -13,26 +9,23 @@ Kata Containers with `virtio-mem` supports memory resize.
 
 ## Requisites
 
-Kata Containers with `virtio-mem` requires Linux and the QEMU that support `virtio-mem`.
-The Linux kernel and QEMU upstream version still not support `virtio-mem`.  @davidhildenbrand is working on them.
-Please use following unofficial version of the Linux kernel and QEMU that support `virtio-mem` with Kata Containers.
+Kata Containers just supports `virtio-mem` with QEMU.
+Install and setup Kata Containers as shown [here](../install/README.md).
 
-The Linux kernel is at https://github.com/davidhildenbrand/linux/tree/virtio-mem-rfc-v4.
-The Linux kernel config that can work with Kata Containers is at https://gist.github.com/teawater/016194ee84748c768745a163d08b0fb9.
-
-The QEMU is at https://github.com/teawater/qemu/tree/kata-virtio-mem. (The original source is at https://github.com/davidhildenbrand/qemu/tree/virtio-mem.  Its base version of QEMU cannot work with Kata Containers.  So merge the commit of `virtio-mem` to upstream QEMU.)
-
-Set Linux and the QEMU that support `virtio-mem` with following line in the Kata Containers QEMU configuration `configuration-qemu.toml`:
-```toml
-[hypervisor.qemu]
-path = "qemu-dir"
-kernel = "vmlinux-dir"
+### With x86_64
+The `virtio-mem` config of the x86_64 Kata Linux kernel is open.
+Enable `virtio-mem` as follows:
+```
+$ sudo sed -i -e 's/^#enable_virtio_mem.*$/enable_virtio_mem = true/g' /etc/kata-containers/configuration.toml
 ```
 
-Enable `virtio-mem` with following line in the Kata Containers configuration:
-```toml
-enable_virtio_mem = true
+### With other architectures
+The `virtio-mem` config of the others Kata Linux kernel is not open.
+You can open `virtio-mem` config as follows:
 ```
+CONFIG_VIRTIO_MEM=y
+```
+Then you can build and install the guest kernel image as shown [here](../../tools/packaging/kernel/README.md#build-kata-containers-kernel).
 
 ## Run a Kata Container utilizing `virtio-mem`
 
@@ -41,13 +34,35 @@ Use following command to enable memory overcommitment of a Linux kernel.  Becaus
 $ echo 1 | sudo tee /proc/sys/vm/overcommit_memory
 ```
 
-Use following command start a Kata Container.
+Use following command to start a Kata Container.
 ```
-$ docker run --rm -it --runtime=kata --name test busybox
+$ pod_yaml=pod.yaml
+$ container_yaml=container.yaml
+$ image="quay.io/prometheus/busybox:latest"
+$ cat << EOF > "${pod_yaml}"
+metadata:
+  name: busybox-sandbox1
+EOF
+$ cat << EOF > "${container_yaml}"
+metadata:
+  name: busybox-killed-vmm
+image:
+  image: "$image"
+command:
+- top
+EOF
+$ sudo crictl pull $image
+$ podid=$(sudo crictl runp $pod_yaml)
+$ cid=$(sudo crictl create $podid $container_yaml $pod_yaml)
+$ sudo crictl start $cid
 ```
 
-Use following command set the memory size of test to default_memory + 512m.
+Use the following command to set the container memory limit to 2g and the memory size of the VM to its default_memory + 2g.
 ```
-$ docker update -m 512m --memory-swap -1 test
+$ sudo crictl update --memory $((2*1024*1024*1024)) $cid
 ```
 
+Use the following command to set the container memory limit to 1g and the memory size of the VM to its default_memory + 1g.
+```
+$ sudo crictl update --memory $((1*1024*1024*1024)) $cid
+```

docs/how-to/privileged.md

@@ -3,11 +3,6 @@
 Kata Containers supports creation of containers that are "privileged" (i.e. have additional capabilities and access
 that is not normally granted).
 
-* [Warnings](#warnings)
-    * [Host Devices](#host-devices)
-        * [Containerd and CRI](#containerd-and-cri)
-        * [CRI-O](#cri-o)
-
 ## Warnings
 
 **Warning:** Whilst this functionality is supported, it can decrease the security of Kata Containers if not configured 

docs/how-to/run-kata-with-crictl.md

@@ -0,0 +1,139 @@
+# Working with `crictl`
+
+## What's `cri-tools`
+
+[`cri-tools`](https://github.com/kubernetes-sigs/cri-tools) provides debugging and validation tools for Kubelet Container Runtime Interface (CRI).
+
+`cri-tools` includes two tools: `crictl` and `critest`. `crictl` is the CLI for Kubelet CRI, in this document, we will show how to use `crictl` to run Pods in Kata containers.
+
+> **Note:** `cri-tools` is only used for debugging and validation purpose, and don't use it to run production workloads.
+
+> **Note:** For how to install and configure `cri-tools` with CRI runtimes like `containerd` or CRI-O, please also refer to other [howtos](./README.md).
+
+## Use `crictl` run Pods in Kata containers
+
+Sample config files in this document can be found [here](./data/crictl/).
+
+### Run `busybox` Pod
+
+#### Run pod sandbox with config file
+
+```bash
+$ sudo crictl runp -r kata sandbox_config.json
+16a62b035940f9c7d79fd53e93902d15ad21f7f9b3735f1ac9f51d16539b836b
+
+$ sudo crictl pods
+POD ID              CREATED             STATE               NAME                NAMESPACE           ATTEMPT
+16a62b035940f       21 seconds ago      Ready               busybox-pod                             0
+```
+
+#### Create container in the pod sandbox with config file
+
+```bash
+$ sudo crictl create 16a62b035940f container_config.json sandbox_config.json 
+e6ca0e0f7f532686236b8b1f549e4878e4fe32ea6b599a5d684faf168b429202
+```
+
+List containers and check the container is in `Created` state:
+
+```bash
+$ sudo crictl ps -a
+CONTAINER           IMAGE                              CREATED             STATE               NAME                ATTEMPT             POD ID
+e6ca0e0f7f532       docker.io/library/busybox:latest   19 seconds ago      Created             busybox-container   0                   16a62b035940f
+```
+
+#### Start container
+
+```bash
+$ sudo crictl start e6ca0e0f7f532
+e6ca0e0f7f532
+```
+
+List containers and we can see that the container state has changed from `Created` to `Running`:
+
+```bash
+$ sudo crictl ps
+CONTAINER           IMAGE                              CREATED              STATE               NAME                ATTEMPT             POD ID
+e6ca0e0f7f532       docker.io/library/busybox:latest   About a minute ago   Running             busybox-container   0                   16a62b035940f
+```
+
+And last we can `exec` into `busybox` container:
+
+```bash
+$ sudo crictl exec -it e6ca0e0f7f532 sh
+```
+
+And run commands in it:
+
+```
+/ # hostname 
+busybox_host
+/ # id
+uid=0(root) gid=0(root)
+```
+
+### Run `redis` Pod
+
+In this example, we will create two Pods: one is for `redis` server, and another one is `redis` client.
+
+#### Create `redis-server` Pod
+
+It's also possible to start a container within a single command:
+
+```bash
+$ sudo crictl run -r kata redis_server_container_config.json redis_server_sandbox_config.json
+bb36e05c599125842c5193909c4de186b1cee3818f5d17b951b6a0422681ce4b
+```
+
+#### Create `redis-client` Pod
+
+```bash
+$ sudo crictl run -r kata redis_client_container_config.json redis_client_sandbox_config.json
+e344346c5414e3f51f97f20b2262e0b7afe457750e94dc0edb109b94622fc693
+```
+
+After the new container started, we can check the running Pods and containers.
+
+```bash
+$ sudo crictl pods
+POD ID              CREATED              STATE               NAME                NAMESPACE           ATTEMPT
+469d08a7950e3       30 seconds ago       Ready               redis-client-pod                        0
+02c12fdb08219       About a minute ago   Ready               redis-server-pod                        0
+
+$ sudo crictl ps
+CONTAINER           IMAGE                                  CREATED              STATE               NAME                ATTEMPT             POD ID
+e344346c5414e       docker.io/library/redis:6.0.8-alpine   35 seconds ago       Running             redis-client        0                   469d08a7950e3
+bb36e05c59912       docker.io/library/redis:6.0.8-alpine   About a minute ago   Running             redis-server        0                   02c12fdb08219
+```
+
+#### Check `redis` server is working
+
+To connect to the `redis-server`. First we need to get the `redis-server`'s IP address.
+
+```bash
+
+$ server=$(sudo crictl inspectp 02c12fdb08219 | jq .status.network.ip | tr -d '"' )
+$ echo $server
+172.19.0.118
+```
+
+Launch `redis-cli` in the new Pod and connect server running at `172.19.0.118`.
+
+```bash
+$ sudo crictl exec -it e344346c5414e redis-cli -h $server
+172.19.0.118:6379> get test-key
+(nil)
+172.19.0.118:6379> set test-key test-value
+OK
+172.19.0.118:6379> get test-key
+"test-value"
+```
+
+Then back to `redis-server`, check if the `test-key` is set in server.
+
+```bash
+$ sudo crictl exec -it bb36e05c59912 redis-cli get test-key
+"test-val"
+```
+
+Returned `test-val` is just set by `redis-cli` in `redis-client` Pod.

docs/how-to/run-kata-with-k8s.md

@@ -1,18 +1,5 @@
 # Run Kata Containers with Kubernetes
 
-* [Run Kata Containers with Kubernetes](#run-kata-containers-with-kubernetes)
-  * [Prerequisites](#prerequisites)
-  * [Install a CRI implementation](#install-a-cri-implementation)
-     * [CRI-O](#cri-o)
-        * [Kubernetes Runtime Class (CRI-O v1.12 )](#kubernetes-runtime-class-cri-o-v112)
-        * [Untrusted annotation (until CRI-O v1.12)](#untrusted-annotation-until-cri-o-v112)
-        * [Network namespace management](#network-namespace-management)
-     * [containerd with CRI plugin](#containerd-with-cri-plugin)
-  * [Install Kubernetes](#install-kubernetes)
-     * [Configure for CRI-O](#configure-for-cri-o)
-     * [Configure for containerd](#configure-for-containerd)
-  * [Run a Kubernetes pod with Kata Containers](#run-a-kubernetes-pod-with-kata-containers)
-
 ## Prerequisites
 This guide requires Kata Containers available on your system, install-able by following [this guide](../install/README.md).
 
@@ -171,10 +158,10 @@ $ sudo systemctl daemon-reload
 $ sudo systemctl restart kubelet
 
 # If using CRI-O
-$ sudo kubeadm init --skip-preflight-checks --cri-socket /var/run/crio/crio.sock --pod-network-cidr=10.244.0.0/16
+$ sudo kubeadm init --ignore-preflight-errors=all --cri-socket /var/run/crio/crio.sock --pod-network-cidr=10.244.0.0/16
 
 # If using CRI-containerd
-$ sudo kubeadm init --skip-preflight-checks --cri-socket /run/containerd/containerd.sock --pod-network-cidr=10.244.0.0/16
+$ sudo kubeadm init --ignore-preflight-errors=all --cri-socket /run/containerd/containerd.sock --pod-network-cidr=10.244.0.0/16
 
 $ export KUBECONFIG=/etc/kubernetes/admin.conf
 ```

docs/how-to/service-mesh.md

@@ -1,21 +1,5 @@
 # Kata Containers and service mesh for Kubernetes
 
-* [Assumptions](#assumptions)
-* [How they work](#how-they-work)
-* [Prerequisites](#prerequisites)
-    * [Kata and Kubernetes](#kata-and-kubernetes)
-    * [Restrictions](#restrictions)
-* [Install and deploy your service mesh](#install-and-deploy-your-service-mesh)
-    * [Service Mesh Istio](#service-mesh-istio)
-    * [Service Mesh Linkerd](#service-mesh-linkerd)
-* [Inject your services with sidecars](#inject-your-services-with-sidecars)
-    * [Sidecar Istio](#sidecar-istio)
-    * [Sidecar Linkerd](#sidecar-linkerd)
-* [Run your services with Kata](#run-your-services-with-kata)
-    * [Lower privileges](#lower-privileges)
-    * [Add annotations](#add-annotations)
-    * [Deploy](#deploy)
-
 A service mesh is a way to monitor and control the traffic between
 micro-services running in your Kubernetes cluster. It is a powerful
 tool that you might want to use in combination with the security
@@ -76,15 +60,16 @@ is not able to perform a proper setup of the rules.
 
 ### Service Mesh Istio
 
-As a reference, you can follow Istio [instructions](https://istio.io/docs/setup/kubernetes/quick-start/#download-and-prepare-for-the-installation).
-
 The following is a summary of what you need to install Istio on your system:
+
 ```
 $ curl -L https://git.io/getLatestIstio | sh -
 $ cd istio-*
 $ export PATH=$PWD/bin:$PATH
 ```
 
+See the [Istio documentation](https://istio.io/docs) for further details.
+
 Now deploy Istio in the control plane of your cluster with the following:
 ```
 $ kubectl apply -f install/kubernetes/istio-demo.yaml

docs/how-to/what-is-vm-cache-and-how-do-I-use-it.md

@@ -1,10 +1,5 @@
 # What Is VMCache and How To Enable It
 
-* [What is VMCache](#what-is-vmcache)
-* [How is this different to VM templating](#how-is-this-different-to-vm-templating)
-* [How to enable VMCache](#how-to-enable-vmcache)
-* [Limitations](#limitations)
-
 ### What is VMCache
 
 VMCache is a new function that creates VMs as caches before using it.

docs/how-to/what-is-vm-templating-and-how-do-I-use-it.md

@@ -1,6 +1,7 @@
 # What Is VM Templating and How To Enable It
 
 ### What is VM templating
+
 VM templating is a Kata Containers feature that enables new VM
 creation using a cloning technique. When enabled, new VMs are created
 by cloning from a pre-created template VM, and they will share the
@@ -8,11 +9,13 @@ same initramfs, kernel and agent memory in readonly mode. It is very
 much like a process fork done by the kernel but here we *fork* VMs.
 
 ### How is this different from VMCache
+
 Both [VMCache](../how-to/what-is-vm-cache-and-how-do-I-use-it.md) and VM templating help speed up new container creation.  
 When VMCache enabled, new VMs are created by the VMCache server.  So it is not vulnerable to share memory CVE because each VM doesn't share the memory.  
 VM templating saves a lot of memory if there are many Kata Containers running on the same host.
 
 ### What are the Pros
+
 VM templating helps speed up new container creation and saves a lot
 of memory if there are many Kata Containers running on the same host.
 If you are running a density workload, or care a lot about container
@@ -29,6 +32,7 @@ showed that VM templating speeds up Kata Containers creation by as much as
 38.68%. See [full results here](https://gist.github.com/bergwolf/06974a3c5981494a40e2c408681c085d).
 
 ### What are the Cons
+
 One drawback of VM templating is that it cannot avoid cross-VM side-channel
 attack such as [CVE-2015-2877](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2015-2877)
 that originally targeted at the Linux KSM feature.
@@ -39,13 +43,15 @@ and can be classified as potentially misunderstood behaviors rather than vulnera
 **Warning**: If you care about such attack vector, do not use VM templating or KSM.
 
 ### How to enable VM templating
+
 VM templating can be enabled by changing your Kata Containers config file (`/usr/share/defaults/kata-containers/configuration.toml`,
 overridden by `/etc/kata-containers/configuration.toml` if provided) such that:
 
-  - `qemu-lite` is specified in `hypervisor.qemu`->`path` section
+  - `qemu` version `v4.1.0` or above is specified in `hypervisor.qemu`->`path` section
   - `enable_template = true`
   - `initrd =` is set
   - `image =` option is commented out or removed
+  - `shared_fs` should not be `virtio-fs`
 
 Then you can create a VM templating for later usage by calling
 ```

docs/hypervisors.md

@@ -0,0 +1,62 @@
+# Hypervisors
+
+## Introduction
+
+Kata Containers supports multiple hypervisors. This document provides a very
+high level overview of the available hypervisors, giving suggestions as to
+which hypervisors you may wish to investigate further.
+
+> **Note:**
+>
+> This document is not prescriptive or authoritative:
+>
+> - It is up to you to decide which hypervisors may be most appropriate for
+>   your use-case.
+> - Refer to the official documentation for each hypervisor for further details.
+
+## Types
+
+Since each hypervisor offers different features and options, Kata Containers
+provides a separate
+[configuration file](/src/runtime/README.md#configuration)
+for each. The configuration files contain comments explaining which options
+are available, their default values and how each setting can be used.
+
+> **Note:**
+>
+> The simplest way to switch between hypervisors is to create a symbolic link
+> to the appropriate hypervisor-specific configuration file.
+
+| Hypervisor | Written in | Architectures | Type | Configuration file |
+|-|-|-|-|-|
+[ACRN] | C | `x86_64` | Type 1 (bare metal) | `configuration-acrn.toml` |
+[Cloud Hypervisor] | rust | `aarch64`, `x86_64` | Type 2 ([KVM]) | `configuration-clh.toml` |
+[Firecracker] | rust | `aarch64`, `x86_64` | Type 2 ([KVM]) | `configuration-fc.toml` |
+[QEMU] | C | all | Type 2 ([KVM]) | `configuration-qemu.toml` |
+
+## Determine currently configured hypervisor
+
+```bash
+$ kata-runtime kata-env | awk -v RS= '/\[Hypervisor\]/' | grep Path
+```
+
+## Choose a Hypervisor
+
+The table below provides a brief summary of some of the differences between
+the hypervisors:
+
+
+| Hypervisor | Summary | Features | Limitations | Container Creation speed | Memory density | Use cases | Comment |
+|-|-|-|-|-|-|-|-|
+[ACRN] | Safety critical and real-time workloads | | | excellent | excellent | Embedded and IOT systems | For advanced users |
+[Cloud Hypervisor] | Low latency, small memory footprint, small attack surface | Minimal | | excellent | excellent | High performance modern cloud workloads | |
+[Firecracker] | Very slimline | Extremely minimal | Doesn't support all device types | excellent | excellent | Serverless / FaaS | |
+[QEMU] | Lots of features | Lots | | good | good | Good option for most users | | All users |
+
+For further details, see the [Virtualization in Kata Containers](design/virtualization.md) document and the official documentation for each hypervisor.
+
+[ACRN]: https://projectacrn.org
+[Cloud Hypervisor]: https://github.com/cloud-hypervisor/cloud-hypervisor
+[Firecracker]: https://github.com/firecracker-microvm/firecracker
+[KVM]: https://en.wikipedia.org/wiki/Kernel-based_Virtual_Machine
+[QEMU]: http://www.qemu-project.org

docs/install/README.md

@@ -1,120 +1,63 @@
-# Kata Containers installation user guides
+# Kata Containers installation guides
 
-* [Prerequisites](#prerequisites)
-* [Packaged installation methods](#packaged-installation-methods)
-   * [Supported Distributions](#supported-distributions)
-      * [Official packages](#official-packages)
-   * [Automatic Installation](#automatic-installation)
-   * [Snap Installation](#snap-installation)
-   * [Scripted Installation](#scripted-installation)
-   * [Manual Installation](#manual-installation)
-* [Build from source installation](#build-from-source-installation)
-* [Installing on a Cloud Service Platform](#installing-on-a-cloud-service-platform)
-* [Further information](#further-information)
-
-The following is an overview of the different installation methods available. All of these methods equally result
-in a system configured to run Kata Containers.
+The following is an overview of the different installation methods available. 
 
 ## Prerequisites
-Kata Containers requires nested virtualization or bare metal.
-See the
-[hardware requirements](../../src/runtime/README.md#hardware-requirements)
-to see if your system is capable of running Kata Containers.
+
+Kata Containers requires nested virtualization or bare metal. Check 
+[hardware requirements](/src/runtime/README.md#hardware-requirements) to see if your system is capable of running Kata 
+Containers.
 
 ## Packaged installation methods
 
-> **Notes:**
->
-> - Packaged installation methods uses your distribution's native package format (such as RPM or DEB).
+Packaged installation methods uses your distribution's native package format (such as RPM or DEB).
 
-| Installation method                                  | Description                                                                             | Distributions supported              |
-|------------------------------------------------------|-----------------------------------------------------------------------------------------|--------------------------------------|
-| [Automatic](#automatic-installation)                 |Run a single command to install a full system                                            |[see table](#supported-distributions) |
-| [Using snap](#snap-installation)                     |Easy to install and automatic updates                                                    |any distro that supports snapd        |
-| [Using official distro packages](#official-packages) |Kata packages provided by Linux distributions official repositories                      |[see table](#supported-distributions) |
-| [Scripted](#scripted-installation)                   |Generates an installation script which will result in a working system when executed     |[see table](#supported-distributions) |
-| [Manual](#manual-installation)                       |Allows the user to read a brief document and execute the specified commands step-by-step |[see table](#supported-distributions) |
+*Note:* We encourage installation methods that provides automatic updates, it ensures security updates and bug fixes are
+easily applied.
 
-### Supported Distributions
+| Installation method                                  | Description                                                         | Automatic updates | Use case                                                 |
+|------------------------------------------------------|---------------------------------------------------------------------|-------------------|----------------------------------------------------------|
+| [Using official distro packages](#official-packages) | Kata packages provided by Linux distributions official repositories | yes               | Recommended for most users.                              |
+| [Using snap](#snap-installation)                     | Easy to install                                                     | yes               | Good alternative to official distro packages.            |
+| [Automatic](#automatic-installation)                 | Run a single command to install a full system                       | **No!**           | For those wanting the latest release quickly.            |
+| [Manual](#manual-installation)                       | Follow a guide step-by-step to install a working system             | **No!**           | For those who want the latest release with more control. |
+| [Build from source](#build-from-source-installation) | Build the software components manually                              | **No!**           | Power users and developers only.                         |
 
-Kata is packaged by the Kata community for:
-
-|Distribution (link to installation guide)                        | Versions                                                                                                          |
-|-----------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------|
-|[CentOS](centos-installation-guide.md)                           | 7                                                                                                                 |
-|[Debian](debian-installation-guide.md)                           | 9, 10                                                                                                             |
-|[Fedora](fedora-installation-guide.md)                           | 28, 29, 30                                                                                                        |
-|[openSUSE](opensuse-installation-guide.md)                       | [Leap](opensuse-leap-installation-guide.md) (15, 15.1)<br>[Tumbleweed](opensuse-tumbleweed-installation-guide.md) |
-|[Red Hat Enterprise Linux (RHEL)](rhel-installation-guide.md)    | 7                                                                                                                 |
-|[SUSE Linux Enterprise Server (SLES)](sles-installation-guide.md)| SLES 12 SP3                                                                                                       |
-|[Ubuntu](ubuntu-installation-guide.md)                           | 16.04, 18.04                                                                                                      |
-
-#### Official packages
+### Official packages
 
 Kata packages are provided by official distribution repositories for:
 
-|Distribution (link to packages)                                  | Versions   |
-|-----------------------------------------------------------------|------------|
-|[openSUSE](https://software.opensuse.org/package/katacontainers) | Tumbleweed |
-
-
-### Automatic Installation
-
-[Use `kata-manager`](installing-with-kata-manager.md) to automatically install Kata packages.
+| Distribution (link to installation guide)                | Minimum versions                                                               |
+|----------------------------------------------------------|--------------------------------------------------------------------------------|
+| [CentOS](centos-installation-guide.md)                   | 8                                                                              |
+| [Fedora](fedora-installation-guide.md)                   | 34                                                                             |
 
 ### Snap Installation
 
-[![Get it from the Snap Store](https://snapcraft.io/static/images/badges/en/snap-store-black.svg)](https://snapcraft.io/kata-containers)
+The snap installation is available for all distributions which support `snapd`.
 
 [Use snap](snap-installation-guide.md) to install Kata Containers from https://snapcraft.io.
 
-### Scripted Installation
-[Use `kata-doc-to-script`](installing-with-kata-doc-to-script.md) to generate installation scripts that can be reviewed before they are executed.
+### Automatic Installation
+
+[Use `kata-manager`](/utils/README.md) to automatically install a working Kata Containers system.
 
 ### Manual Installation
-Manual installation instructions are available for [these distributions](#supported-distributions) and document how to:
-1. Add the Kata Containers repository to your distro package manager, and import the packages signing key.
-2. Install the Kata Containers packages.
-3. Install a supported container manager.
-4. Configure the container manager to use `kata-runtime` as the default OCI runtime. Or, for Kata Containers 1.5.0 or above, configure the
-   `io.containerd.kata.v2` to be the runtime shim (see [containerd runtime v2 (shim API)](https://github.com/containerd/containerd/tree/master/runtime/v2)
-   and [How to use Kata Containers and CRI (containerd plugin) with Kubernetes](../how-to/how-to-use-k8s-with-cri-containerd-and-kata.md)).
 
-> **Notes on upgrading**:
-> - If you are installing Kata Containers on a system that already has Clear Containers or `runv` installed,
->  first read [the upgrading document](../Upgrading.md).
-
-> **Notes on releases**:
-> - [This download server](http://download.opensuse.org/repositories/home:/katacontainers:/releases:/)
-> hosts the Kata Containers packages built by OBS for all the supported architectures.
-> Packages are available for the latest and stable releases (more info [here](../Stable-Branch-Strategy.md)).
->
-> - The following guides apply to the latest Kata Containers release
-> (a.k.a. `master` release).
->
-> - When choosing a stable release, replace all `master` occurrences in the URLs
-> with a `stable-x.y` version available on the [download server](http://download.opensuse.org/repositories/home:/katacontainers:/releases:/).
-
-> **Notes on packages source verification**:
-> - The Kata packages hosted on the download server are signed with GPG to ensure integrity and authenticity.
->
-> - The public key used to sign packages is available [at this link](https://raw.githubusercontent.com/kata-containers/tests/master/data/rpm-signkey.pub); the fingerprint is `9FDC0CB6 3708CF80 3696E2DC D0B37B82 6063F3ED`.
->
-> - Only trust the signing key and fingerprint listed in the previous bullet point. Do not disable GPG checks,
-> otherwise packages source and authenticity is not guaranteed.
+Follow the [containerd installation guide](container-manager/containerd/containerd-install.md).
 
 ## Build from source installation
-> **Notes:**
->
-> - Power users who decide to build from sources should be aware of the
->   implications of using an unpackaged system which will not be automatically
->   updated as new [releases](../Stable-Branch-Strategy.md) are made available.
+
+*Note:* Power users who decide to build from sources should be aware of the
+implications of using an unpackaged system which will not be automatically
+updated as new [releases](../Stable-Branch-Strategy.md) are made available.
 
 [Building from sources](../Developer-Guide.md#initial-setup)  allows power users
 who are comfortable building software from source to use the latest component
 versions. This is not recommended for normal users.
 
 ## Installing on a Cloud Service Platform
+
 * [Amazon Web Services (AWS)](aws-installation-guide.md)
 * [Google Compute Engine (GCE)](gce-installation-guide.md)
 * [Microsoft Azure](azure-installation-guide.md)
@@ -122,6 +65,7 @@ versions. This is not recommended for normal users.
 * [VEXXHOST OpenStack Cloud](vexxhost-installation-guide.md)
 
 ## Further information
-* The [upgrading document](../Upgrading.md).
-* The [developer guide](../Developer-Guide.md).
-* The [runtime documentation](../../src/runtime/README.md).
+
+* [upgrading document](../Upgrading.md)
+* [developer guide](../Developer-Guide.md)
+* [runtime documentation](../../src/runtime/README.md)

docs/install/aws-installation-guide.md

@@ -1,10 +1,5 @@
 # Install Kata Containers on Amazon Web Services
 
-* [Install and Configure AWS CLI](#install-and-configure-aws-cli)
-* [Create or Import an EC2 SSH key pair](#create-or-import-an-ec2-ssh-key-pair)
-* [Launch i3.metal instance](#launch-i3metal-instance)
-* [Install Kata](#install-kata)
-
 Kata Containers on Amazon Web Services (AWS) makes use of [i3.metal](https://aws.amazon.com/ec2/instance-types/i3/) instances. Most of the installation procedure is identical to that for Kata on your preferred distribution, except that you have to run it on bare metal instances since AWS doesn't support nested virtualization yet. This guide walks you through creating an i3.metal instance.
 
 ## Install and Configure AWS CLI

docs/install/azure-installation-guide.md

@@ -15,4 +15,4 @@ Create a new virtual machine with:
 
 ## Set up with distribution specific quick start
 
-Follow distribution specific [install guides](../install/README.md#supported-distributions).
+Follow distribution specific [install guides](../install/README.md#packaged-installation-methods).

docs/install/centos-installation-guide.md

@@ -3,15 +3,19 @@
 1. Install the Kata Containers components with the following commands:
 
    ```bash
+   $ sudo -E dnf install -y centos-release-advanced-virtualization
+   $ sudo -E dnf module disable -y virt:rhel
    $ source /etc/os-release
-   $ sudo yum -y install yum-utils
-   $ ARCH=$(arch)
-   $ BRANCH="${BRANCH:-master}"
-   $ sudo -E yum-config-manager --add-repo "http://download.opensuse.org/repositories/home:/katacontainers:/releases:/${ARCH}:/${BRANCH}/CentOS_${VERSION_ID}/home:katacontainers:releases:${ARCH}:${BRANCH}.repo"
-   $ sudo -E yum -y install kata-runtime kata-proxy kata-shim
+   $ cat <<EOF | sudo -E tee /etc/yum.repos.d/kata-containers.repo
+     [kata-containers]
+     name=Kata Containers
+     baseurl=http://mirror.centos.org/\$contentdir/\$releasever/virt/\$basearch/kata-containers
+     enabled=1
+     gpgcheck=1
+     skip_if_unavailable=1
+     EOF
+   $ sudo -E dnf install -y kata-containers
    ```
 
 2. Decide which container manager to use and select the corresponding link that follows:
-
-   - [Docker](docker/centos-docker-install.md)
    - [Kubernetes](../Developer-Guide.md#run-kata-containers-with-kubernetes)

docs/install/container-manager/containerd/containerd-install.md

@@ -0,0 +1,128 @@
+# Install Kata Containers with containerd
+
+> **Note:**
+>
+> - If Kata Containers and / or containerd are packaged by your distribution,
+>   we recommend you install these versions to ensure they are updated when
+>   new releases are available.
+
+> **Warning:**
+>
+> - These instructions install the **newest** versions of Kata Containers and
+>   containerd from binary release packages. These versions may not have been
+>   tested with your distribution version.
+>
+> - Since your package manager is not being used, it is **your**
+>   responsibility to ensure these packages are kept up-to-date when new
+>   versions are released.
+>
+> - If you decide to proceed and install a Kata Containers release, you can
+>   still check for the latest version of Kata Containers by running
+>   `kata-runtime check --only-list-releases`.
+>
+> - These instructions will not work for Fedora 31 and higher since those
+>   distribution versions only support cgroups version 2 by default. However,
+>   Kata Containers currently requires cgroups version 1 (on the host side). See
+>   https://github.com/kata-containers/kata-containers/issues/927 for further
+>   details.
+
+## Install Kata Containers
+
+> **Note:**
+>
+> If your distribution packages Kata Containers, we recommend you install that
+> version. If it does not, or you wish to perform a manual installation,
+> continue with the steps below.
+
+- Download a release from:
+
+  - https://github.com/kata-containers/kata-containers/releases
+
+  Note that Kata Containers uses [semantic versioning](https://semver.org) so
+  you should install a version that does *not* include a dash ("-"), since this
+  indicates a pre-release version.
+
+- Unpack the downloaded archive.
+
+   Kata Containers packages use a `/opt/kata/` prefix so either add that to
+   your `PATH`, or create symbolic links for the following commands. The
+   advantage of using symbolic links is that the `systemd(1)` configuration file
+   for containerd will not need to be modified to allow the daemon to find this
+   binary (see the [section on installing containerd](#install-containerd) below).
+
+   | Command | Description |
+   |-|-|
+   | `/opt/kata/bin/containerd-shim-kata-v2` | The main Kata 2.x binary |
+   | `/opt/kata/bin/kata-collect-data.sh`    | Data collection script used for [raising issues](https://github.com/kata-containers/kata-containers/issues) |
+   | `/opt/kata/bin/kata-runtime`            | Utility command |
+
+- Check installation by showing version details:
+
+   ```bash
+   $ kata-runtime --version
+   ```
+
+## Install containerd
+
+> **Note:**
+>
+> If your distribution packages containerd, we recommend you install that
+> version. If it does not, or you wish to perform a manual installation,
+> continue with the steps below.
+
+- Download a release from:
+
+  - https://github.com/containerd/containerd/releases
+
+- Unpack the downloaded archive.
+
+- Configure containerd
+
+  - Download the standard `systemd(1)` service file and install to
+    `/etc/systemd/system/`:
+
+    - https://raw.githubusercontent.com/containerd/containerd/master/containerd.service
+
+    > **Notes:**
+    >
+    > - You will need to reload the systemd configuration after installing this
+    >   file.
+    >
+    > - If you have not created a symbolic link for
+    >   `/opt/kata/bin/containerd-shim-kata-v2`, you will need to modify this
+    >   file to ensure the containerd daemon's `PATH` contains `/opt/kata/`.
+    >   See the `Environment=` command in `systemd.exec(5)` for further
+    >   details.
+
+  - Add the Kata Containers configuration to the containerd configuration file:
+
+    ```toml
+    [plugins]
+      [plugins."io.containerd.grpc.v1.cri"]
+        [plugins."io.containerd.grpc.v1.cri".containerd]
+          default_runtime_name = "kata"
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
+              runtime_type = "io.containerd.kata.v2"
+    ```
+
+    > **Note:**
+    >    
+    > The containerd daemon needs to be able to find the
+    > `containerd-shim-kata-v2` binary to allow Kata Containers to be created.
+
+  - Start the containerd service.
+
+## Test the installation
+
+You are now ready to run Kata Containers. You can perform a simple test by
+running the following commands:
+
+```bash
+$ image="docker.io/library/busybox:latest"
+$ sudo ctr image pull "$image"
+$ sudo ctr run --runtime "io.containerd.kata.v2" --rm -t "$image" test-kata uname -r
+```
+
+The last command above shows details of the kernel version running inside the
+container, which will likely be different to the host kernel version.

docs/install/debian-installation-guide.md

@@ -1,22 +0,0 @@
-# Install Kata Containers on Debian
-
-1. Install the Kata Containers components with the following commands:
-
-   ```bash
-   $ export DEBIAN_FRONTEND=noninteractive
-   $ ARCH=$(arch)
-   $ BRANCH="${BRANCH:-master}"
-   $ source /etc/os-release
-   $ [ "$ID" = debian ] && [ -z "$VERSION_ID" ] && echo >&2 "ERROR: Debian unstable not supported.
-     You can try stable packages here:
-     http://download.opensuse.org/repositories/home:/katacontainers:/releases:/${ARCH}:/${BRANCH}" && exit 1
-   $ sudo sh -c "echo 'deb http://download.opensuse.org/repositories/home:/katacontainers:/releases:/${ARCH}:/${BRANCH}/Debian_${VERSION_ID}/ /' > /etc/apt/sources.list.d/kata-containers.list"
-   $ curl -sL  http://download.opensuse.org/repositories/home:/katacontainers:/releases:/${ARCH}:/${BRANCH}/Debian_${VERSION_ID}/Release.key | sudo apt-key add -
-   $ sudo -E apt-get update
-   $ sudo -E apt-get -y install kata-runtime kata-proxy kata-shim
-   ```
-
-2. Decide which container manager to use and select the corresponding link that follows:
-
-   - [Docker](docker/debian-docker-install.md)
-   - [Kubernetes](../Developer-Guide.md#run-kata-containers-with-kubernetes)

docs/install/docker/centos-docker-install.md

@@ -1,75 +0,0 @@
-# Install Docker for Kata Containers on CentOS
-
-> **Note:**
->
-> - This guide assumes you have
->   [already installed the Kata Containers packages](../centos-installation-guide.md).
-
-1. Install the latest version of Docker with the following commands:
-
-   > **Notes:**
-   >
-   > - This step is only required if Docker is not installed on the system.
-   > - Docker version 18.09 [removed devicemapper support](https://github.com/kata-containers/documentation/issues/373).
-   >   If you wish to use a block based backend, see the options listed on https://github.com/kata-containers/documentation/issues/407.
-
-   ```bash
-   $ sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
-   $ sudo yum -y install docker-ce
-   ```
-
-   For more information on installing Docker please refer to the
-   [Docker Guide](https://docs.docker.com/engine/installation/linux/centos).
-
-2. Configure Docker to use Kata Containers by default with **ONE** of the following methods:
-
-   1. systemd (this is the default and is applied automatically if you select the
-      [automatic installation](../../install/README.md#automatic-installation) option)
-
-       ```bash
-       $ sudo mkdir -p /etc/systemd/system/docker.service.d/
-       $ cat <<EOF | sudo tee /etc/systemd/system/docker.service.d/kata-containers.conf
-       [Service]
-       ExecStart=
-       ExecStart=/usr/bin/dockerd -D --add-runtime kata-runtime=/usr/bin/kata-runtime --default-runtime=kata-runtime
-       EOF
-       ```
-
-   2. Docker `daemon.json`
-
-      Create docker configuration folder.
-
-      ```
-      $ sudo mkdir -p /etc/docker
-      ```
-
-      Add the following definitions to `/etc/docker/daemon.json`:
-
-      ```json
-      {
-        "default-runtime": "kata-runtime",
-        "runtimes": {
-          "kata-runtime": {
-            "path": "/usr/bin/kata-runtime"
-          }
-        }
-      }
-      ```
-
-3. Restart the Docker systemd service with the following commands:
-
-   ```bash
-   $ sudo systemctl daemon-reload
-   $ sudo systemctl restart docker
-   ```
-
-4. Run Kata Containers
-
-   You are now ready to run Kata Containers:
-
-   ```bash
-   $ sudo docker run busybox uname -a
-   ```
-
-   The previous command shows details of the kernel version running inside the
-   container, which is different to the host kernel version.

docs/install/docker/debian-docker-install.md

@@ -1,103 +0,0 @@
-# Install Docker for Kata Containers on Debian
-
-> **Note:**
->
-> - This guide assumes you have
->   [already installed the Kata Containers packages](../debian-installation-guide.md).
-> - This guide allows for installation with `systemd` or `sysVinit` init systems.
-
-1. Install the latest version of Docker with the following commands:
-
-   > **Notes:**
-   >
-   > - This step is only required if Docker is not installed on the system.
-   > - Docker version 18.09 [removed devicemapper support](https://github.com/kata-containers/documentation/issues/373).
-   >   If you wish to use a block based backend, see the options listed on https://github.com/kata-containers/documentation/issues/407.
-
-   ```bash
-   $ sudo apt-get -y install apt-transport-https ca-certificates curl gnupg2 software-properties-common
-   $ curl -fsSL https://download.docker.com/linux/$(. /etc/os-release; echo "$ID")/gpg | sudo apt-key add -
-   $ sudo add-apt-repository "deb https://download.docker.com/linux/$(. /etc/os-release; echo "$ID") $(lsb_release -cs) stable"
-   $ sudo apt-get update
-   $ sudo -E apt-get -y install docker-ce
-   ```
-
-   For more information on installing Docker please refer to the
-   [Docker Guide](https://docs.docker.com/engine/installation/linux/debian).
-
-2. Configure Docker to use Kata Containers by default with **ONE** of the following methods:
-
-a. `sysVinit`
-
-    - with `sysVinit`, docker config is stored in `/etc/default/docker`, edit the options similar to the following:
-
-    ```sh
-    $ sudo sh -c "echo '# specify docker runtime for kata-containers
-    DOCKER_OPTS=\"-D --add-runtime kata-runtime=/usr/bin/kata-runtime --default-runtime=kata-runtime\"' >> /etc/default/docker"
-    ```
-
-b. systemd (this is the default and is applied automatically if you select the
-      [automatic installation](../../install/README.md#automatic-installation) option)
-
-    ```bash
-    $ sudo mkdir -p /etc/systemd/system/docker.service.d/
-    $ cat <<EOF | sudo tee /etc/systemd/system/docker.service.d/kata-containers.conf
-    [Service]
-    ExecStart=
-    ExecStart=/usr/bin/dockerd -D --add-runtime kata-runtime=/usr/bin/kata-runtime --default-runtime=kata-runtime
-    EOF
-    ```
-
-c. Docker `daemon.json`
-
-    Create docker configuration folder.
-
-    ```
-    $ sudo mkdir -p /etc/docker
-    ```
-
-    Add the following definitions to `/etc/docker/daemon.json`:
-
-    ```json
-    {
-      "default-runtime": "kata-runtime",
-      "runtimes": {
-        "kata-runtime": {
-          "path": "/usr/bin/kata-runtime"
-        }
-      }
-    }
-    ```
-
-3. Restart the Docker systemd service with one of the following (depending on init choice):
-
-    a. `sysVinit`
-
-    ```sh
-    $ sudo /etc/init.d/docker stop
-    $ sudo /etc/init.d/docker start
-    ```
-
-    To watch for errors:
-
-    ```sh
-    $ tail -f /var/log/docker.log
-    ```
-
-    b. systemd
-
-    ```bash
-    $ sudo systemctl daemon-reload
-    $ sudo systemctl restart docker
-    ```
-
-4. Run Kata Containers
-
-   You are now ready to run Kata Containers:
-
-   ```bash
-   $ sudo docker run busybox uname -a
-   ```
-
-   The previous command shows details of the kernel version running inside the
-   container, which is different to the host kernel version.

docs/install/docker/fedora-docker-install.md

@@ -1,77 +0,0 @@
-# Install Docker for Kata Containers on Fedora
-
-> **Note:**
->
-> - This guide assumes you have
->   [already installed the Kata Containers packages](../fedora-installation-guide.md).
-
-1. Install the latest version of Docker with the following commands:
-
-   > **Notes:**
-   >
-   > - This step is only required if Docker is not installed on the system.
-   > - Docker version 18.09 [removed devicemapper support](https://github.com/kata-containers/documentation/issues/373).
-   >   If you wish to use a block based backend, see the options listed on https://github.com/kata-containers/documentation/issues/407.
-
-   ```bash
-   $ source /etc/os-release
-   $ sudo dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo
-   $ sudo dnf makecache
-   $ sudo dnf -y install docker-ce
-   ```
-
-   For more information on installing Docker please refer to the
-   [Docker Guide](https://docs.docker.com/engine/installation/linux/fedora).
-
-2. Configure Docker to use Kata Containers by default with **ONE** of the following methods:
-
-   1. systemd (this is the default and is applied automatically if you select the
-      [automatic installation](../../install/README.md#automatic-installation) option)
-
-       ```bash
-       $ sudo mkdir -p /etc/systemd/system/docker.service.d/
-       $ cat <<EOF | sudo tee /etc/systemd/system/docker.service.d/kata-containers.conf
-       [Service]
-       ExecStart=
-       ExecStart=/usr/bin/dockerd -D --add-runtime kata-runtime=/usr/bin/kata-runtime --default-runtime=kata-runtime
-       EOF
-       ```
-
-   2. Docker `daemon.json`
-
-      Create docker configuration folder.
-
-      ```
-      $ sudo mkdir -p /etc/docker
-      ```
-
-      Add the following definitions to `/etc/docker/daemon.json`:
-
-      ```json
-      {
-        "default-runtime": "kata-runtime",
-        "runtimes": {
-          "kata-runtime": {
-            "path": "/usr/bin/kata-runtime"
-          }
-        }
-      }
-      ```
-
-3. Restart the Docker systemd service with the following commands:
-
-   ```bash
-   $ sudo systemctl daemon-reload
-   $ sudo systemctl restart docker
-   ```
-
-4. Run Kata Containers
-
-   You are now ready to run Kata Containers:
-
-   ```bash
-   $ sudo docker run busybox uname -a
-   ```
-
-   The previous command shows details of the kernel version running inside the
-   container, which is different to the host kernel version.

docs/install/docker/opensuse-docker-install.md

@@ -1,75 +0,0 @@
-# Install Docker for Kata Containers on openSUSE
-
-> **Note:**
->
-> - This guide assumes you have
->   [already installed the Kata Containers packages](../opensuse-installation-guide.md).
-
-1. Install the latest version of Docker with the following commands:
-
-   > **Notes:**
-   >
-   > - This step is only required if Docker is not installed on the system.
-   > - Docker version 18.09 [removed devicemapper support](https://github.com/kata-containers/documentation/issues/373).
-   >   If you wish to use a block based backend, see the options listed on https://github.com/kata-containers/documentation/issues/407.
-
-   ```bash
-   $ sudo zypper -n install docker
-   ```
-
-   For more information on installing Docker please refer to the
-   [Docker Guide](https://software.opensuse.org/package/docker).
-
-2. Configure Docker to use Kata Containers by default with **ONE** of the following methods:
-
-   1. Specify the runtime options in `/etc/sysconfig/docker` (this is the default and is applied automatically if you select the
-      [automatic installation](../../install/README.md#automatic-installation) option)
-
-       ```bash
-       $ DOCKER_SYSCONFIG=/etc/sysconfig/docker
-       # Add kata-runtime to the list of available runtimes, if not already listed
-       $ grep -qE "^ *DOCKER_OPTS=.+--add-runtime[= ] *kata-runtime" $DOCKER_SYSCONFIG || sudo -E sed -i -E "s|^( *DOCKER_OPTS=.+)\" *$|\1 --add-runtime kata-runtime=/usr/bin/kata-runtime\"|g" $DOCKER_SYSCONFIG
-       # If a current default runtime is specified, overwrite it with kata-runtime
-       $ sudo -E sed -i -E "s|^( *DOCKER_OPTS=.+--default-runtime[= ] *)[^ \"]+(.*\"$)|\1kata-runtime\2|g" $DOCKER_SYSCONFIG
-       # Add kata-runtime as default runtime, if no default runtime is specified
-       $ grep -qE "^ *DOCKER_OPTS=.+--default-runtime" $DOCKER_SYSCONFIG || sudo -E sed -i -E "s|^( *DOCKER_OPTS=.+)(\"$)|\1 --default-runtime=kata-runtime\2|g" $DOCKER_SYSCONFIG
-       ```
-
-   2. Docker `daemon.json`
-
-      Create docker configuration folder.
-
-      ```
-      $ sudo mkdir -p /etc/docker
-      ```
-
-      Add the following definitions to `/etc/docker/daemon.json`:
-
-      ```json
-      {
-        "default-runtime": "kata-runtime",
-        "runtimes": {
-          "kata-runtime": {
-            "path": "/usr/bin/kata-runtime"
-          }
-        }
-      }
-      ```
-
-3. Restart the Docker systemd service with the following commands:
-
-   ```bash
-   $ sudo systemctl daemon-reload
-   $ sudo systemctl restart docker
-   ```
-
-4. Run Kata Containers
-
-   You are now ready to run Kata Containers:
-
-   ```bash
-   $ sudo docker run busybox uname -a
-   ```
-
-   The previous command shows details of the kernel version running inside the
-   container, which is different to the host kernel version.

docs/install/docker/opensuse-leap-docker-install.md

@@ -1,14 +0,0 @@
-# Install Docker for Kata Containers on openSUSE Leap
-
-Follow the instructions in the generic [openSUSE Docker install guide](opensuse-docker-install.md).
-<!--
-You can ignore the content of this comment.
-(test code run by test-install-docs.sh to validate code blocks this document)
-
-```bash
-$ echo "NOTE: this document is just a link to the generic openSUSE install guide located at:
-https://raw.githubusercontent.com/kata-containers/documentation/master/install/docker/opensuse-docker-install.md
-
-Please download this file and run kata-doc-to-script.sh again."
-```
--->

docs/install/docker/opensuse-tumbleweed-docker-install.md

@@ -1,14 +0,0 @@
-# Install Docker for Kata Containers on openSUSE Tumbleweed
-
-Follow the instructions in the generic [openSUSE Docker install guide](opensuse-docker-install.md).
-<!--
-You can ignore the content of this comment.
-(test code run by test-install-docs.sh to validate code blocks this document)
-
-```bash
-$ echo "NOTE: this document is just a link to the generic openSUSE install guide located at:
-https://raw.githubusercontent.com/kata-containers/documentation/master/install/docker/opensuse-docker-install.md
-
-Please download this file and run kata-doc-to-script.sh again."
-```
--->

docs/install/docker/rhel-docker-install.md

@@ -1,76 +0,0 @@
-# Install Docker for Kata Containers on RHEL
-
-> **Note:**
->
-> - This guide assumes you have
->   [already installed the Kata Containers packages](../rhel-installation-guide.md).
-
-1. Install the latest version of Docker with the following commands:
-
-   > **Notes:**
-   >
-   > - This step is only required if Docker is not installed on the system.
-   > - Docker version 18.09 [removed devicemapper support](https://github.com/kata-containers/documentation/issues/373).
-   >   If you wish to use a block based backend, see the options listed on https://github.com/kata-containers/documentation/issues/407.
-
-   ```bash
-   $ export rhel_devtoolset_version="7"
-   $ sudo subscription-manager repos --enable=rhel-${rhel_devtoolset_version}-server-extras-rpms
-   $ sudo yum -y install docker && systemctl enable --now docker
-   ```
-
-   For more information on installing Docker please refer to the
-   [Docker Guide](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux_atomic_host/7/html-single/getting_started_with_containers/#getting_docker_in_rhel_7).
-
-2. Configure Docker to use Kata Containers by default with **ONE** of the following methods:
-
-   1. systemd (this is the default and is applied automatically if you select the
-      [automatic installation](../../install/README.md#automatic-installation) option)
-
-       ```bash
-       $ sudo mkdir -p /etc/systemd/system/docker.service.d/
-       $ cat <<EOF | sudo tee /etc/systemd/system/docker.service.d/kata-containers.conf
-       [Service]
-       ExecStart=
-       ExecStart=/usr/bin/dockerd -D --add-runtime kata-runtime=/usr/bin/kata-runtime --default-runtime=kata-runtime
-       EOF
-       ```
-
-   2. Docker `daemon.json`
-
-      Create docker configuration folder.
-
-      ```
-      $ sudo mkdir -p /etc/docker
-      ```
-
-      Add the following definitions to `/etc/docker/daemon.json`:
-
-      ```json
-      {
-        "default-runtime": "kata-runtime",
-        "runtimes": {
-          "kata-runtime": {
-            "path": "/usr/bin/kata-runtime"
-          }
-        }
-      }
-      ```
-
-3. Restart the Docker systemd service with the following commands:
-
-   ```bash
-   $ sudo systemctl daemon-reload
-   $ sudo systemctl restart docker
-   ```
-
-4. Run Kata Containers
-
-   You are now ready to run Kata Containers:
-
-   ```bash
-   $ sudo docker run busybox uname -a
-   ```
-
-   The previous command shows details of the kernel version running inside the
-   container, which is different to the host kernel version.

docs/install/docker/sles-docker-install.md

@@ -1,74 +0,0 @@
-# Install Docker for Kata Containers on SLES
-
-> **Note:**
->
-> - This guide assumes you have
->   [already installed the Kata Containers packages](../sles-installation-guide.md).
-
-1. Install the latest version of Docker with the following commands:
-
-   > **Notes:**
-   >
-   > - This step is only required if Docker is not installed on the system.
-   > - Docker version 18.09 [removed devicemapper support](https://github.com/kata-containers/documentation/issues/373).
-   >   If you wish to use a block based backend, see the options listed on https://github.com/kata-containers/documentation/issues/407.
-
-   ```bash
-   $ sudo zypper -n install docker
-   ```
-
-   For more information on installing Docker please refer to the
-   [Docker Guide](https://www.suse.com/documentation/sles-12/singlehtml/book_sles_docker/book_sles_docker.html).
-
-2. Configure Docker to use Kata Containers by default with **ONE** of the following methods:
-
-   1. systemd (this is the default and is applied automatically if you select the
-      [automatic installation](../../install/README.md#automatic-installation) option)
-
-       ```bash
-       $ sudo mkdir -p /etc/systemd/system/docker.service.d/
-       $ cat <<EOF | sudo tee /etc/systemd/system/docker.service.d/kata-containers.conf
-       [Service]
-       ExecStart=
-       ExecStart=/usr/bin/dockerd -D --add-runtime kata-runtime=/usr/bin/kata-runtime --default-runtime=kata-runtime
-       EOF
-       ```
-
-   2. Docker `daemon.json`
-
-      Create docker configuration folder.
-
-      ```
-      $ sudo mkdir -p /etc/docker
-      ```
-
-      Add the following definitions to `/etc/docker/daemon.json`:
-
-      ```json
-      {
-        "default-runtime": "kata-runtime",
-        "runtimes": {
-          "kata-runtime": {
-            "path": "/usr/bin/kata-runtime"
-          }
-        }
-      }
-      ```
-
-3. Restart the Docker systemd service with the following commands:
-
-   ```bash
-   $ sudo systemctl daemon-reload
-   $ sudo systemctl restart docker
-   ```
-
-4. Run Kata Containers
-
-   You are now ready to run Kata Containers:
-
-   ```bash
-   $ sudo docker run busybox uname -a
-   ```
-
-   The previous command shows details of the kernel version running inside the
-   container, which is different to the host kernel version.

docs/install/docker/ubuntu-docker-install.md

@@ -1,79 +0,0 @@
-# Install Docker for Kata Containers on Ubuntu
-
-> **Note:**
->
-> - This guide assumes you have
->   [already installed the Kata Containers packages](../ubuntu-installation-guide.md).
-
-1. Install the latest version of Docker with the following commands:
-
-   > **Notes:**
-   >
-   > - This step is only required if Docker is not installed on the system.
-   > - Docker version 18.09 [removed devicemapper support](https://github.com/kata-containers/documentation/issues/373).
-   >   If you wish to use a block based backend, see the options listed on https://github.com/kata-containers/documentation/issues/407.
-
-   ```bash
-   $ sudo -E apt-get -y install apt-transport-https ca-certificates software-properties-common
-   $ curl -sL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-   $ arch=$(dpkg --print-architecture)
-   $ sudo -E add-apt-repository "deb [arch=${arch}] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
-   $ sudo -E apt-get update
-   $ sudo -E apt-get -y install docker-ce
-   ```
-
-   For more information on installing Docker please refer to the
-   [Docker Guide](https://docs.docker.com/engine/installation/linux/ubuntu).
-
-2. Configure Docker to use Kata Containers by default with **ONE** of the following methods:
-
-   1. systemd (this is the default and is applied automatically if you select the
-      [automatic installation](../../install/README.md#automatic-installation) option)
-
-       ```bash
-       $ sudo mkdir -p /etc/systemd/system/docker.service.d/
-       $ cat <<EOF | sudo tee /etc/systemd/system/docker.service.d/kata-containers.conf
-       [Service]
-       ExecStart=
-       ExecStart=/usr/bin/dockerd -D --add-runtime kata-runtime=/usr/bin/kata-runtime --default-runtime=kata-runtime
-       EOF
-       ```
-
-   2. Docker `daemon.json`
-
-      Create docker configuration folder.
-
-      ```
-      $ sudo mkdir -p /etc/docker
-      ```
-
-      Add the following definitions to `/etc/docker/daemon.json`:
-
-      ```json
-      {
-        "default-runtime": "kata-runtime",
-        "runtimes": {
-          "kata-runtime": {
-            "path": "/usr/bin/kata-runtime"
-          }
-        }
-      }
-      ```
-
-3. Restart the Docker systemd service with the following commands:
-
-   ```bash
-   $ sudo systemctl daemon-reload
-   $ sudo systemctl restart docker
-   ```
-
-4. Run Kata Containers
-
-   You are now ready to run Kata Containers:
-
-   ```bash
-   $ sudo docker run busybox uname -a
-   ```
-
-   The previous command shows details of the kernel version running inside the
-   container, which is different to the host kernel version.

docs/install/fedora-installation-guide.md

@@ -3,15 +3,8 @@
 1. Install the Kata Containers components with the following commands:
 
    ```bash
-   $ source /etc/os-release
-   $ ARCH=$(arch)
-   $ BRANCH="${BRANCH:-master}"
-   $ sudo dnf -y install dnf-plugins-core
-   $ sudo -E dnf config-manager --add-repo "http://download.opensuse.org/repositories/home:/katacontainers:/releases:/${ARCH}:/${BRANCH}/Fedora_${VERSION_ID}/home:katacontainers:releases:${ARCH}:${BRANCH}.repo"
-   $ sudo -E dnf -y install kata-runtime kata-proxy kata-shim
+   $ sudo -E dnf -y install kata-containers
    ```
 
 2. Decide which container manager to use and select the corresponding link that follows:
-
-   - [Docker](docker/fedora-docker-install.md)
    - [Kubernetes](../Developer-Guide.md#run-kata-containers-with-kubernetes)

docs/install/gce-installation-guide.md

@@ -1,12 +1,6 @@
 # Install Kata Containers on Google Compute Engine
 
-* [Create an Image with Nested Virtualization Enabled](#create-an-image-with-nested-virtualization-enabled)
-    * [Create the Image](#create-the-image)
-    * [Verify VMX is Available](#verify-vmx-is-available)
-* [Install Kata](#install-kata)
-* [Create a Kata-enabled Image](#create-a-kata-enabled-image)
-
-Kata Containers on Google Compute Engine (GCE) makes use of [nested virtualization](https://cloud.google.com/compute/docs/instances/enable-nested-virtualization-vm-instances). Most of the installation procedure is identical to that for Kata on your preferred distribution, but enabling nested virtualization currently requires extra steps on GCE. This guide walks you through creating an image and instance with nested virtualization enabled. Note that `kata-runtime kata-check` checks for nested virtualization, but does not fail if support is not found.
+Kata Containers on Google Compute Engine (GCE) makes use of [nested virtualization](https://cloud.google.com/compute/docs/instances/enable-nested-virtualization-vm-instances). Most of the installation procedure is identical to that for Kata on your preferred distribution, but enabling nested virtualization currently requires extra steps on GCE. This guide walks you through creating an image and instance with nested virtualization enabled. Note that `kata-runtime check` checks for nested virtualization, but does not fail if support is not found.
 
 As a pre-requisite this guide assumes an installed and configured instance of the [Google Cloud SDK](https://cloud.google.com/sdk/downloads). For a zero-configuration option, all of the commands below were been tested under [Google Cloud Shell](https://cloud.google.com/shell/) (as of Jun 2018). Verify your `gcloud` installation and configuration:
 

docs/install/installing-with-kata-doc-to-script.md

@@ -1,47 +0,0 @@
-# Installing with `kata-doc-to-script`
-
-* [Introduction](#introduction)
-* [Packages Installation](#packages-installation)
-* [Docker Installation and Setup](#docker-installation-and-setup)
-
-## Introduction
-Use [these installation instructions](README.md#supported-distributions) together with
-[`kata-doc-to-script`](https://github.com/kata-containers/tests/blob/master/.ci/kata-doc-to-script.sh)
-to generate installation bash scripts.
-
-> Note:
-> - Only the Docker container manager installation can be scripted. For other setups you must
-> install and configure the container manager manually.
-
-## Packages Installation
-
-```bash
-$ source /etc/os-release
-$ curl -fsSL -O https://raw.githubusercontent.com/kata-containers/documentation/master/install/${ID}-installation-guide.md
-$ bash -c "$(curl -fsSL https://raw.githubusercontent.com/kata-containers/tests/master/.ci/kata-doc-to-script.sh) ${ID}-installation-guide.md ${ID}-install.sh"
-```
-
-For example, if your distribution is CentOS, the previous example will generate a runnable shell script called `centos-install.sh`.
-To proceed with the installation, run:
-
-```bash
-$ source /etc/os-release
-$ bash "./${ID}-install.sh"
-```
-
-## Docker Installation and Setup
-
-```bash
-$ source /etc/os-release
-$ curl -fsSL -O https://raw.githubusercontent.com/kata-containers/documentation/master/install/docker/${ID}-docker-install.md
-$ bash -c "$(curl -fsSL https://raw.githubusercontent.com/kata-containers/tests/master/.ci/kata-doc-to-script.sh) ${ID}-docker-install.md ${ID}-docker-install.sh"
-```
-
-For example, if your distribution is CentOS, this will generate a runnable shell script called `centos-docker-install.sh`.
-
-To proceed with the Docker installation, run:
-
-```bash
-$ source /etc/os-release
-$ bash "./${ID}-docker-install.sh"
-```

docs/install/installing-with-kata-manager.md

@@ -1,47 +0,0 @@
-# Installing with `kata-manager`
-
-* [Introduction](#introduction)
-* [Full Installation](#full-installation)
-* [Install the Kata packages only](#install-the-kata-packages-only)
-* [Further Information](#further-information)
-
-## Introduction
-`kata-manager` automates the Kata Containers installation procedure documented for [these Linux distributions](README.md#supported-distributions).
-
-> **Note**:
-> - `kata-manager` requires `curl` and `sudo` installed on your system.
->
-> - Full installation mode is only available for Docker container manager. For other setups, you
-> can still use `kata-manager` to [install Kata package](#install-the-kata-packages-only), and then setup your container manager manually.
->
-> - You can run `kata-manager` in dry run mode by passing the `-n` flag. Dry run mode allows you to review the
-> commands that `kata-manager` would run, without doing any change to your system.
-
-
-## Full Installation
-This command does the following:
-1. Installs Kata Containers packages
-2. Installs Docker
-3. Configure Docker to use the Kata OCI runtime by default
-
-```bash
-$ bash -c "$(curl -fsSL https://raw.githubusercontent.com/kata-containers/tests/master/cmd/kata-manager/kata-manager.sh) install-docker-system"
-```
-
-<!--
-You can ignore the content of this comment.
-(test code run by test-install-docs.sh to validate code blocks this document)
-
-```bash
-$ bash -c "$(curl -fsSL https://raw.githubusercontent.com/kata-containers/tests/master/cmd/kata-manager/kata-manager.sh) remove-packages"
-```
--->
-## Install the Kata packages only
-Use the following command to only install Kata Containers packages.
-
-```bash
-$ bash -c "$(curl -fsSL https://raw.githubusercontent.com/kata-containers/tests/master/cmd/kata-manager/kata-manager.sh) install-packages"
-```
-
-## Further Information
-For more information on what `kata-manager` can do, refer to the [`kata-manager` page](https://github.com/kata-containers/tests/blob/master/cmd/kata-manager).