Merge pull request #4024 from bergwolf/2.4.0-branch-bump

# Kata Containers 2.4.0

.github/workflows/PR-wip-checks.yaml

@@ -15,6 +15,7 @@ jobs:
     name: WIP Check
     steps:
     - name: WIP Check
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       uses: tim-actions/wip-check@1c2a1ca6c110026b3e2297bb2ef39e1747b5a755
       with:
         labels: '["do-not-merge", "wip", "rfc"]'

.github/workflows/commit-message-check.yaml

@@ -18,24 +18,32 @@ jobs:
     name: Commit Message Check
     steps:
     - name: Get PR Commits
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       id: 'get-pr-commits'
-      uses: tim-actions/get-pr-commits@v1.0.0
+      uses: tim-actions/get-pr-commits@v1.2.0
       with:
         token: ${{ secrets.GITHUB_TOKEN }}
+        # Filter out revert commits
+        # The format of a revert commit is as follows:
+        #
+        # Revert "<original-subject-line>"
+        #
+        filter_out_pattern: '^Revert "'
 
     - name: DCO Check
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       uses: tim-actions/dco@2fd0504dc0d27b33f542867c300c60840c6dcb20
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
 
     - name: Commit Body Missing Check
-      if: ${{ success() || failure() }}
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && ( success() || failure() ) }}
       uses: tim-actions/commit-body-check@v1.0.2
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
 
     - name: Check Subject Line Length
-      if: ${{ success() || failure() }}
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && ( success() || failure() ) }}
       uses: tim-actions/commit-message-checker-with-regex@v0.3.1
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
@@ -44,7 +52,7 @@ jobs:
         post_error: ${{ env.error_msg }}
 
     - name: Check Body Line Length
-      if: ${{ success() || failure() }}
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && ( success() || failure() ) }}
       uses: tim-actions/commit-message-checker-with-regex@v0.3.1
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
@@ -71,7 +79,7 @@ jobs:
         post_error: ${{ env.error_msg }}
 
     - name: Check Fixes
-      if: ${{ success() || failure() }}
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && ( success() || failure() ) }}
       uses: tim-actions/commit-message-checker-with-regex@v0.3.1
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
@@ -82,7 +90,7 @@ jobs:
         one_pass_all_pass: 'true'
 
     - name: Check Subsystem
-      if: ${{ success() || failure() }}
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && ( success() || failure() ) }}
       uses: tim-actions/commit-message-checker-with-regex@v0.3.1
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}

.github/workflows/darwin-tests.yaml

@@ -0,0 +1,25 @@
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+
+name: Darwin tests
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.16.x, 1.17.x]
+        os: [macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Build utils
+      run: ./ci/darwin-test.sh

.github/workflows/kata-deploy-push.yaml

@@ -1,6 +1,15 @@
 name: kata deploy build
 
-on: [push, pull_request]
+on: 
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+    paths:
+      - tools/**
+      - versions.yaml
 
 jobs:
   build-asset:
@@ -9,7 +18,6 @@ jobs:
       matrix:
         asset:
           - kernel
-          - kernel-experimental
           - shim-v2
           - qemu
           - cloud-hypervisor
@@ -19,11 +27,13 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Install docker
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         run: |
           curl -fsSL https://test.docker.com -o test-docker.sh
           sh test-docker.sh
 
       - name: Build ${{ matrix.asset }}
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         run: |
           make "${KATA_ASSET}-tarball"
           build_dir=$(readlink -f build)
@@ -33,6 +43,7 @@ jobs:
           KATA_ASSET: ${{ matrix.asset }}
 
       - name: store-artifact ${{ matrix.asset }}
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         uses: actions/upload-artifact@v2
         with:
           name: kata-artifacts
@@ -45,14 +56,17 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: get-artifacts
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         uses: actions/download-artifact@v2
         with:
           name: kata-artifacts
           path: build
       - name: merge-artifacts
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         run: |
           make merge-builds
       - name: store-artifacts
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         uses: actions/upload-artifact@v2
         with:
           name: kata-static-tarball
@@ -63,6 +77,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: make kata-tarball
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         run: |
           make kata-tarball
           sudo make install-tarball

.github/workflows/kata-deploy-test.yaml

@@ -48,7 +48,16 @@ jobs:
           - rootfs-initrd
           - shim-v2
     steps:
+      - name: get-PR-ref
+        id: get-PR-ref
+        run: |
+            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
+            echo "reference for PR: " ${ref}
+            echo "##[set-output name=pr-ref;]${ref}"
       - uses: actions/checkout@v2
+        with:
+          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
+
       - name: Install docker
         run: |
           curl -fsSL https://test.docker.com -o test-docker.sh
@@ -75,7 +84,15 @@ jobs:
     runs-on: ubuntu-latest
     needs: build-asset
     steps:
+      - name: get-PR-ref
+        id: get-PR-ref
+        run: |
+            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
+            echo "reference for PR: " ${ref}
+            echo "##[set-output name=pr-ref;]${ref}"
       - uses: actions/checkout@v2
+        with:
+          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
       - name: get-artifacts
         uses: actions/download-artifact@v2
         with:
@@ -94,7 +111,15 @@ jobs:
     needs: create-kata-tarball
     runs-on: ubuntu-latest
     steps:
+      - name: get-PR-ref
+        id: get-PR-ref
+        run: |
+            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
+            echo "reference for PR: " ${ref}
+            echo "##[set-output name=pr-ref;]${ref}"
       - uses: actions/checkout@v2
+        with:
+          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
       - name: get-kata-tarball
         uses: actions/download-artifact@v2
         with:
@@ -102,18 +127,14 @@ jobs:
       - name: build-and-push-kata-deploy-ci
         id: build-and-push-kata-deploy-ci
         run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          pushd $GITHUB_WORKSPACE
-          git checkout $tag
-          pkg_sha=$(git rev-parse HEAD)
-          popd
+          PR_SHA=$(git log --format=format:%H -n1)
           mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
-          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t quay.io/kata-containers/kata-deploy-ci:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
+          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t quay.io/kata-containers/kata-deploy-ci:$PR_SHA $GITHUB_WORKSPACE/tools/packaging/kata-deploy
           docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
-          docker push quay.io/kata-containers/kata-deploy-ci:$pkg_sha
+          docker push quay.io/kata-containers/kata-deploy-ci:$PR_SHA
           mkdir -p packaging/kata-deploy
           ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
-          echo "::set-output name=PKG_SHA::${pkg_sha}"
+          echo "::set-output name=PKG_SHA::${PR_SHA}"
       - name: test-kata-deploy-ci-in-aks
         uses: ./packaging/kata-deploy/action
         with:

.github/workflows/move-issues-to-in-progress.yaml

@@ -16,6 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Install hub
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         run: |
           HUB_ARCH="amd64"
           HUB_VER=$(curl -sL "https://api.github.com/repos/github/hub/releases/latest" |\
@@ -26,6 +27,7 @@ jobs:
           sudo install hub /usr/local/bin
 
       - name: Install hub extension script
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         run: |
           # Clone into a temporary directory to avoid overwriting
           # any existing github directory.
@@ -35,9 +37,11 @@ jobs:
           popd &>/dev/null
 
       - name: Checkout code to allow hub to communicate with the project
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         uses: actions/checkout@v2
 
       - name: Move issue to "In progress"
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         env:
           GITHUB_TOKEN: ${{ secrets.KATA_GITHUB_ACTIONS_TOKEN }}
         run: |

.github/workflows/release.yaml

@@ -140,13 +140,10 @@ jobs:
       - uses: actions/checkout@v2
       - name: generate-and-upload-tarball
         run: |
-          pushd $GITHUB_WORKSPACE/src/agent
-          cargo vendor >> .cargo/config
-          popd
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
           tarball="kata-containers-$tag-vendor.tar.gz"
           pushd $GITHUB_WORKSPACE
-          tar -cvzf "${tarball}" src/agent/.cargo/config src/agent/vendor
+          bash -c "tools/packaging/release/generate_vendor.sh ${tarball}"
           GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}" 
           popd
 

.github/workflows/require-pr-porting-labels.yaml

@@ -20,6 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Install hub
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         run: |
           HUB_ARCH="amd64"
           HUB_VER=$(curl -sL "https://api.github.com/repos/github/hub/releases/latest" |\
@@ -30,6 +31,7 @@ jobs:
           sudo install hub /usr/local/bin
 
       - name: Checkout code to allow hub to communicate with the project
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         uses: actions/checkout@v2
 
       - name: Install porting checker script
@@ -42,6 +44,7 @@ jobs:
           popd &>/dev/null
 
       - name: Stop PR being merged unless it has a correct set of porting labels
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         env:
           GITHUB_TOKEN: ${{ secrets.KATA_GITHUB_ACTIONS_TOKEN }}
         run: |

.github/workflows/snap.yaml

@@ -1,17 +1,27 @@
 name: snap CI
-on: ["pull_request"]
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - edited
+
 jobs:
   test:
     runs-on: ubuntu-20.04
     steps:
       - name: Check out
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
 
       - name: Install Snapcraft
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         uses: samuelmeuli/action-snapcraft@v1
 
       - name: Build snap
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         run: |
           snapcraft -d snap --destructive-mode

.github/workflows/static-checks.yaml

@@ -5,8 +5,6 @@ on:
       - edited
       - reopened
       - synchronize
-      - labeled
-      - unlabeled
 
 name: Static checks
 jobs:

CONTRIBUTING.md

@@ -2,4 +2,4 @@
 
 ## This repo is part of [Kata Containers](https://katacontainers.io)
 
-For details on how to contribute to the Kata Containers project, please see the main [contributing document](https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md).
+For details on how to contribute to the Kata Containers project, please see the main [contributing document](https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md).

Glossary.md

@@ -1,94 +1,3 @@
 # Glossary
 
-[A](#a), [B](#b), [C](#c), [D](#d), [E](#e), [F](#f), [G](#g), [H](#h), [I](#i), [J](#j), [K](#k), [L](#l), [M](#m), [N](#n), [O](#o), [P](#p), [Q](#q), [R](#r), [S](#s), [T](#t), [U](#u), [V](#v), [W](#w), [X](#x), [Y](#y), [Z](#z)
-
-## A
-
-### Auto Scaling
-a method used in cloud computing, whereby the amount of computational resources in a server farm, typically measured in terms of the number of active servers, which vary automatically based on the load on the farm.
-
-## B
-
-## C
-
-### Container Security Solutions
-The process of implementing security tools and policies that will give you the assurance that everything in your container is running as intended, and only as intended.
-
-### Container Software
-A standard unit of software that packages up code and all its dependencies so the application runs quickly and reliably from one computing environment to another.
-
-### Container Runtime Interface
-A plugin interface which enables Kubelet to use a wide variety of container runtimes, without the need to recompile.
-
-### Container Virtualization
-A container is a virtual runtime environment that runs on top of a single operating system (OS) kernel and emulates an operating system rather than the underlying hardware.
-
-## D
-
-## E
-
-## F
-
-## G
-
-## H
-
-## I
-
-### Infrastructure Architecture
-A structured and modern approach for supporting an organization and facilitating innovation within an enterprise.
-
-## J
-
-## K
-
-### Kata Containers
-Kata containers is an open source project delivering increased container security and Workload isolation through an implementation of lightweight virtual machines.
-
-## L
-
-## M
-
-## N
-
-## O
-
-## P
-
-### Pod Containers
-A Group of one or more containers , with shared storage/network, and a specification for how to run the containers.
-
-### Private Cloud
-A computing model that offers a proprietary environment dedicated to a single business entity.
-
-### Public Cloud
-Computing services offered by third-party providers over the public Internet, making them available to anyone who wants to use or purchase them.
-
-## Q
-
-## R
-
-## S
-
-### Serverless Containers
-An architecture in which code is executed on-demand. Serverless workloads are typically in the cloud, but on-premises serverless platforms exist, too.
-
-## T
-
-## U
-
-## V
-
-### Virtual Machine Monitor
-Computer software, firmware or hardware that creates and runs virtual machines.
-
-### Virtual Machine Software
-A software program or operating system that not only exhibits the behavior of a separate computer, but is also capable of performing tasks such as running applications and programs like a separate computer.
-
-## W
-
-## X
-
-## Y
-
-## Z
+See the [project glossary hosted in the wiki](https://github.com/kata-containers/kata-containers/wiki/Glossary).

Makefile

@@ -8,12 +8,12 @@ COMPONENTS =
 
 COMPONENTS += agent
 COMPONENTS += runtime
-COMPONENTS += trace-forwarder
 
 # List of available tools
 TOOLS =
 
 TOOLS += agent-ctl
+TOOLS += trace-forwarder
 
 STANDARD_TARGETS = build check clean install test vendor
 
@@ -22,7 +22,7 @@ default: all
 all: logging-crate-tests build
 
 logging-crate-tests:
-	make -C pkg/logging
+	make -C src/libs/logging
 
 include utils.mk
 include ./tools/packaging/kata-deploy/local-build/Makefile

README.md

@@ -17,16 +17,73 @@ standard implementation of lightweight Virtual Machines (VMs) that feel and
 perform like containers, but provide the workload isolation and security
 advantages of VMs.
 
+## License
+
+The code is licensed under the Apache 2.0 license.
+See [the license file](LICENSE) for further details.
+
+## Platform support
+
+Kata Containers currently runs on 64-bit systems supporting the following
+technologies:
+
+| Architecture | Virtualization technology |
+|-|-|
+| `x86_64`, `amd64` | [Intel](https://www.intel.com) VT-x, AMD SVM |
+| `aarch64` ("`arm64`")| [ARM](https://www.arm.com) Hyp |
+| `ppc64le` | [IBM](https://www.ibm.com) Power |
+| `s390x` | [IBM](https://www.ibm.com) Z & LinuxONE SIE |
+
+### Hardware requirements
+
+The [Kata Containers runtime](src/runtime) provides a command to
+determine if your host system is capable of running and creating a
+Kata Container:
+
+```bash
+$ kata-runtime check
+```
+
+> **Notes:**
+>
+> - This command runs a number of checks including connecting to the
+>   network to determine if a newer release of Kata Containers is
+>   available on GitHub. If you do not wish this to check to run, add
+>   the `--no-network-checks` option.
+>
+> - By default, only a brief success / failure message is printed.
+>   If more details are needed, the `--verbose` flag can be used to display the
+>   list of all the checks performed.
+>
+> - If the command is run as the `root` user additional checks are
+>   run (including checking if another incompatible hypervisor is running).
+>   When running as `root`, network checks are automatically disabled.
+
 ## Getting started
 
 See the [installation documentation](docs/install).
 
 ## Documentation
 
-See the [official documentation](docs)
-(including [installation guides](docs/install),
-[the developer guide](docs/Developer-Guide.md),
-[design documents](docs/design) and more).
+See the [official documentation](docs) including:
+
+- [Installation guides](docs/install)
+- [Developer guide](docs/Developer-Guide.md)
+- [Design documents](docs/design)
+  - [Architecture overview](docs/design/architecture)
+
+## Configuration
+
+Kata Containers uses a single
+[configuration file](src/runtime/README.md#configuration)
+which contains a number of sections for various parts of the Kata
+Containers system including the [runtime](src/runtime), the
+[agent](src/agent) and the [hypervisor](#hypervisors).
+
+## Hypervisors
+
+See the [hypervisors document](docs/hypervisors.md) and the
+[Hypervisor specific configuration details](src/runtime/README.md#hypervisor-specific-configuration).
 
 ## Community
 
@@ -48,6 +105,8 @@ Please raise an issue
 
 ## Developers
 
+See the [developer guide](docs/Developer-Guide.md).
+
 ### Components
 
 ### Main components
@@ -70,8 +129,8 @@ The table below lists the remaining parts of the project:
 | [packaging](tools/packaging) | infrastructure | Scripts and metadata for producing packaged binaries<br/>(components, hypervisors, kernel and rootfs). |
 | [kernel](https://www.kernel.org) | kernel | Linux kernel used by the hypervisor to boot the guest image. Patches are stored [here](tools/packaging/kernel). |
 | [osbuilder](tools/osbuilder) | infrastructure | Tool to create "mini O/S" rootfs and initrd images and kernel for the hypervisor. |
-| [`agent-ctl`](tools/agent-ctl) | utility | Tool that provides low-level access for testing the agent. |
-| [`trace-forwarder`](src/trace-forwarder) | utility | Agent tracing helper. |
+| [`agent-ctl`](src/tools/agent-ctl) | utility | Tool that provides low-level access for testing the agent. |
+| [`trace-forwarder`](src/tools/trace-forwarder) | utility | Agent tracing helper. |
 | [`ci`](https://github.com/kata-containers/ci) | CI | Continuous Integration configuration files and scripts. |
 | [`katacontainers.io`](https://github.com/kata-containers/www.katacontainers.io) | Source for the [`katacontainers.io`](https://www.katacontainers.io) site. |
 
@@ -84,8 +143,4 @@ the [components](#components) section for further details.
 
 ## Glossary of Terms
 
-See the [glossary of terms](Glossary.md) related to Kata Containers.
----
-
-[kernel]: https://www.kernel.org
-[github-katacontainers.io]: https://github.com/kata-containers/www.katacontainers.io
+See the [glossary of terms](https://github.com/kata-containers/kata-containers/wiki/Glossary) related to Kata Containers.

VERSION

@@ -1 +1 @@
-2.4.0-alpha0
+2.4.0

ci/darwin-test.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2022 Apple Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+cidir=$(dirname "$0")
+runtimedir=$cidir/../src/runtime
+
+build_working_packages() {
+	# working packages:
+	device_api=$runtimedir/virtcontainers/device/api
+	device_config=$runtimedir/virtcontainers/device/config
+	device_drivers=$runtimedir/virtcontainers/device/drivers
+	device_manager=$runtimedir/virtcontainers/device/manager
+	rc_pkg_dir=$runtimedir/pkg/resourcecontrol/
+	utils_pkg_dir=$runtimedir/virtcontainers/utils
+
+	# broken packages :( :
+	#katautils=$runtimedir/pkg/katautils
+	#oci=$runtimedir/pkg/oci
+	#vc=$runtimedir/virtcontainers
+
+	pkgs=(
+		"$device_api"
+		"$device_config"
+		"$device_drivers"
+		"$device_manager"
+		"$utils_pkg_dir"
+		"$rc_pkg_dir")
+	for pkg in "${pkgs[@]}"; do
+		echo building "$pkg"
+		pushd "$pkg" &>/dev/null
+		go build
+		go test
+		popd &>/dev/null
+	done
+}
+
+build_working_packages

ci/go-test.sh

@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
 #
 # Copyright (c) 2020 Intel Corporation
 #

ci/install_go.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Copyright (c) 2019 Intel Corporation
 #

ci/install_libseccomp.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Copyright 2021 Sony Group Corporation
 #

ci/install_musl.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) 2020 Ant Group
 #
 # SPDX-License-Identifier: Apache-2.0

ci/install_rust.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) 2019 Ant Financial
 #
 # SPDX-License-Identifier: Apache-2.0

ci/install_vc.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Copyright (c) 2018 Intel Corporation
 #

ci/lib.sh

@@ -36,7 +36,7 @@ run_static_checks()
 	# Make sure we have the targeting branch
 	git remote set-branches --add origin "${branch}"
 	git fetch -a
-	bash "$tests_repo_dir/.ci/static-checks.sh" "github.com/kata-containers/kata-containers"
+	bash "$tests_repo_dir/.ci/static-checks.sh" "$@"
 }
 
 run_go_test()

ci/openshift-ci/images/Dockerfile.buildroot

@@ -4,6 +4,11 @@
 #
 # This is the build root image for Kata Containers on OpenShift CI.
 #
-FROM registry.centos.org/centos:8
+FROM quay.io/centos/centos:stream8
 
-RUN yum -y update && yum -y install git sudo wget
+RUN yum -y update && \
+    yum -y install \
+    git \
+    sudo \
+    wget && \
+    yum clean all

ci/run.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Copyright (c) 2019 Ant Financial
 #

ci/setup.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Copyright (c) 2018 Intel Corporation
 #

ci/static-checks.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Copyright (c) 2017-2018 Intel Corporation
 #
@@ -9,4 +9,4 @@ set -e
 cidir=$(dirname "$0")
 source "${cidir}/lib.sh"
 
-run_static_checks
+run_static_checks "${@:-github.com/kata-containers/kata-containers}"

docs/Developer-Guide.md

@@ -212,11 +212,13 @@ $ sudo systemctl restart systemd-journald
 >
 > - You should only do this step if you are testing with the latest version of the agent.
 
-The rust-agent is built with a static linked `musl.` To configure this:
+The agent is built with a statically linked `musl.` The default `libc` used is `musl`, but on `ppc64le` and `s390x`, `gnu` should be used. To configure this:
 
 ```
-rustup target add x86_64-unknown-linux-musl
-sudo ln -s /usr/bin/g++ /bin/musl-g++
+$ export ARCH=$(uname -m)
+$ if [ "$ARCH" = "ppc64le" -o "$ARCH" = "s390x" ]; then export LIBC=gnu; else export LIBC=musl; fi
+$ [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
+$ rustup target add ${ARCH}-unknown-linux-${LIBC}
 ```
 
 To build the agent:

docs/Limitations.md

@@ -57,6 +57,13 @@ for advice on which repository to raise the issue against.
 
 This section lists items that might be possible to fix.
 
+## OCI CLI commands
+
+### Docker and Podman support
+Currently Kata Containers does not support Docker or Podman.
+
+See issue https://github.com/kata-containers/kata-containers/issues/722 for more information.
+
 ## Runtime commands
 
 ### checkpoint and restore
@@ -86,21 +93,6 @@ All other configurations are supported and are working properly.
 
 ## Networking
 
-### Docker swarm and compose support
-
-The newest version of Docker supported is specified by the
-`externals.docker.version` variable in the
-[versions database](https://github.com/kata-containers/runtime/blob/master/versions.yaml).
-
-Basic Docker swarm support works. However, if you want to use custom networks
-with Docker's swarm, an older version of Docker is required. This is specified
-by the `externals.docker.meta.swarm-version` variable in the
-[versions database](https://github.com/kata-containers/runtime/blob/master/versions.yaml).
-
-See issue https://github.com/kata-containers/runtime/issues/175 for more information.
-
-Docker compose normally uses custom networks, so also has the same limitations.
-
 ## Resource management
 
 Due to the way VMs differ in their CPU and memory allocation, and sharing
@@ -112,82 +104,12 @@ See issue https://github.com/clearcontainers/runtime/issues/341 and [the constra
 For CPUs resource management see
 [CPU constraints](design/vcpu-handling.md).
 
-### docker run and shared memory
-
-The runtime does not implement the `docker run --shm-size` command to
-set the size of the `/dev/shm tmpfs` within the container. It is possible to pass this configuration value into the VM container so the appropriate mount command happens at launch time.
-
-See issue https://github.com/kata-containers/kata-containers/issues/21 for more information.
-
-### docker run and sysctl
-
-The `docker run --sysctl` feature is not implemented. At the runtime
-level, this equates to the `linux.sysctl` OCI configuration. Docker
-allows configuring the sysctl settings that support namespacing. From a security and isolation point of view, it might make sense to set them in the VM, which isolates sysctl settings. Also, given that each Kata Container has its own kernel, we can support setting of sysctl settings that are not namespaced. In some cases, we might need to support configuring some of the settings on both the host side Kata Container namespace and the Kata Containers kernel.
-
-See issue https://github.com/kata-containers/runtime/issues/185 for more information.
-
-## Docker daemon features
-
-Some features enabled or implemented via the
-[`dockerd` daemon](https://docs.docker.com/config/daemon/) configuration are not yet
-implemented.
-
-### SELinux support
-
-The `dockerd` configuration option `"selinux-enabled": true` is not presently implemented
-in Kata Containers. Enabling this option causes an OCI runtime error.
-
-See issue https://github.com/kata-containers/runtime/issues/784 for more information.
-
-The consequence of this is that the [Docker --security-opt is only partially supported](#docker---security-opt-option-partially-supported).
-
-Kubernetes [SELinux labels](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#assign-selinux-labels-to-a-container) will also not be applied.
-
 # Architectural limitations
 
 This section lists items that might not be fixed due to fundamental
 architectural differences between "soft containers" (i.e. traditional Linux*
 containers) and those based on VMs.
 
-## Networking limitations
-
-### Support for joining an existing VM network
-
-Docker supports the ability for containers to join another containers
-namespace with the `docker run --net=containers` syntax. This allows
-multiple containers to share a common network namespace and the network
-interfaces placed in the network namespace.  Kata Containers does not
-support network namespace sharing. If a Kata Container is setup to
-share the network namespace of a `runc` container, the runtime
-effectively takes over all the network interfaces assigned to the
-namespace and binds them to the VM. Consequently, the `runc` container loses
-its network connectivity.
-
-### docker --net=host
-
-Docker host network support (`docker --net=host run`) is not supported.
-It is not possible to directly access the host networking configuration
-from within the VM.
-
-The `--net=host` option can still be used with `runc` containers and
-inter-mixed with running Kata Containers, thus enabling use of `--net=host`
-when necessary.
-
-It should be noted, currently passing the `--net=host` option into a
-Kata Container may result in the Kata Container networking setup
-modifying, re-configuring and therefore possibly breaking the host
-networking setup. Do not use `--net=host` with Kata Containers.
-
-### docker run --link
-
-The runtime does not support the `docker run --link` command. This
-command is now deprecated by docker and we have no intention of adding support.
-Equivalent functionality can be achieved with the newer docker networking commands.
-
-See more documentation at
-[docs.docker.com](https://docs.docker.com/engine/userguide/networking/default_network/dockerlinks/).
-
 ## Storage limitations
 
 ### Kubernetes `volumeMounts.subPaths`
@@ -198,15 +120,11 @@ moment.
 See [this issue](https://github.com/kata-containers/runtime/issues/2812) for more details.
 [Another issue](https://github.com/kata-containers/kata-containers/issues/1728) focuses on the case of `emptyDir`.
 
-
 ## Host resource sharing
 
-### docker run --privileged
+### Privileged containers
 
 Privileged support in Kata is essentially different from `runc` containers.
-Kata does support `docker run --privileged` command, but in this case full access
-to the guest VM is provided in addition to some host access.
-
 The container runs with elevated capabilities within the guest and is granted
 access to guest devices instead of the host devices.
 This is also true with using `securityContext privileged=true` with Kubernetes.
@@ -216,17 +134,6 @@ The container may also be granted full access to a subset of host devices
 
 See [Privileged Kata Containers](how-to/privileged.md) for how to configure some of this behavior.
 
-# Miscellaneous
-
-This section lists limitations where the possible solutions are uncertain.
-
-## Docker --security-opt option partially supported
-
-The `--security-opt=` option used by Docker is partially supported.
-We only support `--security-opt=no-new-privileges` and `--security-opt seccomp=/path/to/seccomp/profile.json`
-option as of today.
-
-Note: The `--security-opt apparmor=your_profile` is not yet supported. See https://github.com/kata-containers/runtime/issues/707.
 # Appendices
 
 ## The constraints challenge

docs/README.md

@@ -21,14 +21,13 @@ See the [tracing documentation](tracing.md).
 * [Limitations](Limitations.md): differences and limitations compared with the default [Docker](https://www.docker.com/) runtime,
 [`runc`](https://github.com/opencontainers/runc).
 
-### Howto guides
+### How-to guides
 
-See the [howto documentation](how-to).
+See the [how-to documentation](how-to).
 
 ## Kata Use-Cases
 
 * [GPU Passthrough with Kata](./use-cases/GPU-passthrough-and-Kata.md)
-* [OpenStack Zun with Kata Containers](./use-cases/zun_kata.md)
 * [SR-IOV with Kata](./use-cases/using-SRIOV-and-kata.md)
 * [Intel QAT with Kata](./use-cases/using-Intel-QAT-and-kata.md)
 * [VPP with Kata](./use-cases/using-vpp-and-kata.md)
@@ -41,7 +40,7 @@ Documents that help to understand and contribute to Kata Containers.
 
 ### Design and Implementations
 
-* [Kata Containers Architecture](design/architecture.md): Architectural overview of Kata Containers
+* [Kata Containers Architecture](design/architecture): Architectural overview of Kata Containers
 * [Kata Containers E2E Flow](design/end-to-end-flow.md): The entire end-to-end flow of Kata Containers
 * [Kata Containers design](./design/README.md): More Kata Containers design documents
 * [Kata Containers threat model](./threat-model/threat-model.md): Kata Containers threat model
@@ -49,7 +48,7 @@ Documents that help to understand and contribute to Kata Containers.
 ### How to Contribute
 
 * [Developer Guide](Developer-Guide.md): Setup the Kata Containers developing environments
-* [How to contribute to Kata Containers](https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md)
+* [How to contribute to Kata Containers](https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md)
 * [Code of Conduct](../CODE_OF_CONDUCT.md)
 
 ## Help Writing a Code PR
@@ -59,6 +58,7 @@ Documents that help to understand and contribute to Kata Containers.
 ## Help Writing Unit Tests
 
 * [Unit Test Advice](Unit-Test-Advice.md)
+* [Unit testing presentation](presentations/unit-testing/kata-containers-unit-testing.md)
 
 ## Help Improving the Documents
 
@@ -73,6 +73,10 @@ Documents that help to understand and contribute to Kata Containers.
 * [Release strategy](Stable-Branch-Strategy.md)
 * [Release Process](Release-Process.md)
 
+## Presentations
+
+* [Presentations](presentations)
+
 ## Website Changes
 
 If you have a suggestion for how we can improve the

docs/Release-Process.md

@@ -48,6 +48,7 @@
 ### Merge all bump version Pull requests
 
   - The above step will create a GitHub pull request in the Kata projects. Trigger the CI using `/test` command on each bump Pull request.
+  - Trigger the test-kata-deploy workflow on the kata-containers repository bump Pull request using `/test_kata_deploy` (monitor under the "action" tab).
   - Check any failures and fix if needed.
   - Work with the Kata approvers to verify that the CI works and the pull requests are merged.
 
@@ -64,7 +65,7 @@
 
 ### Check Git-hub Actions
 
-  We make use of [GitHub actions](https://github.com/features/actions) in this [file](https://github.com/kata-containers/kata-containers/blob/main/.github/workflows/release.yaml) in the `kata-containers/kata-containers` repository to build and upload release artifacts. This action is auto triggered with the above step when a new tag is pushed to the `kata-containers/kata-containers` repository.
+  We make use of [GitHub actions](https://github.com/features/actions) in this [file](../.github/workflows/release.yaml) in the `kata-containers/kata-containers` repository to build and upload release artifacts. This action is auto triggered with the above step when a new tag is pushed to the `kata-containers/kata-containers` repository.
 
   Check the [actions status page](https://github.com/kata-containers/kata-containers/actions) to verify all steps in the actions workflow have completed successfully. On success, a static tarball containing Kata release artifacts will be uploaded to the [Release page](https://github.com/kata-containers/kata-containers/releases).
 

docs/Stable-Branch-Strategy.md

@@ -120,7 +120,7 @@ stable and main. While this is not in place currently, it should be considered i
 
 ### Patch releases
 
-Releases are made every three weeks, which include a GitHub release as
+Releases are made every four weeks, which include a GitHub release as
 well as binary packages. These patch releases are made for both stable branches, and a "release candidate"
 for the next `MAJOR` or `MINOR` is created from main. If there are no changes across all the repositories, no
 release is created and an announcement is made on the developer mailing list to highlight this.
@@ -136,8 +136,7 @@ The process followed for making a release can be found at [Release Process](Rele
 
 ###  Frequency
 Minor releases are less frequent in order to provide a more stable baseline for users. They are currently
-running on a twelve week cadence. As the Kata Containers code base has reached a certain level of 
-maturity, we have increased the cadence from six weeks to twelve weeks. The release schedule can be seen on the
+running on a sixteen weeks cadence. The release schedule can be seen on the
 [release rotation wiki page](https://github.com/kata-containers/community/wiki/Release-Team-Rota).
 
 ### Compatibility

docs/Unit-Test-Advice.md

@@ -286,8 +286,8 @@ func TestSomething(t *testing.T) {
     assert := assert.New(t)
 
     // Create a temporary directory
-    tmpdir, err := ioutil.TempDir("", "")    
-    assert.NoError(err)             
+    tmpdir, err := os.MkdirTemp("", "")
+    assert.NoError(err)
 
     // Delete it at the end of the test
     defer os.RemoveAll(tmpdir) 
@@ -337,7 +337,7 @@ will run if the correct type of user is detected and skipped if not.
 
 The main repository has the most comprehensive set of skip abilities. See:
 
-- https://github.com/kata-containers/kata-containers/tree/main/src/runtime/pkg/katatestutils
+- [`katatestutils`](../src/runtime/pkg/katatestutils)
 
 ### Run Rust tests as a different user
 

docs/Upgrading.md

@@ -102,7 +102,7 @@ first
 [install the latest release](#determine-latest-version).
 
 See the
-[manual installation installation documentation](install/README.md#manual-installation)
+[manual installation documentation](install/README.md#manual-installation)
 for details on how to automatically install and configuration a static release
 with containerd.
 
@@ -114,7 +114,7 @@ with containerd.
 > kernel or image.
 
 If you are using custom
-[guest assets](design/architecture.md#guest-assets),
+[guest assets](design/architecture/README.md#guest-assets),
 you must upgrade them to work with Kata Containers 2.x since Kata
 Containers 1.x assets will **not** work.
 

docs/code-pr-advice.md

@@ -154,7 +154,7 @@ func testFoo() error {
 ### Tracing
 
 Consider if the code needs to create a new
-[trace span](https://github.com/kata-containers/kata-containers/blob/main/docs/tracing.md).
+[trace span](./tracing.md).
 
 Ensure any new trace spans added to the code are completed.
 
@@ -165,7 +165,7 @@ Ensure any new trace spans added to the code are completed.
 Where possible, code changes should be accompanied by unit tests.
 
 Consider using the standard
-[table-based approach](https://github.com/kata-containers/tests/blob/main/Unit-Test-Advice.md)
+[table-based approach](Unit-Test-Advice.md)
 as it encourages you to make functions small and simple, and also
 allows you to think about what types of value to test.
 
@@ -198,6 +198,7 @@ The table below lists the small number of cases where use of
 | `defer!()` | Similar to golang's `defer()` but doesn't allow the use of `?`. |
 | `tokio::spawn(async move {})` | Cannot currently return a `Result` from an `async move` closure. |
 | If an explicit test is performed before the `unwrap()` / `expect()` | *"Just about acceptable"*, but not ideal `[*]` |
+| `Mutex.lock()` | Almost unrecoverable if failed in the lock acquisition |
 
 
 `[*]` - There can lead to bad *future* code: consider what would

docs/design/README.md

@@ -2,7 +2,7 @@
 
 Kata Containers design documents:
 
-- [Kata Containers architecture](architecture.md)
+- [Kata Containers architecture](architecture)
 - [API Design of Kata Containers](kata-api-design.md)
 - [Design requirements for Kata Containers](kata-design-requirements.md)
 - [VSocks](VSocks.md)
@@ -10,6 +10,7 @@ Kata Containers design documents:
 - [Host cgroups](host-cgroups.md)
 - [`Inotify` support](inotify.md)
 - [Metrics(Kata 2.0)](kata-2-0-metrics.md)
+- [Design for Kata Containers `Lazyload` ability with `nydus`](kata-nydus-design.md)
 
 ---
 

docs/design/arch-images/kata-nydus.drawio

@@ -0,0 +1 @@
+<mxfile host="app.diagrams.net" modified="2022-01-18T14:06:01.890Z" agent="5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36" etag="nId-8OV6FDjWTDgzqDu-" version="15.8.9" type="device"><diagram id="bkF_ZONM9sPFCpIYoGFl" name="Page-1">5Vtbj6M2GP01eUyEbW55nGSmM2q70mqnUnf6UrnBCdYSnIIzSfbX14AJYDsLSSDZUWdHGvxhDD7n8F1sdoTm6/1zgjfhJxaQaAStYD9CjyMIgQ1d8SezHAqL504LwyqhgexUGV7pdyKNlrRuaUDSRkfOWMTppmlcsDgmC96w4SRhu2a3JYuad93gFdEMrwsc6dY/acDDwupDr7K/ELoKyzuDcn5rXHaWM0lDHLBdzYSeRmieMMaLo/V+TqIMvBKX4rpfTpw9PlhCYt7lgpd4/J1SSP9++bR8Cb6A1de/XsZ2Mco7jrZywiFLuXxgfihR4GQv7jEL+ToSBiAOU56wb2TOIpYIS8xi0XO2pFGkmHBEV7FoLsRTEmGfvZOEU4HvgzyxpkGQ3Wa2Cyknrxu8yO65E2oStoRt44BkE7BES5+xBCEbk+xrJonAM2FrwpOD6CLP+o5kQ8rRsZ2ivavItWWXsMZrSSKWclodR64QFwcSdDMB8ycYj+f+Hw/+b3jxvN57e/vXsa8RoIHfBMEEU42XJYu5fIugfeSplC5QSBpBtHSyf/LKmr340ZgWZ9z858iHBr6BopN8INDkAwGdj6llIMSxh2JkamDEjbhEqEGN+++WlSfGaY76g+gA3c2+OimOVtnf+BBs03Ea400aMp69DHJY8ZTFyEW/H/AP+uC/D9aQNbFAkzjDiwQ8A3H+ULyVSrqCOARNxInQwjGNSRIMzth0OMacCYJN14csnTFnOkG+Tpo3GGnAQJqCJomDhyySZ1EkwmlKFzlKOOG6uYZr023WUBYTRDOBW3L4mp2cOGXzTV6ZNx738sqidWjEIBJoWYMWlFK2TRakg2DFTFaEt3kkndoab47JQ0pbQiLM6XvzeU1Eyjt8ZjR/W0rluErELD10OUQxT3lVPf9QBrIVV2+7ykAFDtpAua6O075Cauh6x97iH8ZpSNfjb5jj8TscxFn04Aocx2n3A65BUMM5AT0L7c+lwqFcqg8UHKEeAVGJdSOXdAYD0rle4tOTucvw4W8wrhyvyZU7NWQr0KB5dzCq3OupMqaZufcRVWnOzwfNVnxbiTlTg4tCP4h5/dPlXZin1KA7phxjkT3DRtZhTbxj+0Tikbc+k4SKCWWFdGHcU/61HF4cv1UJjWhVI2WNITIYdM/MxIOKStSEomtmosrNVVOcoTOTDosAncWl5LNWm6ykgirVvNX0dCMFdciBC0ruJjWkKAReKjWnZaCBpQZNRfLFUmu6sFYPdmdn1bXcuq9Xc1WFqClIV6mpA3nWjaV2aWlfl9oFkql5QgvYTYkC95Ioexd/Z/9MoVWLiJ39HWiJ0UOLEBpEeF6aDXxTmr3akrRzhv0zbZ9cl5grcdBxJL732j6BpqWDM/k1llHFNthHordZifn9EA6A4gmQYZXjtozraxxzoFFyaU2bB4hBalpggROpX1tRO9gaBNTXILLt6GX6IeH0O8KJBoNTXyOg6+zzAhGOPw6sSi3sGTZkgWlDdjhYTdXxmS7eMbn4NBSwBDQZZJ2s9OwRWfJ+qJmq+bxxq/yGxKAOteStNzc0t2BC6aZeodx1/d/LV0kdfeve8jXtB95ZvtNpO0i3VW+Hrbm2Iv70RjysL0DWS/xbrQkVL+e9qmzfP8H2uVW2Fhrs21bZyLTv2K9KykWd4wJkvx9rtK7HFFnIvZQCLNiXVFxVKt7kxmLRq47yo7g8mpmL63Mrahm4TtbTqXDjNF79nnd7tCvLF0leZmLi8mWUazYUFxIxwmyT4ZIj5czEr0Bznq1IOuJZ56INqrb4zbonfM5i8fiY5pojOOW7bO0okzzHHP+Tz1Sv4HvLiFzHLJ2adD3DZwrDxZRet7vO24MIcBoe43mP7qEQ9f3cg6VwrC6/dHUP6kYXALA//yCa1efuRffqPw2gp/8A</diagram></mxfile>

docs/design/architecture.md

@@ -1,290 +0,0 @@
-# Kata Containers Architecture
-
-## Overview
-
-This is an architectural overview of Kata Containers, based on the 2.0 release.
-
-The primary deliverable of the Kata Containers project is a CRI friendly shim. There is also a CRI friendly library API behind them.
-
-The [Kata Containers runtime](../../src/runtime)
-is compatible with the [OCI](https://github.com/opencontainers) [runtime specification](https://github.com/opencontainers/runtime-spec)
-and therefore works seamlessly with the [Kubernetes\* Container Runtime Interface (CRI)](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-node/container-runtime-interface.md)
-through the [CRI-O\*](https://github.com/kubernetes-incubator/cri-o) and
-[Containerd\*](https://github.com/containerd/containerd) implementation.
-
-Kata Containers creates a QEMU\*/KVM virtual machine for pod that `kubelet` (Kubernetes) creates respectively.
-
-The [`containerd-shim-kata-v2` (shown as `shimv2` from this point onwards)](../../src/runtime/cmd/containerd-shim-kata-v2/)
-is the Kata Containers entrypoint, which
-implements the [Containerd Runtime V2 (Shim API)](https://github.com/containerd/containerd/tree/master/runtime/v2) for Kata.
-
-Before `shimv2` (as done in [Kata Containers 1.x releases](https://github.com/kata-containers/runtime/releases)), we need to create a `containerd-shim` and a [`kata-shim`](https://github.com/kata-containers/shim) for each container and the Pod sandbox itself, plus an optional [`kata-proxy`](https://github.com/kata-containers/proxy) when VSOCK is not available. With `shimv2`, Kubernetes can launch Pod and OCI compatible containers with one shim (the `shimv2`) per Pod instead of `2N+1` shims, and no standalone `kata-proxy` process even if no VSOCK is available.
-
-![Kubernetes integration with shimv2](arch-images/shimv2.svg)
-
-The container process is then spawned by
-[`kata-agent`](../../src/agent), an agent process running
-as a daemon inside the virtual machine. `kata-agent` runs a [`ttRPC`](https://github.com/containerd/ttrpc-rust) server in
-the guest using a VIRTIO serial or VSOCK interface which QEMU exposes as a socket
-file on the host. `shimv2` uses a `ttRPC` protocol to communicate with
-the agent. This protocol allows the runtime to send container management
-commands to the agent. The protocol is also used to carry the I/O streams (stdout,
-stderr, stdin) between the containers and the manage engines (e.g. CRI-O or containerd).
-
-For any given container, both the init process and all potentially executed
-commands within that container, together with their related I/O streams, need
-to go through the VSOCK interface exported by QEMU.
-
-The container workload, that is, the actual OCI bundle rootfs, is exported from the
-host to the virtual machine.  In the case where a block-based graph driver is
-configured, `virtio-scsi` will be used. In all other cases a `virtio-fs` VIRTIO mount point
-will be used. `kata-agent` uses this mount point as the root filesystem for the
-container processes.
-
-## Virtualization
-
-How Kata Containers maps container concepts to virtual machine technologies, and how this is realized in the multiple
-hypervisors and VMMs that Kata supports is described within the [virtualization documentation](./virtualization.md)
-
-## Guest assets
-
-The hypervisor will launch a virtual machine which includes a minimal guest kernel
-and a guest image.
-
-### Guest kernel
-
-The guest kernel is passed to the hypervisor and used to boot the virtual
-machine. The default kernel provided in Kata Containers is highly optimized for
-kernel boot time and minimal memory footprint, providing only those services
-required by a container workload. This is based on a very current upstream Linux
-kernel.
-
-### Guest image
-
-Kata Containers supports both an `initrd` and `rootfs` based minimal guest image.
-
-#### Root filesystem image
-
-The default packaged root filesystem image, sometimes referred to as the "mini O/S", is a
-highly optimized container bootstrap system based on [Clear Linux](https://clearlinux.org/). It provides an extremely minimal environment and
-has a highly optimized boot path.
-
-The only services running in the context of the mini O/S are the init daemon
-(`systemd`) and the [Agent](#agent). The real workload the user wishes to run
-is created using libcontainer, creating a container in the same manner that is done
-by `runc`.
-
-For example, when `ctr run -ti ubuntu date` is run:
-
-- The hypervisor will boot the mini-OS image using the guest kernel.
-- `systemd`, running inside the mini-OS context, will launch the `kata-agent` in
-  the same context.
-- The agent will create a new confined context to run the specified command in
-  (`date` in this example).
-- The agent will then execute the command (`date` in this example) inside this
-  new context, first setting the root filesystem to the expected Ubuntu\* root
-  filesystem.
-
-#### Initrd image
-
-A compressed `cpio(1)` archive, created from a rootfs which is loaded into memory and used as part of the Linux startup process. During startup, the kernel unpacks it into a special instance of a `tmpfs` that becomes the initial root filesystem.
-
-The only service running in the context of the initrd is the [Agent](#agent) as the init daemon. The real workload the user wishes to run is created using libcontainer, creating a container in the same manner that is done by `runc`.
-
-## Agent
-
-[`kata-agent`](../../src/agent) is a process running in the guest as a supervisor for managing containers and processes running within those containers.
-
-For the 2.0 release, the `kata-agent` is rewritten in the [RUST programming language](https://www.rust-lang.org/) so that we can minimize its memory footprint while keeping the memory safety of the original GO version of [`kata-agent` used in Kata Container 1.x](https://github.com/kata-containers/agent). This memory footprint reduction is pretty impressive, from tens of megabytes down to less than 100 kilobytes, enabling Kata Containers in more use cases like functional computing and edge computing.
-
-The `kata-agent` execution unit is the sandbox. A `kata-agent` sandbox is a container sandbox defined by a set of namespaces (NS, UTS, IPC and PID). `shimv2` can
-run several containers per VM to support container engines that require multiple
-containers running inside a pod.
-
-`kata-agent` communicates with the other Kata components over `ttRPC`.
-
-## Runtime
-
-`containerd-shim-kata-v2` is a [containerd runtime shimv2](https://github.com/containerd/containerd/blob/v1.4.1/runtime/v2/README.md) implementation and is responsible for handling the `runtime v2 shim APIs`, which is similar to [the OCI runtime specification](https://github.com/opencontainers/runtime-spec) but simplifies the architecture by loading the runtime once and making RPC calls to handle the various container lifecycle commands. This refinement is an improvement on the OCI specification which requires the container manager call the runtime binary multiple times, at least once for each lifecycle command.
-
-`containerd-shim-kata-v2` heavily utilizes the
-[virtcontainers package](../../src/runtime/virtcontainers/), which provides a generic, runtime-specification agnostic, hardware-virtualized containers library.
-
-### Configuration
-
-The runtime uses a TOML format configuration file called `configuration.toml`. By default this file is installed in the `/usr/share/defaults/kata-containers` directory and contains various settings such as the paths to the hypervisor, the guest kernel and the mini-OS image.
-
-The actual configuration file paths can be determined by running:
-```
-$ kata-runtime --show-default-config-paths
-```
-Most users will not need to modify the configuration file.
-
-The file is well commented and provides a few "knobs" that can be used to modify the behavior of the runtime and your chosen hypervisor.
-
-The configuration file is also used to enable runtime [debug output](../Developer-Guide.md#enable-full-debug).
-
-## Networking
-
-Containers will typically live in their own, possibly shared, networking namespace.
-At some point in a container lifecycle, container engines will set up that namespace
-to add the container to a network which is isolated from the host network, but
-which is shared between containers
-
-In order to do so, container engines will usually add one end of a virtual
-ethernet (`veth`) pair into the container networking namespace. The other end of
-the `veth` pair is added to the host networking namespace.
-
-This is a very namespace-centric approach as many hypervisors/VMMs cannot handle `veth` 
-interfaces. Typically, `TAP` interfaces are created for VM connectivity.
-
-To overcome incompatibility between typical container engines expectations
-and virtual machines, Kata Containers networking transparently connects `veth`
-interfaces with `TAP` ones using Traffic Control:
-
-![Kata Containers networking](arch-images/network.png)
-
-With a TC filter in place, a redirection is created between the container network and the
-virtual machine. As an example, the CNI may create a device, `eth0`, in the container's network
-namespace, which is a VETH device. Kata Containers will create a tap device for the VM, `tap0_kata`,
-and setup a TC redirection filter to mirror traffic from `eth0`'s ingress to `tap0_kata`'s egress,
-and a second to mirror traffic from `tap0_kata`'s ingress to `eth0`'s egress.
-
-Kata Containers maintains support for MACVTAP, which was an earlier implementation used in Kata. TC-filter
-is the default because it allows for simpler configuration, better CNI plugin compatibility, and performance
-on par with MACVTAP.
-
-Kata Containers has deprecated support for bridge due to lacking performance relative to TC-filter and MACVTAP.
-
- Kata Containers supports both
-[CNM](https://github.com/docker/libnetwork/blob/master/docs/design.md#the-container-network-model)
-and [CNI](https://github.com/containernetworking/cni) for networking management.
-
-### Network Hotplug
-
-Kata Containers has developed a set of network sub-commands and APIs to add, list and
-remove a guest network endpoint and to manipulate the guest route table.
-
-The following diagram illustrates the Kata Containers network hotplug workflow.
-
-![Network Hotplug](arch-images/kata-containers-network-hotplug.png)
-
-## Storage
-Container workloads are shared with the virtualized environment through [virtio-fs](https://virtio-fs.gitlab.io/).
-
-The [devicemapper `snapshotter`](https://github.com/containerd/containerd/tree/master/snapshots/devmapper) is a special case. The `snapshotter` uses dedicated block devices rather than formatted filesystems, and operates at the block level rather than the file level. This knowledge is used to directly use the underlying block device instead of the overlay file system for the container root file system. The block device maps to the top read-write layer for the overlay. This approach gives much better I/O performance compared to using `virtio-fs` to share the container file system.
-
-Kata Containers has the ability to hotplug and remove block devices, which makes it possible to use block devices for containers started after the VM has been launched.
-
-Users can check to see if the container uses the devicemapper block device as its rootfs by calling `mount(8)` within the container.  If the devicemapper block device
-is used, `/` will be mounted on `/dev/vda`.  Users can disable direct mounting of the underlying block device through the runtime configuration.
-
-## Kubernetes support
-
-[Kubernetes\*](https://github.com/kubernetes/kubernetes/) is a popular open source
-container orchestration engine. In Kubernetes, a set of containers sharing resources
-such as networking, storage, mount, PID, etc. is called a
-[Pod](https://kubernetes.io/docs/user-guide/pods/).
-A node can have multiple pods, but at a minimum, a node within a Kubernetes cluster
-only needs to run a container runtime and a container agent (called a
-[Kubelet](https://kubernetes.io/docs/admin/kubelet/)).
-
-A Kubernetes cluster runs a control plane where a scheduler (typically running on a
-dedicated master node) calls into a compute Kubelet. This Kubelet instance is
-responsible for managing the lifecycle of pods within the nodes and eventually relies
-on a container runtime to handle execution. The Kubelet architecture decouples
-lifecycle management from container execution through the dedicated
-`gRPC` based [Container Runtime Interface (CRI)](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/node/container-runtime-interface-v1.md).
-
-In other words, a Kubelet is a CRI client and expects a CRI implementation to
-handle the server side of the interface.
-[CRI-O\*](https://github.com/kubernetes-incubator/cri-o) and [Containerd\*](https://github.com/containerd/containerd/) are CRI implementations that rely on [OCI](https://github.com/opencontainers/runtime-spec)
-compatible runtimes for managing container instances.
-
-Kata Containers is an officially supported CRI-O and Containerd runtime. Refer to the following guides on how to set up Kata Containers with Kubernetes:
-
-- [How to use Kata Containers and Containerd](../how-to/containerd-kata.md)
-- [Run Kata Containers with Kubernetes](../how-to/run-kata-with-k8s.md)
-
-####  OCI annotations
-
-In order for the Kata Containers runtime (or any virtual machine  based OCI compatible
-runtime) to be able to understand if it needs to create a full virtual machine or if it
-has to create a new container inside an existing pod's virtual machine, CRI-O adds
-specific annotations to the OCI configuration file (`config.json`) which is passed to
-the OCI compatible runtime.
-
-Before calling its runtime, CRI-O will always add a `io.kubernetes.cri-o.ContainerType`
-annotation to the `config.json` configuration file it produces from the Kubelet CRI
-request. The `io.kubernetes.cri-o.ContainerType` annotation can either be set to `sandbox`
-or `container`. Kata Containers will then use this annotation to decide if it needs to
-respectively create a virtual machine or a container inside a virtual machine associated
-with a Kubernetes pod:
-
-```Go
-	containerType, err := ociSpec.ContainerType()
-	if err != nil {
-		return err
-	}
-
-	handleFactory(ctx, runtimeConfig)
-
-	disableOutput := noNeedForOutput(detach, ociSpec.Process.Terminal)
-
-	var process vc.Process
-	switch containerType {
-	case vc.PodSandbox:
-		process, err = createSandbox(ctx, ociSpec, runtimeConfig, containerID, bundlePath, console, disableOutput, systemdCgroup)
-		if err != nil {
-			return err
-		}
-	case vc.PodContainer:
-		process, err = createContainer(ctx, ociSpec, containerID, bundlePath, console, disableOutput)
-		if err != nil {
-			return err
-		}
-	}
-
-```
-
-#### Mixing VM based and namespace based runtimes
-
-> **Note:** Since Kubernetes 1.12, the [`Kubernetes RuntimeClass`](https://kubernetes.io/docs/concepts/containers/runtime-class/)
-> has been supported and the user can specify runtime without the non-standardized annotations.
-
-With `RuntimeClass`, users can define Kata Containers as a `RuntimeClass` and then explicitly specify that a pod being created as a Kata Containers pod. For details, please refer to [How to use Kata Containers and Containerd](../../docs/how-to/containerd-kata.md).
-
-
-# Appendices
-
-## DAX
-
-Kata Containers utilizes the Linux kernel DAX [(Direct Access filesystem)](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/filesystems/dax.rst?h=v5.14)
-feature to efficiently map some host-side files into the guest VM space.
-In particular, Kata Containers uses the QEMU NVDIMM feature to provide a
-memory-mapped virtual device that can be used to DAX map the virtual machine's
-root filesystem into the guest memory address space.
-
-Mapping files using DAX provides a number of benefits over more traditional VM
-file and device mapping mechanisms:
-
-- Mapping as a direct access devices allows the guest to directly access
-  the host memory pages (such as via Execute In Place (XIP)), bypassing the guest
-  page cache. This provides both time and space optimizations.
-- Mapping as a direct access device inside the VM allows pages from the
-  host to be demand loaded using page faults, rather than having to make requests
-  via a virtualized device (causing expensive VM exits/hypercalls), thus providing
-  a speed optimization.
-- Utilizing `MAP_SHARED` shared memory on the host allows the host to efficiently
-  share pages.
-
-Kata Containers uses the following steps to set up the DAX mappings:
-1. QEMU is configured with an NVDIMM memory device, with a memory file
-  backend to map in the host-side file into the virtual NVDIMM space.
-2. The guest kernel command line mounts this NVDIMM device with the DAX
-  feature enabled, allowing direct page mapping and access, thus bypassing the
-  guest page cache.
-
-![DAX](arch-images/DAX.png)
-
-Information on the use of NVDIMM via QEMU is available in the [QEMU source code](http://git.qemu-project.org/?p=qemu.git;a=blob;f=docs/nvdimm.txt;hb=HEAD)

docs/design/architecture/README.md

@@ -0,0 +1,477 @@
+# Kata Containers Architecture
+
+## Overview
+
+Kata Containers is an open source community working to build a secure
+container [runtime](#runtime) with lightweight virtual machines (VM's)
+that feel and perform like standard Linux containers, but provide
+stronger [workload](#workload) isolation using hardware
+[virtualization](#virtualization) technology as a second layer of
+defence.
+
+Kata Containers runs on [multiple architectures](../../../src/runtime/README.md#platform-support)
+and supports [multiple hypervisors](../../hypervisors.md).
+
+This document is a summary of the Kata Containers architecture.
+
+## Background knowledge
+
+This document assumes the reader understands a number of concepts
+related to containers and file systems. The
+[background](background.md) document explains these concepts.
+
+## Example command
+
+This document makes use of a particular [example
+command](example-command.md) throughout the text to illustrate certain
+concepts.
+
+## Virtualization
+
+For details on how Kata Containers maps container concepts to VM
+technologies, and how this is realized in the multiple hypervisors and
+VMMs that Kata supports see the
+[virtualization documentation](../virtualization.md).
+
+## Compatibility
+
+The [Kata Containers runtime](../../../src/runtime) is compatible with
+the [OCI](https://github.com/opencontainers)
+[runtime specification](https://github.com/opencontainers/runtime-spec)
+and therefore works seamlessly with the
+[Kubernetes Container Runtime Interface (CRI)](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-node/container-runtime-interface.md)
+through the [CRI-O](https://github.com/kubernetes-incubator/cri-o)
+and [containerd](https://github.com/containerd/containerd)
+implementations.
+
+Kata Containers provides a ["shimv2"](#shim-v2-architecture) compatible runtime.
+
+## Shim v2 architecture
+
+The Kata Containers runtime is shim v2 ("shimv2") compatible. This
+section explains what this means.
+
+> **Note:**
+>
+> For a comparison with the Kata 1.x architecture, see
+> [the architectural history document](history.md).
+
+The
+[containerd runtime shimv2 architecture](https://github.com/containerd/containerd/tree/main/runtime/v2)
+or _shim API_ architecture resolves the issues with the old
+architecture by defining a set of shimv2 APIs that a compatible
+runtime implementation must supply. Rather than calling the runtime
+binary multiple times for each new container, the shimv2 architecture
+runs a single instance of the runtime binary (for any number of
+containers). This improves performance and resolves the state handling
+issue.
+
+The shimv2 API is similar to the
+[OCI runtime](https://github.com/opencontainers/runtime-spec)
+API in terms of the way the container lifecycle is split into
+different verbs. Rather than calling the runtime multiple times, the
+container manager creates a socket and passes it to the shimv2
+runtime. The socket is a bi-directional communication channel that
+uses a gRPC based protocol to allow the container manager to send API
+calls to the runtime, which returns the result to the container
+manager using the same channel.
+
+The shimv2 architecture allows running several containers per VM to
+support container engines that require multiple containers running
+inside a pod.
+
+With the new architecture [Kubernetes](kubernetes.md) can
+launch both Pod and OCI compatible containers with a single
+[runtime](#runtime) shim per Pod, rather than `2N+1` shims. No stand
+alone `kata-proxy` process is required, even if VSOCK is not
+available.
+
+## Workload
+
+The workload is the command the user requested to run in the
+container and is specified in the [OCI bundle](background.md#oci-bundle)'s
+configuration file.
+
+In our [example](example-command.md), the workload is the `sh(1)` command.
+
+### Workload root filesystem
+
+For details of how the [runtime](#runtime) makes the
+[container image](background.md#container-image) chosen by the user available to
+the workload process, see the
+[Container creation](#container-creation) and [storage](#storage) sections.
+
+Note that the workload is isolated from the [guest VM](#environments) environment by its
+surrounding [container environment](#environments). The guest VM
+environment where the container runs in is also isolated from the _outer_
+[host environment](#environments) where the container manager runs.
+
+## System overview
+
+### Environments
+
+The following terminology is used to describe the different or
+environments (or contexts) various processes run in. It is necessary
+to study this table closely to make sense of what follows:
+
+| Type | Name | Virtualized | Containerized | rootfs | Rootfs device type | Mount type | Description |
+|-|-|-|-|-|-|-|-|
+| Host | Host | no `[1]` | no | Host specific | Host specific | Host specific | The environment provided by a standard, physical non virtualized system. |
+| VM root | Guest VM | yes | no | rootfs inside the [guest image](guest-assets.md#guest-image) | Hypervisor specific `[2]` | `ext4` | The first (or top) level VM environment created on a host system. |
+| VM container root | Container | yes | yes | rootfs type requested by user ([`ubuntu` in the example](example-command.md)) | `kataShared` | [virtio FS](storage.md#virtio-fs) | The first (or top) level container environment created inside the VM. Based on the [OCI bundle](background.md#oci-bundle). |
+
+**Key:**
+
+- `[1]`: For simplicity, this document assumes the host environment
+  runs on physical hardware.
+
+- `[2]`: See the [DAX](#dax) section.
+
+> **Notes:**
+>
+> - The word "root" is used to mean _top level_ here in a similar
+>   manner to the term [rootfs](background.md#root-filesystem).
+>
+> - The term "first level" prefix used above is important since it implies
+>   that it is possible to create multi level systems. However, they do
+>   not form part of a standard Kata Containers environment so will not
+>   be considered in this document.
+
+The reasons for containerizing the [workload](#workload) inside the VM
+are:
+
+- Isolates the workload entirely from the VM environment.
+- Provides better isolation between containers in a [pod](kubernetes.md).
+- Allows the workload to be managed and monitored through its cgroup
+  confinement.
+
+### Container creation
+
+The steps below show at a high level how a Kata Containers container is
+created using the containerd container manager:
+
+1. The user requests the creation of a container by running a command
+   like the [example command](example-command.md).
+1. The container manager daemon runs a single instance of the Kata
+   [runtime](#runtime).
+1. The Kata runtime loads its [configuration file](#configuration).
+1. The container manager calls a set of shimv2 API functions on the runtime.
+1. The Kata runtime launches the configured [hypervisor](#hypervisor).
+1. The hypervisor creates and starts (_boots_) a VM using the
+   [guest assets](guest-assets.md#guest-assets):
+
+   - The hypervisor [DAX](#dax) shares the
+     [guest image](guest-assets.md#guest-image)
+     into the VM to become the VM [rootfs](background.md#root-filesystem) (mounted on a `/dev/pmem*` device),
+     which is known as the [VM root environment](#environments).
+   - The hypervisor mounts the [OCI bundle](background.md#oci-bundle), using [virtio FS](storage.md#virtio-fs),
+     into a container specific directory inside the VM's rootfs.
+
+     This container specific directory will become the
+     [container rootfs](#environments), known as the
+     [container environment](#environments).
+
+1. The [agent](#agent) is started as part of the VM boot.
+
+1. The runtime calls the agent's `CreateSandbox` API to request the
+   agent create a container:
+
+   1. The agent creates a [container environment](#environments)
+      in the container specific directory that contains the [container rootfs](#environments).
+
+      The container environment hosts the [workload](#workload) in the
+      [container rootfs](#environments) directory.
+
+   1. The agent spawns the workload inside the container environment.
+
+   > **Notes:**
+   >
+   > - The container environment created by the agent is equivalent to
+   >   a container environment created by the
+   >   [`runc`](https://github.com/opencontainers/runc) OCI runtime;
+   >   Linux cgroups and namespaces are created inside the VM by the
+   >   [guest kernel](guest-assets.md#guest-kernel) to isolate the
+   >   workload from the VM environment the container is created in.
+   >   See the [Environments](#environments) section for an
+   >   explanation of why this is done.
+   >
+   > - See the [guest image](guest-assets.md#guest-image) section for
+   >   details of exactly how the agent is started.
+
+1. The container manager returns control of the container to the
+   user running the `ctr` command.
+
+> **Note:**
+>
+> At this point, the container is running and:
+>
+> - The [workload](#workload) process ([`sh(1)` in the example](example-command.md))
+>   is running in the [container environment](#environments).
+> - The user is now able to interact with the workload
+>   (using the [`ctr` command in the example](example-command.md)).
+> - The [agent](#agent), running inside the VM is monitoring the
+>   [workload](#workload) process.
+> - The [runtime](#runtime) is waiting for the agent's `WaitProcess` API
+>   call to complete.
+
+Further details of these steps are provided in the sections below.
+
+### Container shutdown
+
+There are two possible ways for the container environment to be
+terminated:
+
+- When the [workload](#workload) exits.
+
+  This is the standard, or _graceful_ shutdown method.
+
+- When the container manager forces the container to be deleted.
+
+#### Workload exit
+
+The [agent](#agent) will detect when the [workload](#workload) process
+exits, capture its exit status (see `wait(2)`) and return that value
+to the [runtime](#runtime) by specifying it as the response to the
+`WaitProcess` agent API call made by the [runtime](#runtime).
+
+The runtime then passes the value back to the container manager by the
+`Wait` [shimv2 API](#shim-v2-architecture) call.
+
+Once the workload has fully exited, the VM is no longer needed and the
+runtime cleans up the environment (which includes terminating the
+[hypervisor](#hypervisor) process).
+
+> **Note:**
+>
+> When [agent tracing is enabled](../../tracing.md#agent-shutdown-behaviour),
+> the shutdown behaviour is different.
+
+#### Container manager requested shutdown
+
+If the container manager requests the container be deleted, the
+[runtime](#runtime) will signal the agent by sending it a
+`DestroySandbox` [ttRPC API](../../../src/libs/protocols/protos/agent.proto) request.
+
+## Guest assets
+
+The guest assets comprise a guest image and a guest kernel that are
+used by the [hypervisor](#hypervisor).
+
+See the [guest assets](guest-assets.md) document for further
+information.
+
+## Hypervisor
+
+The [hypervisor](../../hypervisors.md) specified in the
+[configuration file](#configuration) creates a VM to host the
+[agent](#agent) and the [workload](#workload) inside the
+[container environment](#environments).
+
+> **Note:**
+>
+> The hypervisor process runs inside an environment slightly different
+> to the host environment:
+>
+> - It is run in a different cgroup environment to the host.
+> - It is given a separate network namespace from the host.
+> - If the [OCI configuration specifies a SELinux label](https://github.com/opencontainers/runtime-spec/blob/main/config.md#linux-process),
+>   the hypervisor process will run with that label (*not* the workload running inside the hypervisor's VM).
+
+## Agent
+
+The Kata Containers agent ([`kata-agent`](../../../src/agent)), written
+in the [Rust programming language](https://www.rust-lang.org), is a
+long running process that runs inside the VM. It acts as the
+supervisor for managing the containers and the [workload](#workload)
+running within those containers. Only a single agent process is run
+for each VM created.
+
+### Agent communications protocol
+
+The agent communicates with the other Kata components (primarily the
+[runtime](#runtime)) using a
+[`ttRPC`](https://github.com/containerd/ttrpc-rust) based
+[protocol](../../../src/libs/protocols/protos).
+
+> **Note:**
+>
+> If you wish to learn more about this protocol, a practical way to do
+> so is to experiment with the
+> [agent control tool](#agent-control-tool) on a test system.
+> This tool is for test and development purposes only and can send
+> arbitrary ttRPC agent API commands to the [agent](#agent).
+
+## Runtime
+
+The Kata Containers runtime (the [`containerd-shim-kata-v2`](../../../src/runtime/cmd/containerd-shim-kata-v2
+) binary) is a [shimv2](#shim-v2-architecture) compatible runtime.
+
+> **Note:**
+>
+> The Kata Containers runtime is sometimes referred to as the Kata
+> _shim_. Both terms are correct since the `containerd-shim-kata-v2`
+> is a container runtime, and that runtime implements the containerd
+> shim v2 API.
+
+The runtime makes heavy use of the [`virtcontainers`
+package](../../../src/runtime/virtcontainers), which provides a generic,
+runtime-specification agnostic, hardware-virtualized containers
+library.
+
+The runtime is responsible for starting the [hypervisor](#hypervisor)
+and it's VM, and communicating with the [agent](#agent) using a
+[ttRPC based protocol](#agent-communications-protocol) over a VSOCK
+socket that provides a communications link between the VM and the
+host. 
+
+This protocol allows the runtime to send container management commands
+to the agent. The protocol is also used to carry the standard I/O
+streams (`stdout`, `stderr`, `stdin`) between the containers and
+container managers (such as CRI-O or containerd).
+
+## Utility program
+
+The `kata-runtime` binary is a utility program that provides
+administrative commands to manipulate and query a Kata Containers
+installation.
+
+> **Note:**
+>
+> In Kata 1.x, this program also acted as the main
+> [runtime](#runtime), but this is no longer required due to the
+> improved shimv2 architecture.
+
+### exec command
+
+The `exec` command allows an administrator or developer to enter the
+[VM root environment](#environments) which is not accessible by the container
+[workload](#workload).
+
+See [the developer guide](../../Developer-Guide.md#connect-to-debug-console) for further details.
+
+### Configuration
+
+See the [configuration file details](../../../src/runtime/README.md#configuration).
+
+The configuration file is also used to enable runtime [debug output](../../Developer-Guide.md#enable-full-debug).
+
+## Process overview
+
+The table below shows an example of the main processes running in the
+different [environments](#environments) when a Kata Container is
+created with containerd using our [example command](example-command.md):
+
+| Description | Host | VM root environment | VM container environment |
+|-|-|-|-|
+| Container manager | `containerd` | |
+| Kata Containers | [runtime](#runtime), [`virtiofsd`](storage.md#virtio-fs), [hypervisor](#hypervisor) | [agent](#agent) |
+| User [workload](#workload) | | | [`ubuntu sh`](example-command.md) |
+
+## Networking
+
+See the [networking document](networking.md).
+
+## Storage
+
+See the [storage document](storage.md).
+
+## Kubernetes support
+
+See the [Kubernetes document](kubernetes.md).
+
+####  OCI annotations
+
+In order for the Kata Containers [runtime](#runtime) (or any VM based OCI compatible
+runtime) to be able to understand if it needs to create a full VM or if it
+has to create a new container inside an existing pod's VM, CRI-O adds
+specific annotations to the OCI configuration file (`config.json`) which is passed to
+the OCI compatible runtime.
+
+Before calling its runtime, CRI-O will always add a `io.kubernetes.cri-o.ContainerType`
+annotation to the `config.json` configuration file it produces from the Kubelet CRI
+request. The `io.kubernetes.cri-o.ContainerType` annotation can either be set to `sandbox`
+or `container`. Kata Containers will then use this annotation to decide if it needs to
+respectively create a virtual machine or a container inside a virtual machine associated
+with a Kubernetes pod:
+
+| Annotation value | Kata VM created? | Kata container created? |
+|-|-|-|
+| `sandbox` | yes | yes (inside new VM) |
+| `container`| no | yes (in existing VM) |
+
+#### Mixing VM based and namespace based runtimes
+
+> **Note:** Since Kubernetes 1.12, the [`Kubernetes RuntimeClass`](https://kubernetes.io/docs/concepts/containers/runtime-class/)
+> has been supported and the user can specify runtime without the non-standardized annotations.
+
+With `RuntimeClass`, users can define Kata Containers as a
+`RuntimeClass` and then explicitly specify that a pod must be created
+as a Kata Containers pod. For details, please refer to [How to use
+Kata Containers and containerd](../../../docs/how-to/containerd-kata.md).
+
+## Tracing
+
+The [tracing document](../../tracing.md) provides details on the tracing
+architecture.
+
+# Appendices
+
+## DAX
+
+Kata Containers utilizes the Linux kernel DAX
+[(Direct Access filesystem)](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/filesystems/dax.rst?h=v5.14)
+feature to efficiently map the [guest image](guest-assets.md#guest-image) in the
+[host environment](#environments) into the
+[guest VM environment](#environments) to become the VM's
+[rootfs](background.md#root-filesystem).
+
+If the [configured](#configuration) [hypervisor](#hypervisor) is set
+to either QEMU or Cloud Hypervisor, DAX is used with the feature shown
+in the table below:
+
+| Hypervisor | Feature used | rootfs device type |
+|-|-|-|
+| Cloud Hypervisor (CH) | `dax` `FsConfig` configuration option | PMEM (emulated Persistent Memory device) |
+| QEMU | NVDIMM memory device with a memory file backend | NVDIMM (emulated Non-Volatile Dual In-line Memory Module device) |
+
+The features in the table above are equivalent in that they provide a memory-mapped
+virtual device which is used to DAX map the VM's
+[rootfs](background.md#root-filesystem) into the [VM guest](#environments) memory
+address space.
+
+The VM is then booted, specifying the `root=` kernel parameter to make
+the [guest kernel](guest-assets.md#guest-kernel) use the appropriate emulated device
+as its rootfs.
+
+### DAX advantages
+
+Mapping files using [DAX](#dax) provides a number of benefits over
+more traditional VM file and device mapping mechanisms:
+
+- Mapping as a direct access device allows the guest to directly
+  access the host memory pages (such as via Execute In Place (XIP)),
+  bypassing the [guest kernel](guest-assets.md#guest-kernel)'s page cache. This
+  zero copy provides both time and space optimizations.
+
+- Mapping as a direct access device inside the VM allows pages from the
+  host to be demand loaded using page faults, rather than having to make requests
+  via a virtualized device (causing expensive VM exits/hypercalls), thus providing
+  a speed optimization.
+
+- Utilizing `mmap(2)`'s `MAP_SHARED` shared memory option on the host
+  allows the host to efficiently share pages.
+
+![DAX](../arch-images/DAX.png)
+
+For further details of the use of NVDIMM with QEMU, see the [QEMU
+project documentation](https://www.qemu.org).
+
+## Agent control tool
+
+The [agent control tool](../../../src/tools/agent-ctl) is a test and
+development tool that can be used to learn more about a Kata
+Containers system.
+
+## Terminology
+
+See the [project glossary](../../../Glossary.md).

docs/design/architecture/background.md

@@ -0,0 +1,81 @@
+# Kata Containers architecture background knowledge
+
+The following sections explain some of the background concepts
+required to understand the [architecture document](README.md).
+
+## Root filesystem
+
+This document uses the term _rootfs_ to refer to a root filesystem
+which is mounted as the top-level directory ("`/`") and often referred
+to as _slash_.
+
+It is important to understand this term since the overall system uses
+multiple different rootfs's (as explained in the
+[Environments](README.md#environments) section.
+
+## Container image
+
+In the [example command](example-command.md) the user has specified the
+type of container they wish to run via the container image name:
+`ubuntu`. This image name corresponds to a _container image_ that can
+be used to create a container with an Ubuntu Linux environment. Hence,
+in our [example](example-command.md), the `sh(1)` command will be run
+inside a container which has an Ubuntu rootfs.
+
+> **Note:**
+>
+> The term _container image_ is confusing since the image in question
+> is **not** a container: it is simply a set of files (_an image_)
+> that can be used to _create_ a container. The term _container
+> template_ would be more accurate but the term _container image_ is
+> commonly used so this document uses the standard term.
+
+For the purposes of this document, the most important part of the
+[example command line](example-command.md) is the container image the
+user has requested. Normally, the container manager will _pull_
+(download) a container image from a remote site and store a copy
+locally. This local container image is used by the container manager
+to create an [OCI bundle](#oci-bundle) which will form the environment
+the container will run in. After creating the OCI bundle, the
+container manager launches a [runtime](README.md#runtime) which will create the
+container using the provided OCI bundle.
+
+## OCI bundle
+
+To understand what follows, it is important to know at a high level
+how an OCI ([Open Containers Initiative](https://opencontainers.org)) compatible container is created.
+
+An OCI compatible container is created by taking a
+[container image](#container-image) and converting the embedded rootfs
+into an
+[OCI rootfs bundle](https://github.com/opencontainers/runtime-spec/blob/main/bundle.md),
+or more simply, an _OCI bundle_.
+
+An OCI bundle is a `tar(1)` archive normally created by a container
+manager which is passed to an OCI [runtime](README.md#runtime) which converts
+it into a full container rootfs. The bundle contains two assets:
+
+- A container image [rootfs](#root-filesystem)
+
+  This is simply a directory of files that will be used to represent
+  the rootfs for the container.
+
+  For the [example command](example-command.md), the directory will
+  contain the files necessary to create a minimal Ubuntu root
+  filesystem.
+
+- An [OCI configuration file](https://github.com/opencontainers/runtime-spec/blob/main/config.md)
+
+  This is a JSON file called `config.json`.
+
+  The container manager will create this file so that:
+
+  - The `root.path` value is set to the full path of the specified
+    container rootfs.
+
+    In [the example](example-command.md) this value will be `ubuntu`.
+
+  - The `process.args` array specifies the list of commands the user
+    wishes to run. This is known as the [workload](README.md#workload).
+
+    In [the example](example-command.md) the workload is `sh(1)`.

docs/design/architecture/example-command.md

@@ -0,0 +1,30 @@
+# Example command
+
+The following containerd command creates a container. It is referred
+to throughout the architecture document to help explain various points:
+
+```bash
+$ sudo ctr run --runtime "io.containerd.kata.v2" --rm -t "quay.io/libpod/ubuntu:latest" foo sh
+```
+
+This command requests that containerd:
+
+- Create a container (`ctr run`).
+- Use the Kata [shimv2](README.md#shim-v2-architecture) runtime (`--runtime "io.containerd.kata.v2"`).
+- Delete the container when it [exits](README.md#workload-exit) (`--rm`).
+- Attach the container to the user's terminal (`-t`).
+- Use the Ubuntu Linux [container image](background.md#container-image)
+  to create the container [rootfs](background.md#root-filesystem) that will become
+  the [container environment](README.md#environments)
+  (`quay.io/libpod/ubuntu:latest`).
+- Create the container with the name "`foo`".
+- Run the `sh(1)` command in the Ubuntu rootfs based container
+  environment.
+
+  The command specified here is referred to as the [workload](README.md#workload).
+
+> **Note:**
+>
+> For the purposes of this document and to keep explanations
+> simpler, we assume the user is running this command in the
+> [host environment](README.md#environments).

docs/design/architecture/guest-assets.md

@@ -0,0 +1,152 @@
+# Guest assets
+
+Kata Containers creates a VM in which to run one or more containers.
+It does this by launching a [hypervisor](README.md#hypervisor) to
+create the VM. The hypervisor needs two assets for this task: a Linux
+kernel and a small root filesystem image to boot the VM.
+
+## Guest kernel
+
+The [guest kernel](../../../tools/packaging/kernel)
+is passed to the hypervisor and used to boot the VM.
+The default kernel provided in Kata Containers is highly optimized for
+kernel boot time and minimal memory footprint, providing only those
+services required by a container workload. It is based on the latest
+Linux LTS (Long Term Support) [kernel](https://www.kernel.org).
+
+## Guest image
+
+The hypervisor uses an image file which provides a minimal root
+filesystem used by the guest kernel to boot the VM and host the Kata
+Container. Kata Containers supports both initrd and rootfs based
+minimal guest images. The [default packages](../../install/) provide both
+an image and an initrd, both of which are created using the
+[`osbuilder`](../../../tools/osbuilder) tool.
+
+> **Notes:**
+>
+> - Although initrd and rootfs based images are supported, not all
+>   [hypervisors](README.md#hypervisor) support both types of image.
+>
+> - The guest image is *unrelated* to the image used in a container
+>   workload.
+>
+>   For example, if a user creates a container that runs a shell in a
+>   BusyBox image, they will run that shell in a BusyBox environment.
+>   However, the guest image running inside the VM that is used to
+>   *host* that BusyBox image could be running Clear Linux, Ubuntu,
+>   Fedora or any other distribution potentially.
+>
+>   The `osbuilder` tool provides
+>   [configurations for various common Linux distributions](../../../tools/osbuilder/rootfs-builder)
+>   which can be built into either initrd or rootfs guest images.
+>
+> - If you are using a [packaged version of Kata
+>   Containers](../../install), you can see image details by running the
+>   [`kata-collect-data.sh`](../../../src/runtime/data/kata-collect-data.sh.in)
+>   script as `root` and looking at the "Image details" section of the
+>   output.
+
+#### Root filesystem image
+
+The default packaged rootfs image, sometimes referred to as the _mini
+O/S_, is a highly optimized container bootstrap system.
+
+If this image type is [configured](README.md#configuration), when the
+user runs the [example command](example-command.md):
+
+- The [runtime](README.md#runtime) will launch the configured [hypervisor](README.md#hypervisor).
+- The hypervisor will boot the mini-OS image using the [guest kernel](#guest-kernel).
+- The kernel will start the init daemon as PID 1 (`systemd`) inside the VM root environment.
+- `systemd`, running inside the mini-OS context, will launch the [agent](README.md#agent)
+  in the root context of the VM.
+- The agent will create a new container environment, setting its root
+  filesystem to that requested by the user (Ubuntu in [the example](example-command.md)).
+- The agent will then execute the command (`sh(1)` in [the example](example-command.md))
+  inside the new container.
+
+The table below summarises the default mini O/S showing the
+environments that are created, the services running in those
+environments (for all platforms) and the root filesystem used by
+each service:
+
+| Process | Environment | systemd service? | rootfs | User accessible | Notes |
+|-|-|-|-|-|-|
+| systemd | VM root | n/a | [VM guest image](#guest-image)| [debug console][debug-console] | The init daemon, running as PID 1 |
+| [Agent](README.md#agent) | VM root | yes | [VM guest image](#guest-image)| [debug console][debug-console] | Runs as a systemd service |
+| `chronyd` | VM root | yes | [VM guest image](#guest-image)| [debug console][debug-console] | Used to synchronise the time with the host |
+| container workload (`sh(1)` in [the example](example-command.md)) | VM container | no | User specified (Ubuntu in [the example](example-command.md)) | [exec command](README.md#exec-command) | Managed by the agent |
+
+See also the [process overview](README.md#process-overview).
+
+> **Notes:**
+>
+> - The "User accessible" column shows how an administrator can access
+>   the environment.
+>
+> - The container workload is running inside a full container
+>   environment which itself is running within a VM environment.
+>
+> - See the [configuration files for the `osbuilder` tool](../../../tools/osbuilder/rootfs-builder)
+>   for details of the default distribution for platforms other than
+>   Intel x86_64.
+
+#### Initrd image
+
+The initrd image is a compressed `cpio(1)` archive, created from a
+rootfs which is loaded into memory and used as part of the Linux
+startup process. During startup, the kernel unpacks it into a special
+instance of a `tmpfs` mount that becomes the initial root filesystem.
+
+If this image type is [configured](README.md#configuration), when the user runs
+the [example command](example-command.md):
+
+- The [runtime](README.md#runtime) will launch the configured [hypervisor](README.md#hypervisor).
+- The hypervisor will boot the mini-OS image using the [guest kernel](#guest-kernel).
+- The kernel will start the init daemon as PID 1 (the
+  [agent](README.md#agent))
+  inside the VM root environment.
+- The [agent](README.md#agent) will create a new container environment, setting its root
+  filesystem to that requested by the user (`ubuntu` in
+  [the example](example-command.md)).
+- The agent will then execute the command (`sh(1)` in [the example](example-command.md))
+  inside the new container.
+
+The table below summarises the default mini O/S showing the environments that are created,
+the processes running in those environments (for all platforms) and
+the root filesystem used by each service:
+
+| Process | Environment | rootfs | User accessible | Notes |
+|-|-|-|-|-|
+| [Agent](README.md#agent) | VM root | [VM guest image](#guest-image) | [debug console][debug-console] | Runs as the init daemon (PID 1) |
+| container workload | VM container | User specified (Ubuntu in this example) | [exec command](README.md#exec-command) | Managed by the agent |
+
+> **Notes:**
+>
+> - The "User accessible" column shows how an administrator can access
+>   the environment.
+>
+> - It is possible to use a standard init daemon such as systemd with
+>   an initrd image if this is desirable.
+
+See also the [process overview](README.md#process-overview).
+
+#### Image summary
+
+| Image type | Default distro | Init daemon | Reason | Notes |
+|-|-|-|-|-|
+| [image](background.md#root-filesystem-image) | [Clear Linux](https://clearlinux.org) (for x86_64 systems)| systemd | Minimal and highly optimized | systemd offers flexibility |
+| [initrd](#initrd-image) | [Alpine Linux](https://alpinelinux.org) | Kata [agent](README.md#agent) (as no systemd support) | Security hardened and tiny C library |
+
+See also:
+
+- The [osbuilder](../../../tools/osbuilder) tool
+
+  This is used to build all default image types.
+
+- The [versions database](../../../versions.yaml)
+
+  The `default-image-name` and `default-initrd-name` options specify
+  the default distributions for each image type.
+
+[debug-console]: ../../Developer-Guide.md#connect-to-debug-console

docs/design/architecture/history.md

@@ -0,0 +1,41 @@
+# History
+
+## Kata 1.x architecture
+
+In the old [Kata 1.x architecture](https://github.com/kata-containers/documentation/blob/master/design/architecture.md),
+the Kata [runtime](README.md#runtime) was an executable called `kata-runtime`.
+The container manager called this executable multiple times when
+creating each container. Each time the runtime was called a different
+OCI command-line verb was provided. This architecture was simple, but
+not well suited to creating VM based containers due to the issue of
+handling state between calls. Additionally, the architecture suffered
+from performance issues related to continually having to spawn new
+instances of the runtime binary, and
+[Kata shim](https://github.com/kata-containers/shim) and
+[Kata proxy](https://github.com/kata-containers/proxy) processes for systems
+that did not provide VSOCK.
+
+## Kata 2.x architecture
+
+See the ["shimv2"](README.md#shim-v2-architecture) section of the
+architecture document.
+
+## Architectural comparison
+
+| Kata version | Kata Runtime process calls | Kata shim processes | Kata proxy processes (if no VSOCK) |
+|-|-|-|-|
+| 1.x | multiple per container | 1 per container connection | 1 |
+| 2.x | 1 per VM (hosting any number of containers) | 0 | 0 |
+
+> **Notes:**
+>
+> - A single VM can host one or more containers.
+>
+> - The "Kata shim processes" column refers to the old
+>   [Kata shim](https://github.com/kata-containers/shim) (`kata-shim` binary),
+>   *not* the new shimv2 runtime instance (`containerd-shim-kata-v2` binary).
+
+The diagram below shows how the original architecture was simplified
+with the advent of shimv2.
+
+![Kubernetes integration with shimv2](../arch-images/shimv2.svg)

docs/design/architecture/kubernetes.md

@@ -0,0 +1,35 @@
+# Kubernetes support
+
+[Kubernetes](https://github.com/kubernetes/kubernetes/), or K8s, is a popular open source
+container orchestration engine. In Kubernetes, a set of containers sharing resources
+such as networking, storage, mount, PID, etc. is called a
+[pod](https://kubernetes.io/docs/user-guide/pods/).
+
+A node can have multiple pods, but at a minimum, a node within a Kubernetes cluster
+only needs to run a container runtime and a container agent (called a
+[Kubelet](https://kubernetes.io/docs/admin/kubelet/)).
+
+Kata Containers represents a Kubelet pod as a VM.
+
+A Kubernetes cluster runs a control plane where a scheduler (typically
+running on a dedicated master node) calls into a compute Kubelet. This
+Kubelet instance is responsible for managing the lifecycle of pods
+within the nodes and eventually relies on a container runtime to
+handle execution. The Kubelet architecture decouples lifecycle
+management from container execution through a dedicated gRPC based
+[Container Runtime Interface (CRI)](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/node/container-runtime-interface-v1.md).
+
+In other words, a Kubelet is a CRI client and expects a CRI
+implementation to handle the server side of the interface.
+[CRI-O](https://github.com/kubernetes-incubator/cri-o) and
+[containerd](https://github.com/containerd/containerd/) are CRI
+implementations that rely on
+[OCI](https://github.com/opencontainers/runtime-spec) compatible
+runtimes for managing container instances.
+
+Kata Containers is an officially supported CRI-O and containerd
+runtime. Refer to the following guides on how to set up Kata
+Containers with Kubernetes:
+
+- [How to use Kata Containers and containerd](../../how-to/containerd-kata.md)
+- [Run Kata Containers with Kubernetes](../../how-to/run-kata-with-k8s.md)

docs/design/architecture/networking.md

@@ -0,0 +1,49 @@
+# Networking
+
+Containers typically live in their own, possibly shared, networking namespace.
+At some point in a container lifecycle, container engines will set up that namespace
+to add the container to a network which is isolated from the host network.
+
+In order to setup the network for a container, container engines call into a 
+networking plugin. The network plugin will usually create a virtual
+ethernet (`veth`) pair adding one end of the `veth` pair into the container 
+networking namespace, while the other end of the `veth` pair is added to the 
+host networking namespace.
+
+This is a very namespace-centric approach as many hypervisors or VM
+Managers (VMMs) such as `virt-manager` cannot handle `veth`
+interfaces. Typically, [`TAP`](https://www.kernel.org/doc/Documentation/networking/tuntap.txt) 
+interfaces are created for VM connectivity.
+
+To overcome incompatibility between typical container engines expectations
+and virtual machines, Kata Containers networking transparently connects `veth`
+interfaces with `TAP` ones using [Traffic Control](https://man7.org/linux/man-pages/man8/tc.8.html):
+
+![Kata Containers networking](../arch-images/network.png)
+
+With a TC filter rules in place, a redirection is created between the container network
+ and the virtual machine. As an example, the network plugin may place a device, 
+`eth0`, in the container's network namespace, which is one end of a VETH device. 
+Kata Containers will create a tap device for the VM, `tap0_kata`,
+and setup a TC redirection filter to redirect traffic from `eth0`'s ingress to `tap0_kata`'s egress,
+and a second TC filter to redirect traffic from `tap0_kata`'s ingress to `eth0`'s egress.
+
+Kata Containers maintains support for MACVTAP, which was an earlier implementation used in Kata. 
+With this method, Kata created a MACVTAP device to connect directly to the `eth0` device. 
+TC-filter is the default because it allows for simpler configuration, better CNI plugin 
+compatibility, and performance on par with MACVTAP.
+
+Kata Containers has deprecated support for bridge due to lacking performance relative to TC-filter and MACVTAP.
+
+Kata Containers supports both
+[CNM](https://github.com/docker/libnetwork/blob/master/docs/design.md#the-container-network-model)
+and [CNI](https://github.com/containernetworking/cni) for networking management.
+
+## Network Hotplug
+
+Kata Containers has developed a set of network sub-commands and APIs to add, list and
+remove a guest network endpoint and to manipulate the guest route table.
+
+The following diagram illustrates the Kata Containers network hotplug workflow.
+
+![Network Hotplug](../arch-images/kata-containers-network-hotplug.png)

docs/design/architecture/storage.md

@@ -0,0 +1,44 @@
+# Storage
+
+## virtio SCSI
+
+If a block-based graph driver is [configured](README.md#configuration),
+`virtio-scsi` is used to _share_ the workload image (such as
+`busybox:latest`) into the container's environment inside the VM.
+
+## virtio FS
+
+If a block-based graph driver is _not_ [configured](README.md#configuration), a
+[`virtio-fs`](https://virtio-fs.gitlab.io) (`VIRTIO`) overlay
+filesystem mount point is used to _share_ the workload image instead. The
+[agent](README.md#agent) uses this mount point as the root filesystem for the
+container processes.
+
+For virtio-fs, the [runtime](README.md#runtime) starts one `virtiofsd` daemon
+(that runs in the host context) for each VM created.
+
+## Devicemapper
+
+The
+[devicemapper `snapshotter`](https://github.com/containerd/containerd/tree/master/snapshots/devmapper)
+is a special case. The `snapshotter` uses dedicated block devices
+rather than formatted filesystems, and operates at the block level
+rather than the file level. This knowledge is used to directly use the
+underlying block device instead of the overlay file system for the
+container root file system. The block device maps to the top
+read-write layer for the overlay. This approach gives much better I/O
+performance compared to using `virtio-fs` to share the container file
+system.
+
+#### Hot plug and unplug
+
+Kata Containers has the ability to hot plug add and hot plug remove
+block devices. This makes it possible to use block devices for
+containers started after the VM has been launched.
+
+Users can check to see if the container uses the `devicemapper` block
+device as its rootfs by calling `mount(8)` within the container. If
+the `devicemapper` block device is used, the root filesystem (`/`)
+will be mounted from `/dev/vda`. Users can disable direct mounting of
+the underlying block device through the runtime
+[configuration](README.md#configuration).

docs/design/host-cgroups.md

@@ -19,7 +19,7 @@ Cgroups are hierarchical, and this can be seen with the following pod example:
   - Container 2: `cgroupsPath=/kubepods/pod1/container2`
 
 - Pod 2: `cgroupsPath=/kubepods/pod2`
-  - Container 1: `cgroupsPath=/kubepods/pod2/container2`
+  - Container 1: `cgroupsPath=/kubepods/pod2/container1`
   - Container 2: `cgroupsPath=/kubepods/pod2/container2`
 
 Depending on the upper-level orchestration layers, the cgroup under which the pod is placed is
@@ -242,8 +242,8 @@ On the other hand, running all non vCPU threads under a dedicated overhead cgrou
 accurate metrics on the actual Kata Container pod overhead, allowing for tuning the overhead
 cgroup size and constraints accordingly.
 
-[linux-config]: https://github.com/opencontainers/runtime-spec/blob/master/config-linux.md
-[cgroupspath]: https://github.com/opencontainers/runtime-spec/blob/master/config-linux.md#cgroups-path
+[linux-config]: https://github.com/opencontainers/runtime-spec/blob/main/config-linux.md
+[cgroupspath]: https://github.com/opencontainers/runtime-spec/blob/main/config-linux.md#cgroups-path
 
 # Supported cgroups
 

docs/design/kata-nydus-design.md

@@ -0,0 +1,93 @@
+# Background
+
+[Research](https://www.usenix.org/conference/fast16/technical-sessions/presentation/harter) shows that time to take for pull operation accounts for 76% of container startup time but only 6.4% of that data is read. So if we can get data on demand (lazy load), it will speed up the container start. [`Nydus`](https://github.com/dragonflyoss/image-service) is a project which build image with new format and can get data on demand when container start.
+
+The following benchmarking result shows the performance improvement compared with the OCI image for the container cold startup elapsed time on containerd. As the OCI image size increases, the container startup time of using `nydus` image remains very short. [Click here](https://github.com/dragonflyoss/image-service/blob/master/docs/nydus-design.md) to see `nydus` design.
+
+![`nydus`-performance](arch-images/nydus-performance.png)
+
+## Proposal - Bring `lazyload` ability to Kata Containers
+
+`Nydusd` is a fuse/`virtiofs` daemon which is provided by `nydus` project and it supports `PassthroughFS` and [RAFS](https://github.com/dragonflyoss/image-service/blob/master/docs/nydus-design.md) (Registry Acceleration File System) natively, so in Kata Containers, we can use `nydusd` in place of `virtiofsd` and mount `nydus` image to guest in the meanwhile.
+
+The process of creating/starting Kata Containers with `virtiofsd`,
+
+1. When creating sandbox, the Kata Containers Containerd v2 [shim](https://github.com/kata-containers/kata-containers/blob/main/docs/design/architecture/README.md#runtime) will launch `virtiofsd` before VM starts and share directories with VM.
+2. When creating container, the Kata Containers Containerd v2 shim will mount rootfs to `kataShared`(/run/kata-containers/shared/sandboxes/\<SANDBOX\>/mounts/\<CONTAINER\>/rootfs), so it can be seen at the path `/run/kata-containers/shared/containers/shared/\<CONTAINER\>/rootfs` in the guest and used as container's rootfs.
+
+The process of creating/starting Kata Containers with `nydusd`,
+
+![kata-`nydus`](arch-images/kata-nydus.png)
+
+1. When creating sandbox, the Kata Containers Containerd v2 shim will launch `nydusd` daemon before VM starts.
+After VM starts, `kata-agent` will mount `virtiofs` at the path `/run/kata-containers/shared` and Kata Containers Containerd v2 shim mount `passthroughfs` filesystem to path `/run/kata-containers/shared/containers` when the VM starts.
+
+```bash
+# start nydusd
+$ sandbox_id=my-test-sandbox
+$ sudo /usr/local/bin/nydusd --log-level info --sock /run/vc/vm/${sandbox_id}/vhost-user-fs.sock --apisock /run/vc/vm/${sandbox_id}/api.sock
+```
+
+```bash
+# source: the host sharedir which will pass through to guest
+$ sudo curl -v --unix-socket /run/vc/vm/${sandbox_id}/api.sock \
+    -X POST "http://localhost/api/v1/mount?mountpoint=/containers" -H "accept: */*" \
+    -H "Content-Type: application/json" \
+    -d '{
+            "source":"/path/to/sharedir",
+            "fs_type":"passthrough_fs",
+            "config":""
+    }'
+```
+
+2. When creating normal container, the Kata Containers Containerd v2 shim send request to `nydusd` to mount `rafs` at the path `/run/kata-containers/shared/rafs/<container_id>/lowerdir` in guest.
+
+```bash
+# source: the metafile of nydus image
+# config: the config of this image
+$ sudo curl --unix-socket /run/vc/vm/${sandbox_id}/api.sock \
+    -X POST "http://localhost/api/v1/mount?mountpoint=/rafs/<container_id>/lowerdir" -H "accept: */*" \
+    -H "Content-Type: application/json" \
+    -d '{
+            "source":"/path/to/bootstrap",
+            "fs_type":"rafs",
+            "config":"config":"{\"device\":{\"backend\":{\"type\":\"localfs\",\"config\":{\"dir\":\"blobs\"}},\"cache\":{\"type\":\"blobcache\",\"config\":{\"work_dir\":\"cache\"}}},\"mode\":\"direct\",\"digest_validate\":true}",
+    }'
+```
+
+The Kata Containers Containerd v2 shim will also bind mount `snapshotdir` which `nydus-snapshotter` assigns to `sharedir`。
+So in guest, container rootfs=overlay(`lowerdir=rafs`, `upperdir=snapshotdir/fs`, `workdir=snapshotdir/work`)
+
+> how to transfer the `rafs` info from `nydus-snapshotter` to the Kata Containers Containerd v2 shim?
+
+By default, when creating `OCI` image container, `nydus-snapshotter` will return [`struct` Mount slice](https://github.com/containerd/containerd/blob/main/mount/mount.go#L21) below to containerd and containerd use them to mount rootfs
+
+```
+[
+    {
+        Type: "overlay",
+        Source: "overlay",
+        Options: [lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/<snapshot_A>/mnt,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/<snapshot_B>/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/<snapshot_B>/work],
+    }
+]
+```
+
+Then, we can append `rafs` info into `Options`, but if do this, containerd will mount failed, as containerd can not identify `rafs` info. Here, we can refer to [containerd mount helper](https://github.com/containerd/containerd/blob/main/mount/mount_linux.go#L42) and provide a binary called `nydus-overlayfs`. The `Mount` slice which `nydus-snapshotter` returned becomes
+
+```
+[
+    {
+        Type: "fuse.nydus-overlayfs",
+        Source: "overlay",
+        Options: [lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/<snapshot_A>/mnt,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/<snapshot_B>/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/<snapshot_B>/work,extraoption=base64({source:xxx,config:xxx,snapshotdir:xxx})],
+    }
+]
+```
+
+When containerd find `Type` is `fuse.nydus-overlayfs`,
+
+1. containerd will call `mount.fuse` command;
+2. in `mount.fuse`, it will call `nydus-overlayfs`.
+3. in `nydus-overlayfs`, it will ignore the `extraoption` and do the overlay mount.
+
+Finally, in the Kata Containers Containerd v2 shim, it parse `extraoption` and get the `rafs` info to mount the image in guest.

docs/design/proposals/tracing-proposals.md

@@ -209,5 +209,5 @@ network accessible to the collector.
 - The trace collection proposals are still being considered.
 
 [kata-1x-tracing]: https://github.com/kata-containers/agent/blob/master/TRACING.md
-[trace-forwarder]: /src/trace-forwarder
+[trace-forwarder]: /src/tools/trace-forwarder
 [tracing-doc-pr]: https://github.com/kata-containers/kata-containers/pull/1937

docs/design/vcpu-handling.md

@@ -157,6 +157,32 @@ docker run --cpus 4 -ti debian bash -c "nproc; cat /sys/fs/cgroup/cpu,cpuacct/cp
 400000  # cfs quota
 ```
 
+## Virtual CPU handling without hotplug
+
+In some cases, the hardware and/or software architecture being utilized does not support
+hotplug. For example, Firecracker VMM does not support CPU or memory hotplug. Similarly,
+the current Linux Kernel for aarch64 does not support CPU or memory hotplug. To appropriately
+size the virtual machine for the workload within the container or pod, we provide a `static_sandbox_resource_mgmt`
+flag within the Kata Containers configuration. When this is set, the runtime will:
+ - Size the VM based on the workload requirements as well as the `default_vcpus` option specified in the configuration.
+ - Not resize the virtual machine after it has been launched.
+
+VM size determination varies depending on the type of container being run, and may not always
+be available. If workload sizing information is not available, the virtual machine will be started with the
+`default_vcpus`.
+
+In the case of a pod, the initial sandbox container (pause container) typically doesn't contain any resource
+information in its runtime `spec`. It is possible that the upper layer runtime
+(i.e. containerd or CRI-O) may pass sandbox sizing annotations within the pause container's
+`spec`. If these are provided, we will use this to appropriately size the VM. In particular,
+we'll calculate the number of CPUs required for the workload and augment this by `default_vcpus`
+configuration option, and use this for the virtual machine size.
+
+In the case of a single container (i.e., not a pod), if the container specifies resource requirements,
+the container's `spec` will provide the sizing information directly. If these are set, we will
+calculate the number of CPUs required for the workload and augment this by `default_vcpus`
+configuration option, and use this for the virtual machine size.
+
 
 [1]: https://docs.docker.com/config/containers/resource_constraints/#cpu
 [2]: https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource

docs/design/virtualization.md

@@ -40,8 +40,8 @@ Kata Containers with QEMU has complete compatibility with Kubernetes.
 
 Depending on the host architecture, Kata Containers supports various machine types,
 for example `pc` and `q35` on x86 systems, `virt` on ARM systems and `pseries` on IBM Power systems. The default Kata Containers
-machine type is `pc`. The machine type and its [`Machine accelerators`](#machine-accelerators) can
-be changed by editing the runtime [`configuration`](./architecture.md/#configuration) file.
+machine type is `q35`. The machine type and its [`Machine accelerators`](#machine-accelerators) can
+be changed by editing the runtime [`configuration`](architecture/README.md#configuration) file.
 
 Devices and features used:
 - virtio VSOCK or virtio serial

docs/how-to/README.md

@@ -36,3 +36,5 @@
 - [How to use hotplug memory on arm64 in Kata Containers](how-to-hotplug-memory-arm64.md)
 - [How to setup swap devices in guest kernel](how-to-setup-swap-devices-in-guest-kernel.md)
 - [How to run rootless vmm](how-to-run-rootless-vmm.md)
+- [How to run Docker with Kata Containers](how-to-run-docker-with-kata.md)
+- [How to run Kata Containers with `nydus`](how-to-use-virtio-fs-nydus-with-kata.md)

docs/how-to/containerd-kata.md

@@ -188,7 +188,7 @@ If you use Containerd older than v1.2.4 or a version of Kata older than v1.6.0
 shell script with the following:
 
 ```bash
-#!/bin/bash
+#!/usr/bin/env bash
 KATA_CONF_FILE=/etc/kata-containers/firecracker.toml containerd-shim-kata-v2 $@
 ```
 

docs/how-to/how-to-import-kata-logs-with-fluentd.md

@@ -4,7 +4,7 @@
 
 This document describes how to import Kata Containers logs into [Fluentd](https://www.fluentd.org/),
 typically for importing into an
-Elastic/Fluentd/Kibana([EFK](https://github.com/kubernetes/kubernetes/tree/master/cluster/addons/fluentd-elasticsearch#running-efk-stack-in-production))
+Elastic/Fluentd/Kibana([EFK](https://github.com/kubernetes-sigs/instrumentation-addons/tree/master/fluentd-elasticsearch#running-efk-stack-in-production))
 or Elastic/Logstash/Kibana([ELK](https://www.elastic.co/elastic-stack)) stack.
 
 The majority of this document focusses on CRI-O based (classic) Kata runtime. Much of that information
@@ -257,14 +257,14 @@ go directly to a full Kata specific JSON format logfile test.
 
 Kata runtime has the ability to generate JSON logs directly, rather than its default `logfmt` format. Passing
 the `--log-format=json` argument to the Kata runtime enables this. The easiest way to pass in this extra
-parameter from a [Kata deploy](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy) installation
+parameter from a [Kata deploy](../../tools/packaging/kata-deploy) installation
 is to edit the `/opt/kata/bin/kata-qemu` shell script.
 
 At the same time, we will add the `--log=/var/log/kata-runtime.log` argument to store the Kata logs in their
 own file (rather than into the system journal).
 
 ```bash
-#!/bin/bash
+#!/usr/bin/env bash
 /opt/kata/bin/kata-runtime --config "/opt/kata/share/defaults/kata-containers/configuration-qemu.toml" --log-format=json --log=/var/log/kata-runtime.log $@
 ```
 

docs/how-to/how-to-run-docker-with-kata.md

@@ -0,0 +1,141 @@
+# How to run Docker in Docker with Kata Containers
+
+This document describes the why and how behind running Docker in a Kata Container.
+
+> **Note:** While in other environments this might be described as "Docker in Docker", the new architecture of Kata 2.x means [Docker can no longer be used to create containers using a Kata Containers runtime](https://github.com/kata-containers/kata-containers/issues/722).
+
+## Requirements
+
+- A working Kata Containers installation
+
+## Install and configure Kata Containers
+
+Follow the [Kata Containers installation guide](../install/README.md) to Install Kata Containers on your Kubernetes cluster.
+
+## Background
+
+Docker in Docker ("DinD") is the colloquial name for the ability to run `docker` from inside a container.
+
+You can learn more about about Docker-in-Docker at the following links:
+
+- [The original announcement of DinD](https://www.docker.com/blog/docker-can-now-run-within-docker/)
+- [`docker` image Docker Hub page](https://hub.docker.com/_/docker/) (this page lists the `-dind` releases)
+
+While normally DinD refers to running `docker` from inside a Docker container,
+Kata Containers 2.x allows only [supported runtimes][kata-2.x-supported-runtimes] (such as [`containerd`](../install/container-manager/containerd/containerd-install.md)).
+
+Running `docker` in a Kata Container implies creating Docker containers from inside a container managed by `containerd` (or another supported container manager), as illustrated below:
+
+```
+container manager -> Kata Containers shim     -> Docker Daemon -> Docker container
+(containerd)        (containerd-shim-kata-v2)    (dockerd)        (busybox sh)
+```
+
+[OverlayFS][OverlayFS] is the preferred storage driver for most container runtimes on Linux ([including Docker](https://docs.docker.com/storage/storagedriver/select-storage-driver)).
+
+> **Note:** While in the past Kata Containers did not contain the [`overlay` kernel module (aka OverlayFS)][OverlayFS], the kernel modules have been included since the [Kata Containers v2.0.0 release][v2.0.0].
+
+[OverlayFS]: https://www.kernel.org/doc/html/latest/filesystems/overlayfs.html
+[v2.0.0]: https://github.com/kata-containers/kata-containers/releases/tag/2.0.0
+[kata-2.x-supported-runtimes]: ../install/container-manager/containerd/containerd-install.md
+
+## Why Docker in Kata Containers 2.x requires special measures
+
+Running Docker containers Kata Containers requires care because `VOLUME`s specified in `Dockerfile`s run by Kata Containers are given the `kataShared` mount type by default, which applies to the root directory `/`:
+
+```console
+/ # mount
+kataShared on / type virtiofs (rw,relatime,dax)
+```
+
+`kataShared` mount types are powered by [`virtio-fs`][virtio-fs], a marked improvement over `virtio-9p`, thanks to [PR #1016](https://github.com/kata-containers/runtime/pull/1016). While `virtio-fs` is normally an excellent choice, in the case of DinD workloads `virtio-fs` causes an issue -- [it *cannot* be used as a "upper layer" of `overlayfs` without a custom patch](http://lists.katacontainers.io/pipermail/kata-dev/2020-January/001216.html).
+
+As `/var/lib/docker` is a `VOLUME` specified by DinD (i.e. the `docker` images tagged `*-dind`/`*-dind-rootless`), `docker` fill fail to start (or even worse, silently pick a worse storage driver like `vfs`) when started in a Kata Container. Special measures must be taken when running DinD-powered workloads in Kata Containers.
+
+## Workarounds/Solutions
+
+Thanks to various community contributions (see [issue references below](#references)) the following options, with various trade-offs have been uncovered:
+
+### Use a memory backed volume
+
+For small workloads (small container images, without much generated filesystem load), a memory-backed volume is sufficient. Kubernetes supports a variant of  [the `EmptyDir` volume][k8s-emptydir], which allows for memdisk-backed storage -- the [the `medium: Memory` ][k8s-memory-volume-type]. An example of a `Pod` using such a setup [was contributed](https://github.com/kata-containers/runtime/issues/1429#issuecomment-477385283), and is reproduced below:
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dind
+spec:
+  runtimeClassName: kata
+  containers:
+  - name: dind
+    securityContext:
+      privileged: true
+    image: docker:20.10-dind
+    args: ["--storage-driver=overlay2"]
+    resources:
+      limits:
+        memory: "3G"
+    volumeMounts:
+      - mountPath: /var/run/
+        name: dockersock
+      - mountPath: /var/lib/docker
+        name: docker
+  volumes:
+    - name: dockersock
+      emptyDir: {}
+    - name: docker
+      emptyDir:
+        medium: Memory
+```
+
+Inside the container you can view the mount:
+
+```console
+/ # mount | grep lib\/docker
+tmpfs on /var/lib/docker type tmpfs (rw,relatime)
+```
+
+As is mentioned in the comment encapsulating this code, using volatile memory for container storage backing is a risky and could be possibly wasteful on machines that do not have a lot of RAM.
+
+### Use a loop mounted disk
+
+Using a loop mounted disk that is provisioned shortly before starting of the container workload is another approach that yields good performance.
+
+Contributors provided [an example in issue #1888](https://github.com/kata-containers/runtime/issues/1888#issuecomment-739057384), which is reproduced in part below:
+
+```yaml
+spec:
+  containers:
+    - name: docker
+      image: docker:20.10-dind
+      command: ["sh", "-c"]
+      args:
+      - if [[ $(df -PT /var/lib/docker | awk 'NR==2 {print $2}') == virtiofs ]]; then
+          apk add e2fsprogs &&
+          truncate -s 20G /tmp/disk.img &&
+          mkfs.ext4 /tmp/disk.img &&
+          mount /tmp/disk.img /var/lib/docker; fi &&
+        dockerd-entrypoint.sh;
+      securityContext:
+        privileged: true
+```
+
+Note that loop mounted disks are often sparse, which means they *do not* take up the full amount of space that has been provisioned. This solution seems to produce the best performance and flexibility, at the expense of increased complexity and additional required setup.
+
+### Build a custom kernel
+
+It's possible to [modify the kernel](https://github.com/kata-containers/runtime/issues/1888#issuecomment-616872558) (in addition to applying the earlier mentioned mailing list patch) to support using `virtio-fs` as an upper. Note that if you modify your kernel and use `virtio-fs` you may require [additional changes](https://github.com/kata-containers/runtime/issues/1888#issuecomment-739057384) for decent performance and to address other issues.
+
+> **NOTE:** A future kernel release may rectify the usability and performance issues of using `virtio-fs` as an OverlayFS upper layer.
+
+## References
+
+The solutions proposed in this document are an amalgamation of thoughtful contributions from the Kata Containers community.
+
+Find links to issues & related discussion and the fruits therein below:
+
+- [How to run Docker in Docker with Kata Containers (#2474)](https://github.com/kata-containers/kata-containers/issues/2474)
+- [Does Kata-container support AUFS/OverlayFS? (#2493)](https://github.com/kata-containers/runtime/issues/2493)
+- [Unable to start docker in docker with virtio-fs (#1888)](https://github.com/kata-containers/runtime/issues/1888)
+- [Not using native diff for overlay2 (#1429)](https://github.com/kata-containers/runtime/issues/1429)

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -56,13 +56,14 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.enable_iommu` | `boolean` | enable `iommu` on Q35 (QEMU x86_64) |
 | `io.katacontainers.config.hypervisor.enable_iothreads` | `boolean`| enable IO to be processed in a separate thread. Supported currently for virtio-`scsi` driver |
 | `io.katacontainers.config.hypervisor.enable_mem_prealloc` | `boolean` | the memory space used for `nvdimm` device by the hypervisor |
-| `io.katacontainers.config.hypervisor.enable_swap` | `boolean` | enable swap of VM memory |
 | `io.katacontainers.config.hypervisor.enable_vhost_user_store` | `boolean` | enable vhost-user storage device (QEMU) |
 | `io.katacontainers.config.hypervisor.enable_virtio_mem` | `boolean` | enable virtio-mem (QEMU) |
 | `io.katacontainers.config.hypervisor.entropy_source` (R) | string| the path to a host source of entropy (`/dev/random`, `/dev/urandom` or real hardware RNG device) |
 | `io.katacontainers.config.hypervisor.file_mem_backend` (R) | string | file based memory backend root directory |
 | `io.katacontainers.config.hypervisor.firmware_hash` | string | container firmware SHA-512 hash value |
 | `io.katacontainers.config.hypervisor.firmware` | string | the guest firmware that will run the container VM |
+| `io.katacontainers.config.hypervisor.firmware_volume_hash` | string | container firmware volume SHA-512 hash value |
+| `io.katacontainers.config.hypervisor.firmware_volume` | string | the guest firmware volume that will be passed to the container VM |
 | `io.katacontainers.config.hypervisor.guest_hook_path` | string | the path within the VM that will be used for drop in hooks |
 | `io.katacontainers.config.hypervisor.hotplug_vfio_on_root_bus` | `boolean` | indicate if devices need to be hotplugged on the root bus instead of a bridge|
 | `io.katacontainers.config.hypervisor.hypervisor_hash` | string | container hypervisor binary SHA-512 hash value |

docs/how-to/how-to-use-k8s-with-cri-containerd-and-kata.md

@@ -154,7 +154,7 @@ From Kubernetes v1.12, users can use [`RuntimeClass`](https://kubernetes.io/docs
 
 ```bash
 $ cat > runtime.yaml <<EOF
-apiVersion: node.k8s.io/v1beta1
+apiVersion: node.k8s.io/v1
 kind: RuntimeClass
 metadata:
   name: kata

docs/how-to/how-to-use-virtio-fs-nydus-with-kata.md

@@ -0,0 +1,57 @@
+# Kata Containers with virtio-fs-nydus
+
+## Introduction
+
+Refer to [kata-`nydus`-design](../design/kata-nydus-design.md) for introduction and `nydus` has supported Kata Containers with hypervisor `QEMU` and `CLH` currently.
+
+## How to
+
+You can use Kata Containers with `nydus` as follows,
+
+1. Use [`nydus` latest branch](https://github.com/dragonflyoss/image-service);
+
+2. Deploy `nydus` environment as [`Nydus` Setup for Containerd Environment](https://github.com/dragonflyoss/image-service/blob/master/docs/containerd-env-setup.md);
+
+3. Start `nydus-snapshotter` with `enable_nydus_overlayfs` enabled;
+
+4. Use [kata-containers](https://github.com/kata-containers/kata-containers) `latest` branch to compile and build `kata-containers.img`;
+
+5. Update `configuration-qemu.toml` or `configuration-clh.toml`to include:
+
+```toml
+shared_fs = "virtio-fs-nydus"
+virtio_fs_daemon = "<nydusd binary path>"
+virtio_fs_extra_args = []
+```
+
+6. run `crictl run -r kata nydus-container.yaml nydus-sandbox.yaml`;
+
+The `nydus-sandbox.yaml` looks like below:
+
+```yaml
+metadata:
+  attempt: 1
+  name: nydus-sandbox
+  namespace: default
+log_directory: /tmp
+linux:
+  security_context:
+    namespace_options:
+      network: 2
+annotations:
+  "io.containerd.osfeature": "nydus.remoteimage.v1"
+```
+
+The `nydus-container.yaml` looks like below:
+
+```yaml
+metadata:
+  name: nydus-container
+image:
+  image: localhost:5000/ubuntu-nydus:latest
+command:
+  - /bin/sleep
+args:
+  - 600
+log_path: container.1.log
+```

docs/how-to/how-to-use-virtio-fs-with-kata.md

@@ -6,4 +6,4 @@ Container deployments utilize explicit or implicit file sharing between host fil
 
 As of the 2.0 release of Kata Containers, [virtio-fs](https://virtio-fs.gitlab.io/) is the default filesystem sharing mechanism.
 
-virtio-fs support works out of the box for `cloud-hypervisor` and `qemu`, when Kata Containers is deployed using `kata-deploy`. Learn more about `kata-deploy` and how to use `kata-deploy` in Kubernetes [here](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy#kubernetes-quick-start).
+virtio-fs support works out of the box for `cloud-hypervisor` and `qemu`, when Kata Containers is deployed using `kata-deploy`. Learn more about `kata-deploy` and how to use `kata-deploy` in Kubernetes [here](../../tools/packaging/kata-deploy/README.md#kubernetes-quick-start).

docs/how-to/offline_cpu.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) 2019 Intel Corporation
 #
 # SPDX-License-Identifier: Apache-2.0

docs/how-to/run-kata-with-k8s.md

@@ -104,26 +104,69 @@ $ sudo kubeadm init --ignore-preflight-errors=all --cri-socket /run/containerd/c
 $ export KUBECONFIG=/etc/kubernetes/admin.conf
 ```
 
-You can force Kubelet to use Kata Containers by adding some `untrusted`
-annotation to your pod configuration. In our case, this ensures Kata
-Containers is the selected runtime to run the described workload.
+### Allow pods to run in the master node
 
-`nginx-untrusted.yaml`
-```yaml
-apiVersion: v1
-kind: Pod
+By default, the cluster will not schedule pods in the master node. To enable master node scheduling:
+```bash
+$ sudo -E kubectl taint nodes --all node-role.kubernetes.io/master-
+```
+
+### Create runtime class for Kata Containers
+
+Users can use [`RuntimeClass`](https://kubernetes.io/docs/concepts/containers/runtime-class/#runtime-class) to specify a different runtime for Pods.
+
+```bash
+$ cat > runtime.yaml <<EOF
+apiVersion: node.k8s.io/v1
+kind: RuntimeClass
 metadata:
-  name: nginx-untrusted
-  annotations:
-    io.kubernetes.cri.untrusted-workload: "true"
-spec:
-  containers:
+  name: kata
+handler: kata
+EOF
+
+$ sudo -E kubectl apply -f runtime.yaml
+```
+
+### Run pod in Kata Containers
+
+If a pod has the `runtimeClassName` set to `kata`, the CRI plugin runs the pod with the
+[Kata Containers runtime](../../src/runtime/README.md).
+
+- Create an pod configuration that using Kata Containers runtime
+
+  ```bash
+  $ cat << EOF | tee nginx-kata.yaml
+  apiVersion: v1
+  kind: Pod
+  metadata:
+    name: nginx-kata
+  spec:
+    runtimeClassName: kata
+    containers:
     - name: nginx
       image: nginx
-```
 
-Next, you run your pod:
-```
-$ sudo -E kubectl apply -f nginx-untrusted.yaml
-```
+  EOF
+  ```
 
+- Create the pod
+  ```bash
+  $ sudo -E kubectl apply -f nginx-kata.yaml
+  ```
+
+- Check pod is running
+
+  ```bash
+  $ sudo -E kubectl get pods
+  ```
+
+- Check hypervisor is running
+  ```bash
+  $ ps aux | grep qemu
+  ```
+
+### Delete created pod
+
+```bash
+$ sudo -E kubectl delete -f nginx-kata.yaml
+```

docs/install/minikube-installation-guide.md

@@ -6,7 +6,7 @@
 cluster locally. It creates a single node Kubernetes stack in a local VM.
 
 [Kata Containers](https://github.com/kata-containers) can be installed into a Minikube cluster using
-[`kata-deploy`](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy).
+[`kata-deploy`](../../tools/packaging/kata-deploy).
 
 This document details the pre-requisites, installation steps, and how to check
 the installation has been successful.
@@ -123,7 +123,7 @@ $ kubectl apply -f kata-deploy/base/kata-deploy.yaml
 This installs the Kata Containers components into `/opt/kata` inside the Minikube node. It can take
 a few minutes for the operation to complete. You can check the installation has worked by checking
 the status of the `kata-deploy` pod, which will be executing
-[this script](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy/scripts/kata-deploy.sh),
+[this script](../../tools/packaging/kata-deploy/scripts/kata-deploy.sh),
 and will be executing a `sleep infinity` once it has successfully completed its work.
 You can accomplish this by running the following:
 

docs/install/snap-installation-guide.md

@@ -39,8 +39,8 @@ can be used as runtime.
 
 Read the following documents to know how to run Kata Containers 2.x with `containerd`.
 
-* [How to use Kata Containers and Containerd](https://github.com/kata-containers/kata-containers/blob/main/docs/how-to/containerd-kata.md)
-* [Install Kata Containers with containerd](https://github.com/kata-containers/kata-containers/blob/main/docs/install/container-manager/containerd/containerd-install.md)
+* [How to use Kata Containers and Containerd](../how-to/containerd-kata.md)
+* [Install Kata Containers with containerd](./container-manager/containerd/containerd-install.md)
 
 
 ## Remove Kata Containers snap package

docs/presentations/README.md

@@ -0,0 +1,3 @@
+# Kata Containers presentations
+
+* [Unit testing](unit-testing)

docs/presentations/unit-testing/README.md

@@ -0,0 +1,14 @@
+# Kata Containers unit testing presentation
+
+## Markdown version
+
+See [the Kata Containers unit testing presentation](kata-containers-unit-testing.md).
+
+### To view as an HTML presentation
+
+```bash
+$ infile="kata-containers-unit-testing.md"
+$ outfile="/tmp/kata-containers-unit-testing.html"
+$ pandoc -s --metadata title="Kata Containers unit testing" -f markdown -t revealjs --highlight-style="zenburn" -i -o "$outfile" "$infile"
+$ xdg-open "file://$outfile"
+```

docs/presentations/unit-testing/kata-containers-unit-testing.md

@@ -0,0 +1,335 @@
+## Why write unit tests?
+
+- Catch regressions
+
+- Improve the code being tested
+
+  Structure, quality, security, performance, "shakes out" implicit
+  assumptions, _etc_
+
+- Extremely instructive
+
+  Once you've fully tested a single function, you'll understand that
+  code very well indeed.
+
+## Why write unit tests? (continued)
+
+- Fun!
+
+  Yes, really! Don't believe me? Try it! ;)
+
+## Run all Kata Containers agent unit tests
+
+As an example, to run all agent unit tests:
+
+```bash
+$ cd $GOPATH/src/github.com/kata-containers/kata-containers
+$ cd src/agent
+$ make test
+```
+
+## List all unit tests
+
+- Identify the full name of all the tests _in the current package_:
+
+  ```bash
+  $ cargo test -- --list
+  ```
+
+- Identify the full name of all tests in the `foo` "local crate"
+  (sub-directory containing another `Cargo.toml` file):
+
+  ```bash
+  $ cargo test -p "foo" -- --list
+  ```
+
+## Run a single unit test
+
+- Run a test in the current package in verbose mode:
+
+  ```bash
+  # Example 
+  $ test="config::tests::test_get_log_level"
+
+  $ cargo test "$test" -vv -- --exact --nocapture
+  ```
+
+## Test coverage setup
+
+```bash
+$ cargo install cargo-tarpaulin
+```
+
+## Show test coverage
+
+```bash
+$ cd $GOPATH/src/github.com/kata-containers/kata-containers/src/agent
+$ cargo -v tarpaulin --all-features --run-types AllTargets --count --force-clean -o Html
+$ xdg-open "file://$PWD/tarpaulin-report.html"
+```
+
+## Testability (part 1)
+
+- To be testable, a function should:
+  - Not be "too long" (say >100 lines).
+  - Not be "too complex" (say >3 levels of indentation).
+  - Should return a `Result` or an `Option` so error paths
+    can be tested.
+
+- If functions don't conform, they need to be reworked (refactored)
+  before writing tests.
+
+## Testability (part 2)
+
+- Some functions can't be fully tested.
+- However, you _can_ test the initial code that checks
+  the parameter values (test error paths only).
+
+## Writing new tests: General advice (part 1)
+
+- KISS: Keep It Simple Stupid
+
+  You don't get extra points for cryptic code.
+
+- DRY: Don't Repeat Yourself
+
+  Make use of existing facilities (don't "re-invert the wheel").
+
+- Read the [unit test advice document](https://github.com/kata-containers/kata-containers/blob/main/docs/Unit-Test-Advice.md)
+
+## Writing new tests: General advice (part 2)
+
+- Attack the function in all possible ways
+
+- Use the _table driven_ approach:
+  - Simple
+  - Compact
+  - Easy to debug
+  - Makes boundary analysis easy
+  - Encourages functions to be testable
+
+## Writing new tests: Specific advice (part 1)
+
+- Create a new "`tests`" module if necessary.
+- Give each test function a "`test_`" prefix.
+- Add the "`#[test]`" annotation on each test function.
+
+## Writing new tests: Specific advice (part 2)
+
+- If you need to `use` (import) packages for the tests,
+  _only do it in the `tests` module_:
+  ```rust
+  use some_test_pkg::{foo, bar}; // <-- Not here
+
+  #[cfg(test)]
+  mod tests {
+    use super::*;
+    use some_test_pkg:{foo, bar}; // <-- Put it here
+  }
+  ```
+
+## Writing new tests: Specific advice (part 3)
+
+- You can add test-specific dependencies in `Cargo.toml`:
+  ```toml
+  [dev-dependencies]
+  serial_test = "0.5.1"
+  ```
+
+## Writing new tests: Specific advice (part 4)
+
+- Don't add in lots of error handling code: let the test panic!
+  ```rust
+  // This will panic if the unwrap fails.
+  // - NOT acceptable generally for production code.
+  // - PERFECTLY acceptable for test code since:
+  //   - Keeps the test code simple.
+  //   - Rust will detect the panic and fail the test.
+  let result = func().unwrap();
+  ```
+
+## Debugging tests (part 1)
+
+- Comment out all tests in your `TestData` array apart from the failing test.
+
+- Add temporary `println!("FIXME: ...")` statements in the code.
+
+- Set `RUST_BACKTRACE=full` before running `cargo test`.
+
+## Debugging tests (part 2)
+
+- Use a debugger (not normally necessary though):
+  ```bash
+  # Disable optimisation
+  $ RUSTFLAGS="-C opt-level=0" cargo test --no-run
+
+  # Find the test binary
+  $ test_binary=$(find target/debug/deps | grep "kata_agent-[a-z0-9][a-z0-9]*$" | tail -1)
+
+  $ rust-gdb "$test_binary"
+  ```
+
+## Useful tips
+
+- Always start a test with a "clean environment":
+
+  Create new set of objects / files / directories / _etc_
+  for each test.
+
+- Mounts
+  - Linux allows mounts on top of existing mounts.
+  - Bind mounts and read-only mounts can be useful.
+
+## Gotchas (part 1)
+
+If a test runs successfully _most of the time_:
+
+- Review the test logic.
+
+- Add a `#[serial]` annotation on the test function
+  Requires the `serial_test` package in the `[dev-dependencies]`
+  section of `Cargo.toml`.
+
+  If this makes it work the test is probably sharing resources with
+  another task (thread).
+
+## Gotchas (part 2)
+
+If a test works locally but fails in the CI, consider the following
+attributes of each environment (local and CI):
+
+- The version of rust being used.
+- The hardware architecture.
+- Number (and spec) of the CPUs.
+
+## Gotchas (part 3)
+
+If in doubt, look at the
+["test artifacts" attached to the failing CI test](http://jenkins.katacontainers.io).
+
+## Before raising a PR
+
+- Remember to check that the test runs locally:
+  - As a non-privileged user.
+  - As the `root` user (carefully!)
+
+- Run the [static checker](https://github.com/kata-containers/tests/blob/main/.ci/static-checks.sh)
+  on your changes.
+
+  Checks formatting and many other things.
+
+## If in doubt
+
+- Ask for help! ;)
+
+## Quiz 1
+
+What's wrong with this function?
+
+```rust
+fn foo(config: &Config, path_prefix: String, container_id: String, pid: String) -> Result<()> {
+    let mut full_path = format!("{}/{}", path_prefix, container_id);
+
+    let _ = remove_recursively(&mut full_path);
+
+    write_number_to_file(pid, full_path);
+
+    Ok(())
+}
+```
+
+## Quiz 1: Answers (part 1)
+
+- No check that `path_prefix`, `container_id` and `pid` are not `""`.
+- No check that `path_prefix` is absolute.
+- No check that `container_id` does not contain slashes / contains only valid characters.
+- Result of `remove_recursively()` discarded.
+- `remove_recursively()` _may_ modify `full_path` without `foo()` knowing!
+
+## Quiz 1: Answers (part 2)
+
+- Why is `pid` not a numeric?
+- No check to ensure the PID is positive.
+- No check to recreate any directories in the original `path_prefix`.
+- `write_number_to_file()` could fail so why doesn't it return a value?
+- The `config` parameter is unused.
+
+## Quiz 1: What if...
+
+Imagine if the caller managed to do this:
+
+```rust
+foo(config, "", "sbin/init", r#"#!/bin/sh\n/sbin/reboot"#);
+```
+
+## Quiz 2
+
+What makes this function difficult to test?
+
+```rust
+fn get_user_id(username: String) -> i32 {
+    let line = grep_file(username, "/etc/passwd").unwrap();
+    let fields = line.split(':');
+
+    let uid = fields.nth(2).ok_or("failed").unwrap();
+
+    uid.parse::<i32>()
+}
+```
+
+## Quiz 2: Answers (part 1)
+
+- Unhelpful error message ("failed").
+
+- Panics on error! Return a `Result` instead!
+
+- UID's cannot be negative so function should return an unsigned
+  value.
+
+## Quiz 2: Answers (part 2)
+
+- Hard-coded filename.
+
+  This would be better:
+
+  ```rust
+  const PASSWD_DB: &str = "/etc/passwd";
+
+  // Test code can now pass valid and invalid files!
+  fn get_user_id(filename: String, username: String) -> i32 {
+    // ...
+  }
+
+  let id = get_user_id(PASSWD_DB, username);
+  ```
+
+## Quiz 3
+
+What's wrong with this test code?
+
+```rust
+let mut obj = Object::new();
+
+// Sanity check
+assert_eq!(obj.num, 0);
+assert_eq!(obj.wibble, false);
+
+// Test 1
+obj->foo_method(7);
+assert_eq!(obj.num, 7);
+
+// Test 2
+obj->bar_method(true);
+assert_eq!(obj.wibble, true);
+```
+
+## Quiz 3: Answers
+
+- The test code is "fragile":
+  - The 2nd test re-uses the object created in the first test.
+
+## Finally
+
+- [We need a GH action to run the unit tests](https://github.com/kata-containers/kata-containers/issues/2934)
+
+  Needs to fail PRs that decrease test coverage<br/> by "x%".

docs/tracing.md

@@ -203,12 +203,11 @@ is highly recommended. For working with the agent, you may also wish to
 [enable a debug console][setup-debug-console]
 to allow you to access the VM environment.
 
-[agent-ctl]: https://github.com/kata-containers/kata-containers/blob/main/tools/agent-ctl
-[enable-full-debug]: https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md#enable-full-debug
+[enable-full-debug]: ./Developer-Guide.md#enable-full-debug
 [jaeger-all-in-one]: https://www.jaegertracing.io/docs/getting-started/
 [jaeger-tracing]: https://www.jaegertracing.io
 [opentelemetry]: https://opentelemetry.io
-[osbuilder]: https://github.com/kata-containers/kata-containers/blob/main/tools/osbuilder
-[setup-debug-console]: https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md#set-up-a-debug-console
-[trace-forwarder]: https://github.com/kata-containers/kata-containers/blob/main/src/trace-forwarder
+[osbuilder]: ../tools/osbuilder
+[setup-debug-console]: ./Developer-Guide.md#set-up-a-debug-console
+[trace-forwarder]: /src/tools/trace-forwarder
 [vsock]: https://wiki.qemu.org/Features/VirtioVsock

docs/use-cases/using-Intel-QAT-and-kata.md

@@ -231,11 +231,11 @@ $ cp ${GOPATH}/${LINUX_VER}/vmlinux ${KATA_KERNEL_LOCATION}/${KATA_KERNEL_NAME}
 These instructions build upon the OS builder instructions located in the 
 [Developer Guide](../Developer-Guide.md). At this point it is recommended that
 [Docker](https://docs.docker.com/engine/install/ubuntu/) is installed first, and
-then [Kata-deploy](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy)
+then [Kata-deploy](../../tools/packaging/kata-deploy)
 is use to install Kata. This will make sure that the correct `agent` version 
 is installed into the rootfs in the steps below.
 
-The following instructions use Debian as the root filesystem with systemd as 
+The following instructions use Ubuntu as the root filesystem with systemd as 
 the init and will add in the `kmod` binary, which is not a standard binary in 
 a Kata rootfs image. The `kmod` binary is necessary to load the Intel® QAT 
 kernel modules when the virtual machine rootfs boots. 
@@ -257,7 +257,7 @@ $ cd $GOPATH
 $ export AGENT_VERSION=$(kata-runtime version | head -n 1 | grep -o "[0-9.]\+")
 $ cd ${OSBUILDER}/rootfs-builder
 $ sudo rm -rf ${ROOTFS_DIR}
-$ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true SECCOMP=no ./rootfs.sh debian'
+$ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true SECCOMP=no ./rootfs.sh ubuntu'
 ```
 
 ### Compile Intel® QAT drivers for Kata Containers kernel and add to Kata Containers rootfs
@@ -355,10 +355,10 @@ this small script so that it redirects to be able to use either QEMU or
 Cloud Hypervisor with Kata.
 
 ```bash
-$ echo '#!/bin/bash' | sudo tee /usr/local/bin/containerd-shim-kata-qemu-v2
+$ echo '#!/usr/bin/env bash' | sudo tee /usr/local/bin/containerd-shim-kata-qemu-v2
 $ echo 'KATA_CONF_FILE=/opt/kata/share/defaults/kata-containers/configuration-qemu.toml /opt/kata/bin/containerd-shim-kata-v2 $@' | sudo tee -a /usr/local/bin/containerd-shim-kata-qemu-v2
 $ sudo chmod +x /usr/local/bin/containerd-shim-kata-qemu-v2
-$ echo '#!/bin/bash' | sudo tee /usr/local/bin/containerd-shim-kata-clh-v2
+$ echo '#!/usr/bin/env bash' | sudo tee /usr/local/bin/containerd-shim-kata-clh-v2
 $ echo 'KATA_CONF_FILE=/opt/kata/share/defaults/kata-containers/configuration-clh.toml /opt/kata/bin/containerd-shim-kata-v2 $@' | sudo tee -a /usr/local/bin/containerd-shim-kata-clh-v2
 $ sudo chmod +x /usr/local/bin/containerd-shim-kata-clh-v2
 ```
@@ -419,11 +419,11 @@ You might need to disable Docker before initializing Kubernetes. Be aware
 that the OpenSSL container image built above will need to be exported from
 Docker and imported into containerd.
 
-If Kata is installed through [`kata-deploy`](https://github.com/kata-containers/kata-containers/blob/stable-2.0/tools/packaging/kata-deploy/README.md)
+If Kata is installed through [`kata-deploy`](../../tools/packaging/kata-deploy/README.md)
 there will be multiple `configuration.toml` files associated with different 
 hypervisors. Rather than add in the custom Kata kernel, Kata rootfs, and 
 kernel modules to each `configuration.toml` as the default, instead use
-[annotations](https://github.com/kata-containers/kata-containers/blob/stable-2.0/docs/how-to/how-to-load-kernel-modules-with-kata.md)
+[annotations](../how-to/how-to-load-kernel-modules-with-kata.md)
 in the Kubernetes YAML file to tell Kata which kernel and rootfs to use. The 
 easy way to do this is to use `kata-deploy` which will install the Kata binaries
 to `/opt` and properly configure the `/etc/containerd/config.toml` with annotation 

docs/use-cases/using-Intel-SGX-and-kata.md

@@ -17,24 +17,11 @@ CONFIG_X86_SGX_KVM=y
 ```
 
 * Kubernetes cluster configured with:
-   * [`kata-deploy`](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy) based Kata Containers installation
+   * [`kata-deploy`](../../tools/packaging/kata-deploy) based Kata Containers installation
    * [Intel SGX Kubernetes device plugin](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/main/cmd/sgx_plugin#deploying-with-pre-built-images)
 
 > Note: Kata Containers supports creating VM sandboxes with Intel® SGX enabled
-> using [cloud-hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor/) VMM only. QEMU support is waiting to get the
-> Intel SGX enabled QEMU upstream release.
-
-## Installation
-
-### Kata Containers Guest Kernel
-
-Follow the instructions to [setup](../../tools/packaging/kernel/README.md#setup-kernel-source-code) and [build](../../tools/packaging/kernel/README.md#build-the-kernel) the experimental guest kernel. Then, install as:
-
-```sh
-$ sudo cp kata-linux-experimental-*/vmlinux /opt/kata/share/kata-containers/vmlinux.sgx
-$ sudo sed -i 's|vmlinux.container|vmlinux.sgx|g' \
-  /opt/kata/share/defaults/kata-containers/configuration-clh.toml
-```
+> using [cloud-hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor/) and [QEMU](https://www.qemu.org/) VMMs only.
 
 ### Kata Containers Configuration
 
@@ -48,6 +35,8 @@ to the `sandbox` are: `["io.katacontainers.*", "sgx.intel.com/epc"]`.
 
 With the following sample job deployed using `kubectl apply -f`:
 
+> Note: Change the `runtimeClassName` option accordingly, only `kata-clh` and `kata-qemu` support Intel® SGX.
+
 ```yaml
 apiVersion: batch/v1
 kind: Job

docs/use-cases/using-SPDK-vhostuser-and-kata.md

@@ -104,7 +104,7 @@ devices:
 
 - `vhost-user-blk`
 - `vhost-user-scsi`
-- `vhost-user-nvme`
+- `vhost-user-nvme` (deprecated from SPDK 21.07 release)
 
 For more information, visit [SPDK](https://spdk.io) and [SPDK vhost-user target](https://spdk.io/doc/vhost.html).
 

docs/use-cases/zun_kata.md

@@ -1,121 +0,0 @@
-# OpenStack Zun DevStack working with Kata Containers
-
-## Introduction
-
-This guide describes how to get Kata Containers to work with OpenStack Zun
-using DevStack on Ubuntu 16.04. Running DevStack with this guide will setup
-Docker and Clear Containers 2.0, which you replace with Kata Containers.
-Currently, the instructions are based on the following links:
-
-- https://docs.openstack.org/zun/latest/contributor/quickstart.html
-
-- https://docs.openstack.org/zun/latest/admin/clear-containers.html
-
-## Install Git to use with DevStack
-
-```sh
-$ sudo apt install git
-```
-
-## Setup OpenStack DevStack
-The following commands will sync DevStack from GitHub, create your
-`local.conf` file, assign your host IP to this file, enable Clear
-Containers, start DevStack, and set the environment variables to use
-`zun` on the command line.
-
-```sh
-$ sudo mkdir -p /opt/stack
-$ sudo chown $USER /opt/stack
-$ git clone https://github.com/openstack-dev/devstack /opt/stack/devstack
-$ HOST_IP="$(ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1 -d'/')"
-$ git clone https://github.com/openstack/zun /opt/stack/zun
-$ cat /opt/stack/zun/devstack/local.conf.sample \
-$     | sed "s/HOST_IP=.*/HOST_IP=$HOST_IP/" \
-$     > /opt/stack/devstack/local.conf
-$ sed -i "s/KURYR_CAPABILITY_SCOPE=.*/KURYR_CAPABILITY_SCOPE=local/" /opt/stack/devstack/local.conf
-$ echo "ENABLE_CLEAR_CONTAINER=true" >> /opt/stack/devstack/local.conf
-$ echo "enable_plugin zun-ui https://git.openstack.org/openstack/zun-ui" >> /opt/stack/devstack/local.conf
-$ /opt/stack/devstack/stack.sh
-$ source /opt/stack/devstack/openrc admin admin
-```
-
-The previous commands start OpenStack DevStack with Zun support. You can test
-it using `runc` as shown by the following commands to make sure everything
-installed correctly and is working.
-
-```sh
-$ zun run --name test cirros ping -c 4 8.8.8.8
-$ zun list
-$ zun logs test
-$ zun delete test
-```
-
-## Install Kata Containers
-
-Follow [these instructions](../install/README.md)
-to install the Kata Containers components.
-
-## Update Docker with new Kata Containers runtime
-
-The following commands replace the Clear Containers 2.x runtime setup with
-DevStack, with Kata Containers:
-
-```sh
-$ sudo sed -i 's/"cor"/"kata-runtime"/' /etc/docker/daemon.json
-$ sudo sed -i 's/"\/usr\/bin\/cc-oci-runtime"/"\/usr\/bin\/kata-runtime"/' /etc/docker/daemon.json
-$ sudo systemctl daemon-reload
-$ sudo systemctl restart docker
-```
-
-## Test that everything works in both Docker and OpenStack Zun
-
-```sh
-$ sudo docker run -ti --runtime kata-runtime busybox sh
-$ zun run --name kata --runtime kata-runtime cirros ping -c 4 8.8.8.8
-$ zun list
-$ zun logs kata
-$ zun delete kata
-```
-
-## Stop DevStack and clean up system (Optional)
-
-```sh
-$ /opt/stack/devstack/unstack.sh
-$ /opt/stack/devstack/clean.sh
-```
-
-## Restart DevStack and reset CC 2.x runtime to `kata-runtime`
-
-Run the following commands if you already setup Kata Containers and want to
-restart DevStack:
-
-```sh
-$ /opt/stack/devstack/unstack.sh
-$ /opt/stack/devstack/clean.sh
-$ /opt/stack/devstack/stack.sh
-$ source /opt/stack/devstack/openrc admin admin
-$ sudo sed -i 's/"cor"/"kata-runtime"/' /etc/docker/daemon.json
-$ sudo sed -i 's/"\/usr\/bin\/cc-oci-runtime"/"\/usr\/bin\/kata-runtime"/' /etc/docker/daemon.json
-$ sudo systemctl daemon-reload
-$ sudo systemctl restart docker
-```
-
-![Kata Zun image 1](./images/kata-zun1.png)
-
-Figure 1: Create a BusyBox container image
-
-![Kata Zun image 2](./images/kata-zun2.png)
-
-Figure 2: Select `kata-runtime` to use
-
-![Kata Zun image 3](./images/kata-zun3.png)
-
-Figure 3: Two BusyBox containers successfully launched
-
-![Kata Zun image 4](./images/kata-zun4.png)
-
-Figure 4: Test connectivity between Kata Containers
-
-![Kata Zun image 5](./images/kata-zun5.png)
-
-Figure 5: CLI for Zun

snap/README.md

@@ -76,7 +76,7 @@ then a new configuration file can be [created](#configure-kata-containers)
 and [configured][7].
 
 [1]: https://docs.snapcraft.io/snaps/intro
-[2]: ../docs/design/architecture.md#root-filesystem-image
+[2]: ../docs/design/architecture/README.md#root-filesystem-image
 [3]: https://docs.snapcraft.io/reference/confinement#classic
 [4]: https://github.com/kata-containers/runtime#configuration
 [5]: https://docs.docker.com/engine/reference/commandline/dockerd

snap/snapcraft.yaml

@@ -118,18 +118,19 @@ parts:
       export AGENT_INIT=yes
       export USE_DOCKER=1
       export DEBUG=1
-      case "$(uname -m)" in
-        aarch64)
-          sudo -E PATH=$PATH make initrd DISTRO=alpine
-        ;;
-        ppc64le|s390x)
-          # Cannot use alpine on ppc64le/s390x because it would require a musl agent
-          sudo -E PATH=$PATH make initrd DISTRO=ubuntu
-        ;;
+      arch="$(uname -m)"
+      initrd_distro=$(${yq} r -X ${kata_dir}/versions.yaml assets.initrd.architecture.${arch}.name)
+      image_distro=$(${yq} r -X ${kata_dir}/versions.yaml assets.image.architecture.${arch}.name)
+      case "$arch" in
         x86_64)
           # In some build systems it's impossible to build a rootfs image, try with the initrd image
-          sudo -E PATH=$PATH make image DISTRO=clearlinux || sudo -E PATH=$PATH make initrd DISTRO=alpine
+          sudo -E PATH=$PATH make image DISTRO=${image_distro} || sudo -E PATH=$PATH make initrd DISTRO=${initrd_distro}
         ;;
+
+        aarch64|ppc64le|s390x)
+          sudo -E PATH=$PATH make initrd DISTRO=${initrd_distro}
+        ;;
+
         *) echo "unsupported architecture: $(uname -m)"; exit 1;;
       esac
 
@@ -203,14 +204,7 @@ parts:
       kernel_dir_prefix="kata-linux-"
 
       # Setup and build kernel
-      if [ "$(uname -m)" = "x86_64" ]; then
-        kernel_version="$(${yq} r $versions_file assets.kernel-experimental.tag)"
-        kernel_version=${kernel_version#v}
-        kernel_dir_prefix="kata-linux-experimental-"
-        ./build-kernel.sh -e -v ${kernel_version} -d setup
-      else
-        ./build-kernel.sh -v ${kernel_version} -d setup
-      fi
+      ./build-kernel.sh -v ${kernel_version} -d setup
       cd ${kernel_dir_prefix}*
       make -j $(($(nproc)-1)) EXTRAVERSION=".container"
 
@@ -261,34 +255,21 @@ parts:
       kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
 
       versions_file="${kata_dir}/versions.yaml"
-      # arch-specific definition
-      case "$(uname -m)" in
-        "aarch64")
-          branch="$(${yq} r ${versions_file} assets.hypervisor.qemu.architecture.aarch64.version)"
-          url="$(${yq} r ${versions_file} assets.hypervisor.qemu.url)"
-          commit="$(${yq} r ${versions_file} assets.hypervisor.qemu.architecture.aarch64.commit)"
-          patches_dir="${kata_dir}/tools/packaging/qemu/patches/$(echo ${branch} | sed -e 's/.[[:digit:]]*$//' -e 's/^v//').x"
-          patches_version_dir="${kata_dir}/tools/packaging/qemu/patches/tag_patches/${branch}"
-        ;;
-
-        *)
-          branch="$(${yq} r ${versions_file} assets.hypervisor.qemu.version)"
-          url="$(${yq} r ${versions_file} assets.hypervisor.qemu.url)"
-          commit=""
-          patches_dir="${kata_dir}/tools/packaging/qemu/patches/$(echo ${branch} | sed -e 's/.[[:digit:]]*$//' -e 's/^v//').x"
-          patches_version_dir="${kata_dir}/tools/packaging/qemu/patches/tag_patches/${branch}"
-        ;;
-      esac
+      branch="$(${yq} r ${versions_file} assets.hypervisor.qemu.version)"
+      url="$(${yq} r ${versions_file} assets.hypervisor.qemu.url)"
+      commit=""
+      patches_dir="${kata_dir}/tools/packaging/qemu/patches/$(echo ${branch} | sed -e 's/.[[:digit:]]*$//' -e 's/^v//').x"
+      patches_version_dir="${kata_dir}/tools/packaging/qemu/patches/tag_patches/${branch}"
 
       # download source
       qemu_dir=${SNAPCRAFT_STAGE}/qemu
       rm -rf "${qemu_dir}"
-      git clone --branch ${branch} --single-branch ${url} "${qemu_dir}"
+      git clone --depth 1 --branch ${branch} --single-branch ${url} "${qemu_dir}"
       cd ${qemu_dir}
       [ -z "${commit}" ] || git checkout ${commit}
 
-      [ -n "$(ls -A ui/keycodemapdb)" ] || git clone https://github.com/qemu/keycodemapdb ui/keycodemapdb/
-      [ -n "$(ls -A capstone)" ] || git clone https://github.com/qemu/capstone capstone
+      [ -n "$(ls -A ui/keycodemapdb)" ] || git clone --depth 1 https://github.com/qemu/keycodemapdb ui/keycodemapdb/
+      [ -n "$(ls -A capstone)" ] || git clone --depth 1 https://github.com/qemu/capstone capstone
 
       # Apply branch patches
       [ -d "${patches_version_dir}" ] || mkdir "${patches_version_dir}"
@@ -339,17 +320,23 @@ parts:
     plugin: nil
     after: [godeps]
     override-build: |
-      export GOPATH=${SNAPCRAFT_STAGE}/gopath
-      yq=${SNAPCRAFT_STAGE}/yq
-      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
-      versions_file="${kata_dir}/versions.yaml"
-      version="$(${yq} r ${versions_file} assets.hypervisor.cloud_hypervisor.version)"
-      url="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/${version}"
-      curl -L ${url}/cloud-hypervisor-static -o cloud-hypervisor
-      curl -LO ${url}/clh-remote
+      arch=$(uname -m)
+      if [ "{$arch}" == "aarch64" ] || [ "${arch}" == "x64_64" ]; then 
+          sudo apt-get -y update
+          sudo apt-get -y install ca-certificates curl gnupg lsb-release
+          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get -y update
+          sudo apt-get -y install docker-ce docker-ce-cli containerd.io
+          sudo systemctl start docker.socket
 
-      install -D cloud-hypervisor ${SNAPCRAFT_PART_INSTALL}/usr/bin/cloud-hypervisor
-      install -D clh-remote ${SNAPCRAFT_PART_INSTALL}/usr/bin/clh-remote
+          export GOPATH=${SNAPCRAFT_STAGE}/gopath
+          kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
+          cd ${kata_dir}
+          sudo -E NO_TTY=true make cloud-hypervisor-tarball
+          tar xvJpf build/kata-static-cloud-hypervisor.tar.xz -C /tmp/
+          install -D /tmp/opt/kata/bin/cloud-hypervisor ${SNAPCRAFT_PART_INSTALL}/usr/bin/cloud-hypervisor
+      fi
 
 apps:
   runtime:

src/agent/Cargo.toml

@@ -5,29 +5,30 @@ authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
 edition = "2018"
 
 [dependencies]
-oci = { path = "oci" }
+oci = { path = "../libs/oci" }
 rustjail = { path = "rustjail" }
-protocols = { path = "protocols" }
+protocols = { path = "../libs/protocols" }
 lazy_static = "1.3.0"
 ttrpc = { version = "0.5.0", features = ["async", "protobuf-codec"], default-features = false }
 protobuf = "=2.14.0"
 libc = "0.2.58"
-nix = "0.21.0"
+nix = "0.23.0"
 capctl = "0.2.0"
 serde_json = "1.0.39"
 scan_fmt = "0.2.3"
 scopeguard = "1.0.0"
 thiserror = "1.0.26"
-regex = "1"
+regex = "1.5.4"
 serial_test = "0.5.1"
+sysinfo = "0.23.0"
 
 # Async helpers
 async-trait = "0.1.42"
 async-recursion = "0.3.2"
-futures = "0.3.12"
+futures = "0.3.17"
 
 # Async runtime
-tokio = { version = "1", features = ["full"] }
+tokio = { version = "1.14.0", features = ["full"] }
 tokio-vsock = "0.3.1"
 
 netlink-sys = { version = "0.7.0", features = ["tokio_socket",]}
@@ -37,7 +38,7 @@ ipnetwork = "0.17.0"
 
 # Note: this crate sets the slog 'max_*' features which allows the log level
 # to be modified at runtime.
-logging = { path = "../../pkg/logging" }
+logging = { path = "../libs/logging" }
 slog = "2.5.2"
 slog-scope = "4.1.2"
 
@@ -45,10 +46,10 @@ slog-scope = "4.1.2"
 slog-stdlog = "4.0.0"
 log = "0.4.11"
 
-prometheus = { version = "0.9.0", features = ["process"] }
-procfs = "0.7.9"
+prometheus = { version = "0.13.0", features = ["process"] }
+procfs = "0.12.0"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.2.5" }
+cgroups = { package = "cgroups-rs", version = "0.2.8" }
 
 # Tracing
 tracing = "0.1.26"
@@ -60,14 +61,13 @@ vsock-exporter = { path = "vsock-exporter" }
 # Configuration
 serde = { version = "1.0.129", features = ["derive"] }
 toml = "0.5.8"
+clap = { version = "3.0.1", features = ["derive"] }
 
 [dev-dependencies]
 tempfile = "3.1.0"
 
 [workspace]
 members = [
-    "oci",
-    "protocols",
     "rustjail",
 ]
 

src/agent/Makefile

@@ -98,13 +98,15 @@ define INSTALL_FILE
 	install -D -m 644 $1 $(DESTDIR)$2/$1 || exit 1;
 endef
 
+.DEFAULT_GOAL := default
+
 ##TARGET default: build code
 default: $(TARGET) show-header
 
 $(TARGET): $(GENERATED_CODE) logging-crate-tests $(TARGET_PATH)
 
 logging-crate-tests:
-	make -C $(CWD)/../../pkg/logging
+	make -C $(CWD)/../libs/logging
 
 $(TARGET_PATH): $(SOURCES) | show-summary
 	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
@@ -116,17 +118,6 @@ $(GENERATED_FILES): %: %.in
 optimize: $(SOURCES) | show-summary show-header
 	@RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
 
-##TARGET clippy: run clippy linter
-clippy: $(GENERATED_CODE)
-	cargo clippy --all-targets --all-features --release \
-		-- \
-		-Aclippy::redundant_allocation \
-		-D warnings
-
-format:
-	cargo fmt -- --check
-
-
 ##TARGET install: install agent
 install: install-services
 	@install -D $(TARGET_PATH) $(DESTDIR)/$(BINDIR)/$(TARGET)
@@ -146,7 +137,7 @@ test:
 	@cargo test --all --target $(TRIPLE) $(EXTRA_RUSTFEATURES) -- --nocapture
 
 ##TARGET check: run test
-check: clippy format
+check: $(GENERATED_FILES) standard_rust_check
 
 ##TARGET run: build and run agent
 run:
@@ -216,4 +207,4 @@ codecov-html: check_tarpaulin
 
 ##TARGET generate-protocols: generate/update grpc agent protocols
 generate-protocols:
-	protocols/hack/update-generated-proto.sh all
+	../libs/protocols/hack/update-generated-proto.sh all

src/agent/README.md

@@ -6,14 +6,14 @@ The Kata agent is a long running process that runs inside the Virtual Machine
 (VM) (also known as the "pod" or "sandbox").
 
 The agent is packaged inside the Kata Containers
-[guest image](../../docs/design/architecture.md#guest-image)
+[guest image](../../docs/design/architecture/README.md#guest-image)
 which is used to boot the VM. Once the runtime has launched the configured
 [hypervisor](../../docs/hypervisors.md) to create a new VM, the agent is
 started. From this point on, the agent is responsible for creating and
 managing the life cycle of the containers inside the VM.
 
 For further details, see the
-[architecture document](../../docs/design/architecture.md).
+[architecture document](../../docs/design/architecture).
 
 ## Audience
 
@@ -60,17 +60,21 @@ $ make -C ${dir}/kata-containers/src/agent
 
 The Kata runtime communicates with the Kata agent using a ttRPC based API protocol.
 
-This ttRPC API is defined by a set of [protocol buffers files](protocols/protos).
+This ttRPC API is defined by a set of [protocol buffers files](../libs/protocols/protos).
 The protocol files are used to generate the bindings for the following components:
 
-| Component | Language | Generation method | Tooling required |
+| Component | Language | Generation method `[*]` | Tooling required |
 |-|-|-|-|
 | runtime | Golang | Run, `make generate-protocols` | `protoc` |
 | agent | Rust | Run, `make` |  |
 
+> **Key:**
+>
+> `[*]` - All commands must be run in the agent repository.
+
 If you wish to change the API, these files must be regenerated. Although the
 rust code will be automatically generated by the
-[build script](protocols/build.rs),
+[build script](../libs/protocols/build.rs),
 the Golang code generation requires the external `protoc` command to be
 available in `$PATH`.
 
@@ -97,7 +101,7 @@ these custom assets to allow you to test your changes.
 ## Tracing
 
 For details of tracing the operation of the agent, see the
-[tracing documentation](../../docs/tracing.md).
+[tracing documentation](/docs/tracing.md).
 
 ## Run the agent stand alone
 
@@ -108,4 +112,4 @@ When run in this way, the agent can be controlled using the low-level Kata
 agent control tool, rather than the Kata runtime.
 
 For further details, see the
-[agent control tool documentation](../../tools/agent-ctl/README.md#run-the-tool-and-the-agent-in-the-same-environment).
+[agent control tool documentation](../tools/agent-ctl/README.md#run-the-tool-and-the-agent-in-the-same-environment).

src/agent/rustjail/Cargo.toml

@@ -8,10 +8,10 @@ edition = "2018"
 serde = "1.0.91"
 serde_json = "1.0.39"
 serde_derive = "1.0.91"
-oci = { path = "../oci" }
-protocols = { path ="../protocols" }
+oci = { path = "../../libs/oci" }
+protocols = { path ="../../libs/protocols" }
 caps = "0.5.0"
-nix = "0.21.0"
+nix = "0.23.0"
 scopeguard = "1.0.0"
 capctl = "0.2.0"
 lazy_static = "1.3.0"
@@ -19,15 +19,15 @@ libc = "0.2.58"
 protobuf = "=2.14.0"
 slog = "2.5.2"
 slog-scope = "4.1.2"
-scan_fmt = "0.2"
-regex = "1.1"
+scan_fmt = "0.2.6"
+regex = "1.5.4"
 path-absolutize = "1.2.0"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.2.5" }
+cgroups = { package = "cgroups-rs", version = "0.2.8" }
 rlimit = "0.5.3"
 
 tokio = { version = "1.2.0", features = ["sync", "io-util", "process", "time", "macros"] }
-futures = "0.3"
+futures = "0.3.17"
 async-trait = "0.1.31"
 inotify = "0.9.2"
 libseccomp = { version = "0.1.3", optional = true }

src/agent/rustjail/src/cgroups/fs/mod.rs

@@ -22,7 +22,6 @@ use crate::cgroups::Manager as CgroupManager;
 use crate::container::DEFAULT_DEVICES;
 use anyhow::{anyhow, Context, Result};
 use libc::{self, pid_t};
-use nix::errno::Errno;
 use oci::{
     LinuxBlockIo, LinuxCpu, LinuxDevice, LinuxDeviceCgroup, LinuxHugepageLimit, LinuxMemory,
     LinuxNetwork, LinuxPids, LinuxResources,
@@ -175,7 +174,7 @@ impl CgroupManager for Manager {
                 freezer_controller.freeze()?;
             }
             _ => {
-                return Err(nix::Error::Sys(Errno::EINVAL).into());
+                return Err(anyhow!(nix::Error::EINVAL));
             }
         }
 
@@ -906,13 +905,7 @@ pub fn get_paths() -> Result<HashMap<String, String>> {
 
         let keys: Vec<&str> = fl[1].split(',').collect();
         for key in &keys {
-            // this is a workaround, cgroup file are using `name=systemd`,
-            // but if file system the name is `systemd`
-            if *key == "name=systemd" {
-                m.insert("systemd".to_string(), fl[2].to_string());
-            } else {
-                m.insert(key.to_string(), fl[2].to_string());
-            }
+            m.insert(key.to_string(), fl[2].to_string());
         }
     }
     Ok(m)

src/agent/rustjail/src/container.rs

@@ -215,7 +215,6 @@ pub trait BaseContainer {
     async fn start(&mut self, p: Process) -> Result<()>;
     async fn run(&mut self, p: Process) -> Result<()>;
     async fn destroy(&mut self) -> Result<()>;
-    fn signal(&self, sig: Signal, all: bool) -> Result<()>;
     fn exec(&mut self) -> Result<()>;
 }
 
@@ -419,7 +418,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
                         ns.r#type.clone(),
                         ns.path.clone()
                     );
-                    log_child!(cfd_log, "error is : {:?}", e.as_errno());
+                    log_child!(cfd_log, "error is : {:?}", e);
                     e
                 })?;
 
@@ -496,7 +495,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         log_child!(cfd_log, "join namespace {:?}", s);
         sched::setns(fd, s).or_else(|e| {
             if s == CloneFlags::CLONE_NEWUSER {
-                if e.as_errno().unwrap() != Errno::EINVAL {
+                if e != Errno::EINVAL {
                     let _ = write_sync(cwfd, SYNC_FAILED, format!("{:?}", e).as_str());
                     return Err(e);
                 }
@@ -600,6 +599,14 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         capctl::prctl::set_no_new_privs().map_err(|_| anyhow!("cannot set no new privileges"))?;
     }
 
+    // Log unknown seccomp system calls in advance before the log file descriptor closes.
+    #[cfg(feature = "seccomp")]
+    if let Some(ref scmp) = linux.seccomp {
+        if let Some(syscalls) = seccomp::get_unknown_syscalls(scmp) {
+            log_child!(cfd_log, "unknown seccomp system calls: {:?}", syscalls);
+        }
+    }
+
     // Without NoNewPrivileges, we need to set seccomp
     // before dropping capabilities because the calling thread
     // must have the CAP_SYS_ADMIN.
@@ -677,8 +684,8 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
             Mode::from_bits_truncate(0),
         )?;
         unistd::close(fifofd)?;
-        let mut buf: &mut [u8] = &mut [0];
-        unistd::read(fd, &mut buf)?;
+        let buf: &mut [u8] = &mut [0];
+        unistd::read(fd, buf)?;
     }
 
     // With NoNewPrivileges, we should set seccomp as close to
@@ -1049,18 +1056,6 @@ impl BaseContainer for LinuxContainer {
         Ok(())
     }
 
-    fn signal(&self, sig: Signal, all: bool) -> Result<()> {
-        if all {
-            for pid in self.processes.keys() {
-                signal::kill(Pid::from_raw(*pid), Some(sig))?;
-            }
-        }
-
-        signal::kill(Pid::from_raw(self.init_process_pid), Some(sig))?;
-
-        Ok(())
-    }
-
     fn exec(&mut self) -> Result<()> {
         let fifo = format!("{}/{}", &self.root, EXEC_FIFO_FILENAME);
         let fd = fcntl::open(fifo.as_str(), OFlag::O_WRONLY, Mode::from_bits_truncate(0))?;
@@ -1108,10 +1103,8 @@ fn do_exec(args: &[String]) -> ! {
         .collect();
 
     let _ = unistd::execvp(p.as_c_str(), &sa).map_err(|e| match e {
-        nix::Error::Sys(errno) => {
-            std::process::exit(errno as i32);
-        }
-        _ => std::process::exit(-2),
+        nix::Error::UnknownErrno => std::process::exit(-2),
+        _ => std::process::exit(e as i32),
     });
 
     unreachable!()
@@ -1157,7 +1150,7 @@ fn get_pid_namespace(logger: &Logger, linux: &Linux) -> Result<Option<RawFd>> {
                         ns.r#type.clone(),
                         ns.path.clone()
                     );
-                    error!(logger, "error is : {:?}", e.as_errno());
+                    error!(logger, "error is : {:?}", e);
 
                     e
                 })?;
@@ -1390,13 +1383,13 @@ impl LinuxContainer {
         .context(format!("cannot change onwer of container {} root", id))?;
 
         if config.spec.is_none() {
-            return Err(nix::Error::Sys(Errno::EINVAL).into());
+            return Err(anyhow!(nix::Error::EINVAL));
         }
 
         let spec = config.spec.as_ref().unwrap();
 
         if spec.linux.is_none() {
-            return Err(nix::Error::Sys(Errno::EINVAL).into());
+            return Err(anyhow!(nix::Error::EINVAL));
         }
 
         let linux = spec.linux.as_ref().unwrap();
@@ -1473,18 +1466,18 @@ async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
     let binary = PathBuf::from(h.path.as_str());
     let path = binary.canonicalize()?;
     if !path.exists() {
-        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
+        return Err(anyhow!(nix::Error::EINVAL));
     }
 
-    let args = h.args.clone();
-    let env: HashMap<String, String> = h
-        .env
-        .iter()
-        .map(|e| {
-            let v: Vec<&str> = e.split('=').collect();
-            (v[0].to_string(), v[1].to_string())
-        })
-        .collect();
+    let mut args = h.args.clone();
+    // the hook.args[0] is the hook binary name which shouldn't be included
+    // in the Command.args
+    if args.len() > 1 {
+        args.remove(0);
+    }
+
+    // all invalid envs will be omitted, only valid envs will be passed to hook.
+    let env: HashMap<&str, &str> = h.env.iter().filter_map(|e| valid_env(e)).collect();
 
     // Avoid the exit signal to be reaped by the global reaper.
     let _wait_locker = WAIT_PID_LOCKER.lock().await;
@@ -1495,8 +1488,7 @@ async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
         .stdin(Stdio::piped())
         .stdout(Stdio::piped())
         .stderr(Stdio::piped())
-        .spawn()
-        .unwrap();
+        .spawn()?;
 
     // default timeout 10s
     let mut timeout: u64 = 10;
@@ -1512,27 +1504,39 @@ async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
     let path = h.path.clone();
 
     let join_handle = tokio::spawn(async move {
-        child
-            .stdin
-            .as_mut()
-            .unwrap()
-            .write_all(state.as_bytes())
-            .await
-            .unwrap();
+        if let Some(mut stdin) = child.stdin.take() {
+            match stdin.write_all(state.as_bytes()).await {
+                Ok(_) => {}
+                Err(e) => {
+                    info!(logger, "write to child stdin failed: {:?}", e);
+                }
+            }
+        }
 
-        // Close stdin so that hook program could receive EOF
-        child.stdin.take();
+        // read something from stdout and stderr for debug
+        if let Some(stdout) = child.stdout.as_mut() {
+            let mut out = String::new();
+            match stdout.read_to_string(&mut out).await {
+                Ok(_) => {
+                    info!(logger, "child stdout: {}", out.as_str());
+                }
+                Err(e) => {
+                    info!(logger, "read from child stdout failed: {:?}", e);
+                }
+            }
+        }
 
-        // read something from stdout for debug
-        let mut out = String::new();
-        child
-            .stdout
-            .as_mut()
-            .unwrap()
-            .read_to_string(&mut out)
-            .await
-            .unwrap();
-        info!(logger, "child stdout: {}", out.as_str());
+        let mut err = String::new();
+        if let Some(stderr) = child.stderr.as_mut() {
+            match stderr.read_to_string(&mut err).await {
+                Ok(_) => {
+                    info!(logger, "child stderr: {}", err.as_str());
+                }
+                Err(e) => {
+                    info!(logger, "read from child stderr failed: {:?}", e);
+                }
+            }
+        }
 
         match child.wait().await {
             Ok(exit) => {
@@ -1541,8 +1545,11 @@ async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
                     .ok_or_else(|| anyhow!("hook exit status has no status code"))?;
 
                 if code != 0 {
-                    error!(logger, "hook {} exit status is {}", &path, code);
-                    return Err(anyhow!(nix::Error::from_errno(Errno::UnknownErrno)));
+                    error!(
+                        logger,
+                        "hook {} exit status is {}, error message is {}", &path, code, err
+                    );
+                    return Err(anyhow!(nix::Error::UnknownErrno));
                 }
 
                 debug!(logger, "hook {} exit status is 0", &path);
@@ -1558,7 +1565,7 @@ async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
 
     match tokio::time::timeout(Duration::new(timeout, 0), join_handle).await {
         Ok(r) => r.unwrap(),
-        Err(_) => Err(anyhow!(nix::Error::from_errno(Errno::ETIMEDOUT))),
+        Err(_) => Err(anyhow!(nix::Error::ETIMEDOUT)),
     }
 }
 
@@ -1618,13 +1625,47 @@ mod tests {
 
     #[tokio::test]
     async fn test_execute_hook() {
-        let xargs = which("xargs").await;
+        let temp_file = "/tmp/test_execute_hook";
+
+        let touch = which("touch").await;
+
+        defer!(fs::remove_file(temp_file).unwrap(););
+        let invalid_str = vec![97, b'\0', 98];
+        let invalid_string = std::str::from_utf8(&invalid_str).unwrap();
+        let invalid_env = format!("{}=value", invalid_string);
 
         execute_hook(
             &slog_scope::logger(),
             &Hook {
-                path: xargs,
-                args: vec![],
+                path: touch,
+                args: vec!["touch".to_string(), temp_file.to_string()],
+                env: vec![invalid_env],
+                timeout: Some(10),
+            },
+            &OCIState {
+                version: "1.2.3".to_string(),
+                id: "321".to_string(),
+                status: ContainerState::Running,
+                pid: 2,
+                bundle: "".to_string(),
+                annotations: Default::default(),
+            },
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(Path::new(&temp_file).exists(), true);
+    }
+
+    #[tokio::test]
+    async fn test_execute_hook_with_error() {
+        let ls = which("ls").await;
+
+        let res = execute_hook(
+            &slog_scope::logger(),
+            &Hook {
+                path: ls,
+                args: vec!["ls".to_string(), "/tmp/not-exist".to_string()],
                 env: vec![],
                 timeout: None,
             },
@@ -1637,8 +1678,13 @@ mod tests {
                 annotations: Default::default(),
             },
         )
-        .await
-        .unwrap()
+        .await;
+
+        let expected_err = nix::Error::UnknownErrno;
+        assert_eq!(
+            res.unwrap_err().downcast::<nix::Error>().unwrap(),
+            expected_err
+        );
     }
 
     #[tokio::test]
@@ -1649,7 +1695,7 @@ mod tests {
             &slog_scope::logger(),
             &Hook {
                 path: sleep,
-                args: vec!["2".to_string()],
+                args: vec!["sleep".to_string(), "2".to_string()],
                 env: vec![],
                 timeout: Some(1),
             },
@@ -1664,7 +1710,7 @@ mod tests {
         )
         .await;
 
-        let expected_err = nix::Error::from_errno(Errno::ETIMEDOUT);
+        let expected_err = nix::Error::ETIMEDOUT;
         assert_eq!(
             res.unwrap_err().downcast::<nix::Error>().unwrap(),
             expected_err
@@ -1990,14 +2036,6 @@ mod tests {
         assert!(ret.is_ok(), "Expecting Ok, Got {:?}", ret);
     }
 
-    #[test]
-    fn test_linuxcontainer_signal() {
-        let ret = new_linux_container_and_then(|c: LinuxContainer| {
-            c.signal(nix::sys::signal::SIGCONT, true)
-        });
-        assert!(ret.is_ok(), "Expecting Ok, Got {:?}", ret);
-    }
-
     #[test]
     fn test_linuxcontainer_exec() {
         let ret = new_linux_container_and_then(|mut c: LinuxContainer| c.exec());

src/agent/rustjail/src/mount.rs

@@ -5,7 +5,6 @@
 
 use anyhow::{anyhow, Context, Result};
 use libc::uid_t;
-use nix::errno::Errno;
 use nix::fcntl::{self, OFlag};
 #[cfg(not(test))]
 use nix::mount;
@@ -35,17 +34,9 @@ use crate::log_child;
 // struct is populated from the content in the /proc/<pid>/mountinfo file.
 #[derive(std::fmt::Debug)]
 pub struct Info {
-    id: i32,
-    parent: i32,
-    major: i32,
-    minor: i32,
-    root: String,
     mount_point: String,
-    opts: String,
     optional: String,
     fstype: String,
-    source: String,
-    vfs_opts: String,
 }
 
 const MOUNTINFOFORMAT: &str = "{d} {d} {d}:{d} {} {} {} {}";
@@ -563,7 +554,20 @@ fn parse_mount_table() -> Result<Vec<Info>> {
     for (_index, line) in reader.lines().enumerate() {
         let line = line?;
 
-        let (id, parent, major, minor, root, mount_point, opts, optional) = scan_fmt!(
+        //Example mountinfo format:
+        // id
+        // |  / parent
+        // |  |   / major:minor
+        // |  |   |   / root
+        // |  |   |   |  / mount_point
+        // |  |   |   |  |        / opts
+        // |  |   |   |  |        |                           / optional
+        // |  |   |   |  |        |                           |          / fstype
+        // |  |   |   |  |        |                           |          |     / source
+        // |  |   |   |  |        |                           |          |     |      / vfs_opts
+        // 22 96 0:21 / /sys rw,nosuid,nodev,noexec,relatime shared:2 - sysfs sysfs rw,seclabel
+
+        let (_id, _parent, _major, _minor, _root, mount_point, _opts, optional) = scan_fmt!(
             &line,
             MOUNTINFOFORMAT,
             i32,
@@ -578,7 +582,7 @@ fn parse_mount_table() -> Result<Vec<Info>> {
 
         let fields: Vec<&str> = line.split(" - ").collect();
         if fields.len() == 2 {
-            let (fstype, source, vfs_opts) =
+            let (fstype, _source, _vfs_opts) =
                 scan_fmt!(fields[1], "{} {} {}", String, String, String)?;
 
             let mut optional_new = String::new();
@@ -587,17 +591,9 @@ fn parse_mount_table() -> Result<Vec<Info>> {
             }
 
             let info = Info {
-                id,
-                parent,
-                major,
-                minor,
-                root,
                 mount_point,
-                opts,
                 optional: optional_new,
                 fstype,
-                source,
-                vfs_opts,
             };
 
             infos.push(info);
@@ -655,7 +651,7 @@ pub fn ms_move_root(rootfs: &str) -> Result<bool> {
             None::<&str>,
         )?;
         umount2(abs_mount_point, MntFlags::MNT_DETACH).or_else(|e| {
-            if e.ne(&nix::Error::from(Errno::EINVAL)) && e.ne(&nix::Error::from(Errno::EPERM)) {
+            if e.ne(&nix::Error::EINVAL) && e.ne(&nix::Error::EPERM) {
                 return Err(anyhow!(e));
             }
 
@@ -732,7 +728,7 @@ fn secure_join(rootfs: &str, unsafe_path: &str) -> String {
         path.push(it);
         if let Ok(v) = path.read_link() {
             if v.is_absolute() {
-                path = PathBuf::from(format!("{}{}", rootfs, v.to_str().unwrap().to_string()));
+                path = PathBuf::from(format!("{}{}", rootfs, v.to_str().unwrap()));
             } else {
                 path.pop();
                 for it in v.iter() {
@@ -777,7 +773,7 @@ fn mount_from(
         let _ = fs::create_dir_all(&dir).map_err(|e| {
             log_child!(
                 cfd_log,
-                "creat dir {}: {}",
+                "create dir {}: {}",
                 dir.to_str().unwrap(),
                 e.to_string()
             )
@@ -798,14 +794,8 @@ fn mount_from(
         }
     };
 
-    let _ = stat::stat(dest.as_str()).map_err(|e| {
-        log_child!(
-            cfd_log,
-            "dest stat error. {}: {:?}",
-            dest.as_str(),
-            e.as_errno()
-        )
-    });
+    let _ = stat::stat(dest.as_str())
+        .map_err(|e| log_child!(cfd_log, "dest stat error. {}: {:?}", dest.as_str(), e));
 
     mount(
         Some(src.as_str()),
@@ -815,7 +805,7 @@ fn mount_from(
         Some(d.as_str()),
     )
     .map_err(|e| {
-        log_child!(cfd_log, "mount error: {:?}", e.as_errno());
+        log_child!(cfd_log, "mount error: {:?}", e);
         e
     })?;
 
@@ -837,7 +827,7 @@ fn mount_from(
             None::<&str>,
         )
         .map_err(|e| {
-            log_child!(cfd_log, "remout {}: {:?}", dest.as_str(), e.as_errno());
+            log_child!(cfd_log, "remout {}: {:?}", dest.as_str(), e);
             e
         })?;
     }
@@ -1006,7 +996,7 @@ pub fn finish_rootfs(cfd_log: RawFd, spec: &Spec, process: &Process) -> Result<(
 
 fn mask_path(path: &str) -> Result<()> {
     if !path.starts_with('/') || path.contains("..") {
-        return Err(nix::Error::Sys(Errno::EINVAL).into());
+        return Err(anyhow!(nix::Error::EINVAL));
     }
 
     match mount(
@@ -1016,49 +1006,30 @@ fn mask_path(path: &str) -> Result<()> {
         MsFlags::MS_BIND,
         None::<&str>,
     ) {
-        Err(nix::Error::Sys(e)) => {
-            if e != Errno::ENOENT && e != Errno::ENOTDIR {
-                //info!("{}: {}", path, e.desc());
-                return Err(nix::Error::Sys(e).into());
-            }
-        }
-
-        Err(e) => {
-            return Err(e.into());
-        }
-
-        Ok(_) => {}
+        Err(e) => match e {
+            nix::Error::ENOENT | nix::Error::ENOTDIR => Ok(()),
+            _ => Err(e.into()),
+        },
+        Ok(_) => Ok(()),
     }
-
-    Ok(())
 }
 
 fn readonly_path(path: &str) -> Result<()> {
     if !path.starts_with('/') || path.contains("..") {
-        return Err(nix::Error::Sys(Errno::EINVAL).into());
+        return Err(anyhow!(nix::Error::EINVAL));
     }
 
-    match mount(
+    if let Err(e) = mount(
         Some(&path[1..]),
         path,
         None::<&str>,
         MsFlags::MS_BIND | MsFlags::MS_REC,
         None::<&str>,
     ) {
-        Err(nix::Error::Sys(e)) => {
-            if e == Errno::ENOENT {
-                return Ok(());
-            } else {
-                //info!("{}: {}", path, e.desc());
-                return Err(nix::Error::Sys(e).into());
-            }
-        }
-
-        Err(e) => {
-            return Err(e.into());
-        }
-
-        Ok(_) => {}
+        match e {
+            nix::Error::ENOENT => return Ok(()),
+            _ => return Err(e.into()),
+        };
     }
 
     mount(
@@ -1414,7 +1385,7 @@ mod tests {
 
         for (i, t) in tests.iter().enumerate() {
             // Create a string containing details of the test
-            let msg = format!("test[{}]: {:?}", i, t);
+            let msg = format!("test[{}]: {:?}", i, t.name);
 
             // if is_symlink, then should be prepare the softlink environment
             if t.symlink_path != "" {

src/agent/rustjail/src/pipestream.rs

@@ -30,7 +30,7 @@ impl io::Read for &StreamFd {
     fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
         match unistd::read(self.0, buf) {
             Ok(l) => Ok(l),
-            Err(e) => Err(e.as_errno().unwrap().into()),
+            Err(e) => Err(e.into()),
         }
     }
 }
@@ -39,7 +39,7 @@ impl io::Write for &StreamFd {
     fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
         match unistd::write(self.0, buf) {
             Ok(l) => Ok(l),
-            Err(e) => Err(e.as_errno().unwrap().into()),
+            Err(e) => Err(e.into()),
         }
     }
 
@@ -52,7 +52,7 @@ impl StreamFd {
     fn close(&mut self) -> io::Result<()> {
         match unistd::close(self.0) {
             Ok(()) => Ok(()),
-            Err(e) => Err(e.as_errno().unwrap().into()),
+            Err(e) => Err(e.into()),
         }
     }
 }

src/agent/rustjail/src/process.rs

@@ -8,8 +8,8 @@ use std::fs::File;
 use std::os::unix::io::RawFd;
 use tokio::sync::mpsc::Sender;
 
+use nix::errno::Errno;
 use nix::fcntl::{fcntl, FcntlArg, OFlag};
-use nix::sys::signal::{self, Signal};
 use nix::sys::wait::{self, WaitStatus};
 use nix::unistd::{self, Pid};
 use nix::Result;
@@ -80,7 +80,7 @@ pub struct Process {
 pub trait ProcessOperations {
     fn pid(&self) -> Pid;
     fn wait(&self) -> Result<WaitStatus>;
-    fn signal(&self, sig: Signal) -> Result<()>;
+    fn signal(&self, sig: libc::c_int) -> Result<()>;
 }
 
 impl ProcessOperations for Process {
@@ -92,8 +92,10 @@ impl ProcessOperations for Process {
         wait::waitpid(Some(self.pid()), None)
     }
 
-    fn signal(&self, sig: Signal) -> Result<()> {
-        signal::kill(self.pid(), Some(sig))
+    fn signal(&self, sig: libc::c_int) -> Result<()> {
+        let res = unsafe { libc::kill(self.pid().into(), sig) };
+
+        Errno::result(res).map(drop)
     }
 }
 
@@ -281,6 +283,6 @@ mod tests {
         // signal to every process in the process
         // group of the calling process.
         process.pid = 0;
-        assert!(process.signal(Signal::SIGCONT).is_ok());
+        assert!(process.signal(libc::SIGCONT).is_ok());
     }
 }

src/agent/rustjail/src/seccomp.rs

@@ -39,6 +39,24 @@ fn get_rule_conditions(args: &[LinuxSeccompArg]) -> Result<Vec<ScmpArgCompare>>
     Ok(conditions)
 }
 
+pub fn get_unknown_syscalls(scmp: &LinuxSeccomp) -> Option<Vec<String>> {
+    let mut unknown_syscalls: Vec<String> = Vec::new();
+
+    for syscall in &scmp.syscalls {
+        for name in &syscall.names {
+            if get_syscall_from_name(name, None).is_err() {
+                unknown_syscalls.push(name.to_string());
+            }
+        }
+    }
+
+    if unknown_syscalls.is_empty() {
+        None
+    } else {
+        Some(unknown_syscalls)
+    }
+}
+
 // init_seccomp creates a seccomp filter and loads it for the current process
 // including all the child processes.
 pub fn init_seccomp(scmp: &LinuxSeccomp) -> Result<()> {
@@ -68,7 +86,14 @@ pub fn init_seccomp(scmp: &LinuxSeccomp) -> Result<()> {
         }
 
         for name in &syscall.names {
-            let syscall_num = get_syscall_from_name(name, None)?;
+            let syscall_num = match get_syscall_from_name(name, None) {
+                Ok(num) => num,
+                Err(_) => {
+                    // If we cannot resolve the given system call, we assume it is not supported
+                    // by the kernel. Hence, we skip it without generating an error.
+                    continue;
+                }
+            };
 
             if syscall.args.is_empty() {
                 filter.add_rule(action, syscall_num, None)?;
@@ -109,6 +134,72 @@ mod tests {
         };
     }
 
+    const TEST_DATA: &str = r#"{
+          "defaultAction": "SCMP_ACT_ALLOW",
+          "architectures": [
+          ],
+          "flags": [
+              "SECCOMP_FILTER_FLAG_LOG"
+          ],
+          "syscalls": [
+              {
+                 "names": [
+                      "dup3",
+                      "invalid_syscall1",
+                      "invalid_syscall2"
+                  ],
+                  "action": "SCMP_ACT_ERRNO"
+              },
+              {
+                 "names": [
+                      "process_vm_readv"
+                  ],
+                  "action": "SCMP_ACT_ERRNO",
+                  "errnoRet": 111,
+                  "args": [
+                      {
+                          "index": 0,
+                          "value": 10,
+                          "op": "SCMP_CMP_EQ"
+                      }
+                  ]
+              },
+              {
+                 "names": [
+                      "process_vm_readv"
+                  ],
+                  "action": "SCMP_ACT_ERRNO",
+                  "errnoRet": 111,
+                  "args": [
+                      {
+                          "index": 0,
+                          "value": 20,
+                          "op": "SCMP_CMP_EQ"
+                      }
+                  ]
+              },
+              {
+                 "names": [
+                      "process_vm_readv"
+                  ],
+                  "action": "SCMP_ACT_ERRNO",
+                  "errnoRet": 222,
+                  "args": [
+                      {
+                          "index": 0,
+                          "value": 30,
+                          "op": "SCMP_CMP_EQ"
+                      },
+                      {
+                          "index": 2,
+                          "value": 40,
+                          "op": "SCMP_CMP_EQ"
+                      }
+                  ]
+              }
+          ]
+      }"#;
+
     #[test]
     fn test_get_filter_attr_from_flag() {
         skip_if_not_root!();
@@ -121,75 +212,19 @@ mod tests {
         assert_eq!(get_filter_attr_from_flag("ERROR").is_err(), true);
     }
 
+    #[test]
+    fn test_get_unknown_syscalls() {
+        let scmp: oci::LinuxSeccomp = serde_json::from_str(TEST_DATA).unwrap();
+        let syscalls = get_unknown_syscalls(&scmp).unwrap();
+
+        assert_eq!(syscalls, vec!["invalid_syscall1", "invalid_syscall2"]);
+    }
+
     #[test]
     fn test_init_seccomp() {
         skip_if_not_root!();
 
-        let data = r#"{
-            "defaultAction": "SCMP_ACT_ALLOW",
-            "architectures": [
-            ],
-            "flags": [
-                "SECCOMP_FILTER_FLAG_LOG"
-            ],
-            "syscalls": [
-                {
-                   "names": [
-                        "dup3"
-                    ],
-                    "action": "SCMP_ACT_ERRNO"
-                },
-                {
-                   "names": [
-                        "process_vm_readv"
-                    ],
-                    "action": "SCMP_ACT_ERRNO",
-                    "errnoRet": 111,
-                    "args": [
-                        {
-                            "index": 0,
-                            "value": 10,
-                            "op": "SCMP_CMP_EQ"
-                        }
-                    ]
-                },
-                {
-                   "names": [
-                        "process_vm_readv"
-                    ],
-                    "action": "SCMP_ACT_ERRNO",
-                    "errnoRet": 111,
-                    "args": [
-                        {
-                            "index": 0,
-                            "value": 20,
-                            "op": "SCMP_CMP_EQ"
-                        }
-                    ]
-                },
-                {
-                   "names": [
-                        "process_vm_readv"
-                    ],
-                    "action": "SCMP_ACT_ERRNO",
-                    "errnoRet": 222,
-                    "args": [
-                        {
-                            "index": 0,
-                            "value": 30,
-                            "op": "SCMP_CMP_EQ"
-                        },
-                        {
-                            "index": 2,
-                            "value": 40,
-                            "op": "SCMP_CMP_EQ"
-                        }
-                    ]
-                }
-            ]
-        }"#;
-
-        let mut scmp: oci::LinuxSeccomp = serde_json::from_str(data).unwrap();
+        let mut scmp: oci::LinuxSeccomp = serde_json::from_str(TEST_DATA).unwrap();
         let mut arch: Vec<oci::Arch>;
 
         if cfg!(target_endian = "little") {

src/agent/rustjail/src/sync.rs

@@ -3,7 +3,6 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use nix::errno::Errno;
 use nix::unistd;
 use std::mem;
 use std::os::unix::io::RawFd;
@@ -41,7 +40,7 @@ pub fn write_count(fd: RawFd, buf: &[u8], count: usize) -> Result<usize> {
             }
 
             Err(e) => {
-                if e != nix::Error::from_errno(Errno::EINTR) {
+                if e != nix::Error::EINTR {
                     return Err(e.into());
                 }
             }
@@ -65,7 +64,7 @@ fn read_count(fd: RawFd, count: usize) -> Result<Vec<u8>> {
             }
 
             Err(e) => {
-                if e != nix::Error::from_errno(Errno::EINTR) {
+                if e != nix::Error::EINTR {
                     return Err(e.into());
                 }
             }

src/agent/rustjail/src/validator.rs

@@ -5,13 +5,12 @@
 
 use crate::container::Config;
 use anyhow::{anyhow, Context, Error, Result};
-use nix::errno::Errno;
 use oci::{Linux, LinuxIdMapping, LinuxNamespace, Spec};
 use std::collections::HashMap;
 use std::path::{Component, PathBuf};
 
 fn einval() -> Error {
-    anyhow!(nix::Error::from_errno(Errno::EINVAL))
+    anyhow!(nix::Error::EINVAL)
 }
 
 fn get_linux(oci: &Spec) -> Result<&Linux> {

src/agent/samples/configuration-all-endpoints.toml

@@ -25,6 +25,7 @@ allowed = [
         "ReadStreamRequest",
         "RemoveContainerRequest",
         "ReseedRandomDevRequest",
+        "ResizeVolumeRequest",
         "ResumeContainerRequest",
         "SetGuestDateTimeRequest",
         "SignalProcessRequest",
@@ -34,6 +35,7 @@ allowed = [
         "UpdateContainerRequest",
         "UpdateInterfaceRequest",
         "UpdateRoutesRequest",
+        "VolumeStatsRequest",
         "WaitProcessRequest",
         "WriteStreamRequest"
 ]

src/agent/src/config.rs

@@ -194,7 +194,17 @@ impl FromStr for AgentConfig {
 
 impl AgentConfig {
     #[instrument]
-    pub fn from_cmdline(file: &str) -> Result<AgentConfig> {
+    pub fn from_cmdline(file: &str, args: Vec<String>) -> Result<AgentConfig> {
+        // If config file specified in the args, generate our config from it
+        let config_position = args.iter().position(|a| a == "--config" || a == "-c");
+        if let Some(config_position) = config_position {
+            if let Some(config_file) = args.get(config_position + 1) {
+                return AgentConfig::from_config_file(config_file);
+            } else {
+                panic!("The config argument wasn't formed properly: {:?}", args);
+            }
+        }
+
         let mut config: AgentConfig = Default::default();
         let cmdline = fs::read_to_string(file)?;
         let params: Vec<&str> = cmdline.split_ascii_whitespace().collect();
@@ -896,7 +906,8 @@ mod tests {
                 vars_to_unset.push(name);
             }
 
-            let config = AgentConfig::from_cmdline(filename).expect("Failed to parse command line");
+            let config =
+                AgentConfig::from_cmdline(filename, vec![]).expect("Failed to parse command line");
 
             assert_eq!(d.debug_console, config.debug_console, "{}", msg);
             assert_eq!(d.dev_mode, config.dev_mode, "{}", msg);
@@ -917,6 +928,40 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_from_cmdline_with_args_overwrites() {
+        let expected = AgentConfig {
+            dev_mode: true,
+            server_addr: "unix://@/tmp/foo.socket".to_string(),
+            ..Default::default()
+        };
+
+        let example_config_file_contents =
+            "dev_mode = true\nserver_addr = 'unix://@/tmp/foo.socket'";
+        let dir = tempdir().expect("failed to create tmpdir");
+        let file_path = dir.path().join("config.toml");
+        let filename = file_path.to_str().expect("failed to create filename");
+        let mut file = File::create(filename).unwrap_or_else(|_| panic!("failed to create file"));
+        file.write_all(example_config_file_contents.as_bytes())
+            .unwrap_or_else(|_| panic!("failed to write file contents"));
+
+        let config =
+            AgentConfig::from_cmdline("", vec!["--config".to_string(), filename.to_string()])
+                .expect("Failed to parse command line");
+
+        assert_eq!(expected.debug_console, config.debug_console);
+        assert_eq!(expected.dev_mode, config.dev_mode);
+        assert_eq!(
+            expected.unified_cgroup_hierarchy,
+            config.unified_cgroup_hierarchy,
+        );
+        assert_eq!(expected.log_level, config.log_level);
+        assert_eq!(expected.hotplug_timeout, config.hotplug_timeout);
+        assert_eq!(expected.container_pipe_size, config.container_pipe_size);
+        assert_eq!(expected.server_addr, config.server_addr);
+        assert_eq!(expected.tracing, config.tracing);
+    }
+
     #[test]
     fn test_logrus_to_slog_level() {
         #[derive(Debug)]

src/agent/src/console.rs

@@ -149,10 +149,8 @@ fn run_in_child(slave_fd: libc::c_int, shell: String) -> Result<()> {
 
     // run shell
     let _ = unistd::execvp(cmd.as_c_str(), &args).map_err(|e| match e {
-        nix::Error::Sys(errno) => {
-            std::process::exit(errno as i32);
-        }
-        _ => std::process::exit(-2),
+        nix::Error::UnknownErrno => std::process::exit(-2),
+        _ => std::process::exit(e as i32),
     });
 
     Ok(())

src/agent/src/device.rs

@@ -51,6 +51,8 @@ pub const DRIVER_VFIO_GK_TYPE: &str = "vfio-gk";
 // VFIO device to be bound to vfio-pci and made available inside the
 // container as a VFIO device node
 pub const DRIVER_VFIO_TYPE: &str = "vfio";
+pub const DRIVER_OVERLAYFS_TYPE: &str = "overlayfs";
+pub const FS_TYPE_HUGETLB: &str = "hugetlbfs";
 
 #[instrument]
 pub fn online_device(path: &str) -> Result<()> {
@@ -592,38 +594,38 @@ fn update_spec_devices(spec: &mut Spec, mut updates: HashMap<&str, DevUpdate>) -
     Ok(())
 }
 
-// update_spec_pci PCI addresses in the OCI spec to be guest addresses
-// instead of host addresses.  It is given a map of (host address =>
-// guest address)
+// update_env_pci alters PCI addresses in a set of environment
+// variables to be correct for the VM instead of the host.  It is
+// given a map of (host address => guest address)
 #[instrument]
-fn update_spec_pci(spec: &mut Spec, updates: HashMap<pci::Address, pci::Address>) -> Result<()> {
-    // Correct PCI addresses in the environment
-    if let Some(process) = spec.process.as_mut() {
-        for envvar in process.env.iter_mut() {
-            let eqpos = envvar
-                .find('=')
-                .ok_or_else(|| anyhow!("Malformed OCI env entry {:?}", envvar))?;
+pub fn update_env_pci(
+    env: &mut [String],
+    pcimap: &HashMap<pci::Address, pci::Address>,
+) -> Result<()> {
+    for envvar in env {
+        let eqpos = envvar
+            .find('=')
+            .ok_or_else(|| anyhow!("Malformed OCI env entry {:?}", envvar))?;
 
-            let (name, eqval) = envvar.split_at(eqpos);
-            let val = &eqval[1..];
+        let (name, eqval) = envvar.split_at(eqpos);
+        let val = &eqval[1..];
 
-            if !name.starts_with("PCIDEVICE_") {
-                continue;
-            }
-
-            let mut guest_addrs = Vec::<String>::new();
-
-            for host_addr in val.split(',') {
-                let host_addr = pci::Address::from_str(host_addr)
-                    .with_context(|| format!("Can't parse {} environment variable", name))?;
-                let guest_addr = updates
-                    .get(&host_addr)
-                    .ok_or_else(|| anyhow!("Unable to translate host PCI address {}", host_addr))?;
-                guest_addrs.push(format!("{}", guest_addr));
-            }
-
-            envvar.replace_range(eqpos + 1.., guest_addrs.join(",").as_str());
+        if !name.starts_with("PCIDEVICE_") {
+            continue;
         }
+
+        let mut guest_addrs = Vec::<String>::new();
+
+        for host_addr in val.split(',') {
+            let host_addr = pci::Address::from_str(host_addr)
+                .with_context(|| format!("Can't parse {} environment variable", name))?;
+            let guest_addr = pcimap
+                .get(&host_addr)
+                .ok_or_else(|| anyhow!("Unable to translate host PCI address {}", host_addr))?;
+            guest_addrs.push(format!("{}", guest_addr));
+        }
+
+        envvar.replace_range(eqpos + 1.., guest_addrs.join(",").as_str());
     }
 
     Ok(())
@@ -746,7 +748,7 @@ async fn vfio_device_handler(device: &Device, sandbox: &Arc<Mutex<Sandbox>>) ->
 
     let dev_update = if vfio_in_guest {
         // If there are any devices at all, logic above ensures that group is not None
-        let group = group.ok_or_else(|| anyhow!("failed to get VFIO group: {:?}"))?;
+        let group = group.ok_or_else(|| anyhow!("failed to get VFIO group"))?;
 
         let vm_path = get_vfio_device_name(sandbox, group).await?;
 
@@ -768,7 +770,6 @@ pub async fn add_devices(
     sandbox: &Arc<Mutex<Sandbox>>,
 ) -> Result<()> {
     let mut dev_updates = HashMap::<&str, DevUpdate>::with_capacity(devices.len());
-    let mut pci_updates = HashMap::<pci::Address, pci::Address>::new();
 
     for device in devices.iter() {
         let update = add_device(device, sandbox).await?;
@@ -783,8 +784,9 @@ pub async fn add_devices(
                 ));
             }
 
+            let mut sb = sandbox.lock().await;
             for (host, guest) in update.pci {
-                if let Some(other_guest) = pci_updates.insert(host, guest) {
+                if let Some(other_guest) = sb.pcimap.insert(host, guest) {
                     return Err(anyhow!(
                         "Conflicting guest address for host device {} ({} versus {})",
                         host,
@@ -796,6 +798,9 @@ pub async fn add_devices(
         }
     }
 
+    if let Some(process) = spec.process.as_mut() {
+        update_env_pci(&mut process.env, &sandbox.lock().await.pcimap)?
+    }
     update_spec_devices(spec, dev_updates)
 }
 
@@ -860,7 +865,7 @@ pub fn update_device_cgroup(spec: &mut Spec) -> Result<()> {
 mod tests {
     use super::*;
     use crate::uevent::spawn_test_watcher;
-    use oci::{Linux, Process};
+    use oci::Linux;
     use std::iter::FromIterator;
     use tempfile::tempdir;
 
@@ -1199,7 +1204,7 @@ mod tests {
     }
 
     #[test]
-    fn test_update_spec_pci() {
+    fn test_update_env_pci() {
         let example_map = [
             // Each is a host,guest pair of pci addresses
             ("0000:1a:01.0", "0000:01:01.0"),
@@ -1209,17 +1214,11 @@ mod tests {
             ("0000:01:01.0", "ffff:02:1f.7"),
         ];
 
-        let mut spec = Spec {
-            process: Some(Process {
-                env: vec![
-                    "PCIDEVICE_x=0000:1a:01.0,0000:1b:02.0".to_string(),
-                    "PCIDEVICE_y=0000:01:01.0".to_string(),
-                    "NOTAPCIDEVICE_blah=abcd:ef:01.0".to_string(),
-                ],
-                ..Process::default()
-            }),
-            ..Spec::default()
-        };
+        let mut env = vec![
+            "PCIDEVICE_x=0000:1a:01.0,0000:1b:02.0".to_string(),
+            "PCIDEVICE_y=0000:01:01.0".to_string(),
+            "NOTAPCIDEVICE_blah=abcd:ef:01.0".to_string(),
+        ];
 
         let pci_fixups = example_map
             .iter()
@@ -1231,10 +1230,9 @@ mod tests {
             })
             .collect();
 
-        let res = update_spec_pci(&mut spec, pci_fixups);
+        let res = update_env_pci(&mut env, &pci_fixups);
         assert!(res.is_ok());
 
-        let env = &spec.process.as_ref().unwrap().env;
         assert_eq!(env[0], "PCIDEVICE_x=0000:01:01.0,0000:01:02.0");
         assert_eq!(env[1], "PCIDEVICE_y=ffff:02:1f.7");
         assert_eq!(env[2], "NOTAPCIDEVICE_blah=abcd:ef:01.0");

src/agent/src/main.rs

@@ -20,6 +20,7 @@ extern crate scopeguard;
 extern crate slog;
 
 use anyhow::{anyhow, Context, Result};
+use clap::{AppSettings, Parser};
 use nix::fcntl::OFlag;
 use nix::sys::socket::{self, AddressFamily, SockAddr, SockFlag, SockType};
 use nix::unistd::{self, dup, Pid};
@@ -80,10 +81,32 @@ const NAME: &str = "kata-agent";
 
 lazy_static! {
     static ref AGENT_CONFIG: Arc<RwLock<AgentConfig>> = Arc::new(RwLock::new(
-        AgentConfig::from_cmdline("/proc/cmdline").unwrap()
+        // Note: We can't do AgentOpts.parse() here to send through the processed arguments to AgentConfig
+        // clap::Parser::parse() greedily process all command line input including cargo test parameters,
+        // so should only be used inside main.
+        AgentConfig::from_cmdline("/proc/cmdline", env::args().collect()).unwrap()
     ));
 }
 
+#[derive(Parser)]
+// The default clap version info doesn't match our form, so we need to override it
+#[clap(global_setting(AppSettings::DisableVersionFlag))]
+struct AgentOpts {
+    /// Print the version information
+    #[clap(short, long)]
+    version: bool,
+    #[clap(subcommand)]
+    subcmd: Option<SubCommand>,
+    /// Specify a custom agent config file
+    #[clap(short, long)]
+    config: Option<String>,
+}
+
+#[derive(Parser)]
+enum SubCommand {
+    Init {},
+}
+
 #[instrument]
 fn announce(logger: &Logger, config: &AgentConfig) {
     info!(logger, "announce";
@@ -255,9 +278,9 @@ async fn real_main() -> std::result::Result<(), Box<dyn std::error::Error>> {
 }
 
 fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
-    let args: Vec<String> = env::args().collect();
+    let args = AgentOpts::parse();
 
-    if args.len() == 2 && args[1] == "--version" {
+    if args.version {
         println!(
             "{} version {} (api version: {}, commit version: {}, type: rust)",
             NAME,
@@ -265,11 +288,10 @@ fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
             version::API_VERSION,
             version::VERSION_COMMIT,
         );
-
         exit(0);
     }
 
-    if args.len() == 2 && args[1] == "init" {
+    if let Some(SubCommand::Init {}) = args.subcmd {
         reset_sigpipe();
         rustjail::container::init_child();
         exit(0);