Merge pull request #4384 from snir911/2.5.0-alpha2-branch-bump

# Kata Containers 2.5.0-alpha2

.github/workflows/add-pr-sizing-label.yaml

@@ -0,0 +1,38 @@
+# Copyright (c) 2022 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+name: Add PR sizing label
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - reopened
+      - synchronize
+
+jobs:
+  add-pr-size-label:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v1
+
+      - name: Install PR sizing label script
+        run: |
+          # Clone into a temporary directory to avoid overwriting
+          # any existing github directory.
+          pushd $(mktemp -d) &>/dev/null
+          git clone --single-branch --depth 1 "https://github.com/kata-containers/.github" && cd .github/scripts
+          sudo install pr-add-size-label.sh /usr/local/bin
+          popd &>/dev/null
+
+      - name: Add PR sizing label
+        env:
+          GITHUB_TOKEN: ${{ secrets.KATA_GITHUB_ACTIONS_PR_SIZE_TOKEN }}
+        run: |
+          pr=${{ github.event.number }}
+          sudo apt -y install diffstat patchutils
+
+          pr-add-size-label.sh -p "$pr"

.github/workflows/commit-message-check.yaml

@@ -5,14 +5,12 @@ on:
       - opened
       - reopened
       - synchronize
-      - labeled
-      - unlabeled
 
 env:
   error_msg: |+
     See the document below for help on formatting commits for the project.
 
-    https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md#patch-format
+    https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md#patch-format
 
 jobs:
   commit-message-check:
@@ -22,9 +20,15 @@ jobs:
     - name: Get PR Commits
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       id: 'get-pr-commits'
-      uses: tim-actions/get-pr-commits@v1.0.0
+      uses: tim-actions/get-pr-commits@v1.2.0
       with:
         token: ${{ secrets.GITHUB_TOKEN }}
+        # Filter out revert commits
+        # The format of a revert commit is as follows:
+        #
+        # Revert "<original-subject-line>"
+        #
+        filter_out_pattern: '^Revert "'
 
     - name: DCO Check
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}

.github/workflows/darwin-tests.yaml

@@ -0,0 +1,25 @@
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+
+name: Darwin tests
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.16.x, 1.17.x]
+        os: [macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Build utils
+      run: ./ci/darwin-test.sh

.github/workflows/docs-url-alive-check.yaml

@@ -0,0 +1,44 @@
+on:
+  schedule:
+    - cron:  '0 23 * * 0'
+
+name: Docs URL Alive Check
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.17.x]
+        os: [ubuntu-20.04]
+    runs-on: ${{ matrix.os }}
+    env:
+      target_branch: ${{ github.base_ref }}
+    steps:
+    - name: Install Go
+      if: github.repository_owner == 'kata-containers'
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+      env:
+        GOPATH: ${{ runner.workspace }}/kata-containers
+    - name: Set env
+      if: github.repository_owner == 'kata-containers'
+      run: |
+        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
+        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+    - name: Checkout code
+      if: github.repository_owner == 'kata-containers'
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+        path: ./src/github.com/${{ github.repository }}
+    - name: Setup
+      if: github.repository_owner == 'kata-containers'
+      run: |
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
+      env:
+        GOPATH: ${{ runner.workspace }}/kata-containers
+    # docs url alive check
+    - name: Docs URL Alive Check
+      if: github.repository_owner == 'kata-containers'
+      run: |
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && make docs-url-alive-check

.github/workflows/kata-deploy-push.yaml

@@ -7,9 +7,9 @@ on:
       - edited
       - reopened
       - synchronize
-      - labeled
-      - unlabeled
-  push:
+    paths:
+      - tools/**
+      - versions.yaml
 
 jobs:
   build-asset:
@@ -18,13 +18,13 @@ jobs:
       matrix:
         asset:
           - kernel
-          - kernel-experimental
           - shim-v2
           - qemu
           - cloud-hypervisor
           - firecracker
           - rootfs-image
           - rootfs-initrd
+          - virtiofsd
     steps:
       - uses: actions/checkout@v2
       - name: Install docker

.github/workflows/kata-deploy-test.yaml

@@ -1,4 +1,5 @@
 on:
+  workflow_dispatch: # this is used to trigger the workflow on non-main branches
   issue_comment:
     types: [created, edited]
 
@@ -47,19 +48,18 @@ jobs:
           - rootfs-image
           - rootfs-initrd
           - shim-v2
+          - virtiofsd
     steps:
-      # As Github action event `issue_comment` does not provide the right ref
-      # (commit/branch) to be tested, let's use this third part action to work
-      # this limitation around.
-      - name: resolve pr refs
-        id: refs
-        uses: kata-containers/resolve-pr-refs@v0.0.3
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-
+      - name: get-PR-ref
+        id: get-PR-ref
+        run: |
+            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
+            echo "reference for PR: " ${ref}
+            echo "##[set-output name=pr-ref;]${ref}"
       - uses: actions/checkout@v2
         with:
-          ref: ${{ steps.refs.outputs.base_ref }}
+          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
+
       - name: Install docker
         run: |
           curl -fsSL https://test.docker.com -o test-docker.sh
@@ -86,17 +86,15 @@ jobs:
     runs-on: ubuntu-latest
     needs: build-asset
     steps:
-      # As Github action event `issue_comment` does not provide the right ref
-      # (commit/branch) to be tested, let's use this third part action to work
-      # this limitation around.
-      - name: resolve pr refs
-        id: refs
-        uses: kata-containers/resolve-pr-refs@v0.0.3
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: get-PR-ref
+        id: get-PR-ref
+        run: |
+            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
+            echo "reference for PR: " ${ref}
+            echo "##[set-output name=pr-ref;]${ref}"
       - uses: actions/checkout@v2
         with:
-          ref: ${{ steps.refs.outputs.base_ref }}
+          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
       - name: get-artifacts
         uses: actions/download-artifact@v2
         with:
@@ -115,17 +113,15 @@ jobs:
     needs: create-kata-tarball
     runs-on: ubuntu-latest
     steps:
-      # As Github action event `issue_comment` does not provide the right ref
-      # (commit/branch) to be tested, let's use this third part action to work
-      # this limitation around.
-      - name: resolve pr refs
-        id: refs
-        uses: kata-containers/resolve-pr-refs@v0.0.3
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: get-PR-ref
+        id: get-PR-ref
+        run: |
+            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
+            echo "reference for PR: " ${ref}
+            echo "##[set-output name=pr-ref;]${ref}"
       - uses: actions/checkout@v2
         with:
-          ref: ${{ steps.refs.outputs.base_ref }}
+          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
       - name: get-kata-tarball
         uses: actions/download-artifact@v2
         with:
@@ -133,18 +129,14 @@ jobs:
       - name: build-and-push-kata-deploy-ci
         id: build-and-push-kata-deploy-ci
         run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          pushd $GITHUB_WORKSPACE
-          git checkout $tag
-          pkg_sha=$(git rev-parse HEAD)
-          popd
+          PR_SHA=$(git log --format=format:%H -n1)
           mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
-          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t quay.io/kata-containers/kata-deploy-ci:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
+          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t quay.io/kata-containers/kata-deploy-ci:$PR_SHA $GITHUB_WORKSPACE/tools/packaging/kata-deploy
           docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
-          docker push quay.io/kata-containers/kata-deploy-ci:$pkg_sha
+          docker push quay.io/kata-containers/kata-deploy-ci:$PR_SHA
           mkdir -p packaging/kata-deploy
           ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
-          echo "::set-output name=PKG_SHA::${pkg_sha}"
+          echo "::set-output name=PKG_SHA::${PR_SHA}"
       - name: test-kata-deploy-ci-in-aks
         uses: ./packaging/kata-deploy/action
         with:

.github/workflows/move-issues-to-in-progress.yaml

@@ -10,8 +10,6 @@ on:
     types:
       - opened
       - reopened
-      - labeled
-      - unlabeled
 
 jobs:
   move-linked-issues-to-in-progress:

.github/workflows/release.yaml

@@ -17,6 +17,7 @@ jobs:
           - rootfs-image
           - rootfs-initrd
           - shim-v2
+          - virtiofsd
     steps:
       - uses: actions/checkout@v2
       - name: Install docker
@@ -26,6 +27,7 @@ jobs:
 
       - name: Build ${{ matrix.asset }}
         run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh
           ./tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh --build="${KATA_ASSET}"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
@@ -140,13 +142,10 @@ jobs:
       - uses: actions/checkout@v2
       - name: generate-and-upload-tarball
         run: |
-          pushd $GITHUB_WORKSPACE/src/agent
-          cargo vendor >> .cargo/config
-          popd
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
           tarball="kata-containers-$tag-vendor.tar.gz"
           pushd $GITHUB_WORKSPACE
-          tar -cvzf "${tarball}" src/agent/.cargo/config src/agent/vendor
+          bash -c "tools/packaging/release/generate_vendor.sh ${tarball}"
           GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}" 
           popd
 

.github/workflows/snap.yaml

@@ -6,8 +6,6 @@ on:
       - synchronize
       - reopened
       - edited
-      - labeled
-      - unlabeled
 
 jobs:
   test:

.github/workflows/static-checks.yaml

@@ -5,8 +5,6 @@ on:
       - edited
       - reopened
       - synchronize
-      - labeled
-      - unlabeled
 
 name: Static checks
 jobs:

.gitignore

@@ -9,4 +9,6 @@ src/agent/src/version.rs
 src/agent/kata-agent.service
 src/agent/protocols/src/*.rs
 !src/agent/protocols/src/lib.rs
+build
+src/tools/log-parser/kata-log-parser
 

CONTRIBUTING.md

@@ -2,4 +2,4 @@
 
 ## This repo is part of [Kata Containers](https://katacontainers.io)
 
-For details on how to contribute to the Kata Containers project, please see the main [contributing document](https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md).
+For details on how to contribute to the Kata Containers project, please see the main [contributing document](https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md).

Makefile

@@ -14,6 +14,8 @@ TOOLS =
 
 TOOLS += agent-ctl
 TOOLS += trace-forwarder
+TOOLS += runk
+TOOLS += log-parser
 
 STANDARD_TARGETS = build check clean install test vendor
 
@@ -39,10 +41,16 @@ generate-protocols:
 static-checks: build
 	bash ci/static-checks.sh
 
+docs-url-alive-check:
+	bash ci/docs-url-alive-check.sh
+
 .PHONY: \
 	all \
 	binary-tarball \
 	default \
 	install-binary-tarball \
 	logging-crate-tests \
-	static-checks
+	static-checks \
+	docs-url-alive-check
+
+

README.md

@@ -17,16 +17,73 @@ standard implementation of lightweight Virtual Machines (VMs) that feel and
 perform like containers, but provide the workload isolation and security
 advantages of VMs.
 
+## License
+
+The code is licensed under the Apache 2.0 license.
+See [the license file](LICENSE) for further details.
+
+## Platform support
+
+Kata Containers currently runs on 64-bit systems supporting the following
+technologies:
+
+| Architecture | Virtualization technology |
+|-|-|
+| `x86_64`, `amd64` | [Intel](https://www.intel.com) VT-x, AMD SVM |
+| `aarch64` ("`arm64`")| [ARM](https://www.arm.com) Hyp |
+| `ppc64le` | [IBM](https://www.ibm.com) Power |
+| `s390x` | [IBM](https://www.ibm.com) Z & LinuxONE SIE |
+
+### Hardware requirements
+
+The [Kata Containers runtime](src/runtime) provides a command to
+determine if your host system is capable of running and creating a
+Kata Container:
+
+```bash
+$ kata-runtime check
+```
+
+> **Notes:**
+>
+> - This command runs a number of checks including connecting to the
+>   network to determine if a newer release of Kata Containers is
+>   available on GitHub. If you do not wish this to check to run, add
+>   the `--no-network-checks` option.
+>
+> - By default, only a brief success / failure message is printed.
+>   If more details are needed, the `--verbose` flag can be used to display the
+>   list of all the checks performed.
+>
+> - If the command is run as the `root` user additional checks are
+>   run (including checking if another incompatible hypervisor is running).
+>   When running as `root`, network checks are automatically disabled.
+
 ## Getting started
 
 See the [installation documentation](docs/install).
 
 ## Documentation
 
-See the [official documentation](docs)
-(including [installation guides](docs/install),
-[the developer guide](docs/Developer-Guide.md),
-[design documents](docs/design) and more).
+See the [official documentation](docs) including:
+
+- [Installation guides](docs/install)
+- [Developer guide](docs/Developer-Guide.md)
+- [Design documents](docs/design)
+  - [Architecture overview](docs/design/architecture)
+
+## Configuration
+
+Kata Containers uses a single
+[configuration file](src/runtime/README.md#configuration)
+which contains a number of sections for various parts of the Kata
+Containers system including the [runtime](src/runtime), the
+[agent](src/agent) and the [hypervisor](#hypervisors).
+
+## Hypervisors
+
+See the [hypervisors document](docs/hypervisors.md) and the
+[Hypervisor specific configuration details](src/runtime/README.md#hypervisor-specific-configuration).
 
 ## Community
 
@@ -48,6 +105,8 @@ Please raise an issue
 
 ## Developers
 
+See the [developer guide](docs/Developer-Guide.md).
+
 ### Components
 
 ### Main components
@@ -59,6 +118,7 @@ The table below lists the core parts of the project:
 | [runtime](src/runtime) | core | Main component run by a container manager and providing a containerd shimv2 runtime implementation. |
 | [agent](src/agent) | core | Management process running inside the virtual machine / POD that sets up the container environment. |
 | [documentation](docs) | documentation | Documentation common to all components (such as design and install documentation). |
+| [libraries](src/libs) | core | Library crates shared by multiple Kata Container components or published to [`crates.io`](https://crates.io/index.html) |
 | [tests](https://github.com/kata-containers/tests) | tests | Excludes unit tests which live with the main code. |
 
 ### Additional components
@@ -72,6 +132,7 @@ The table below lists the remaining parts of the project:
 | [osbuilder](tools/osbuilder) | infrastructure | Tool to create "mini O/S" rootfs and initrd images and kernel for the hypervisor. |
 | [`agent-ctl`](src/tools/agent-ctl) | utility | Tool that provides low-level access for testing the agent. |
 | [`trace-forwarder`](src/tools/trace-forwarder) | utility | Agent tracing helper. |
+| [`runk`](src/tools/runk) | utility | Standard OCI container runtime based on the agent. |
 | [`ci`](https://github.com/kata-containers/ci) | CI | Continuous Integration configuration files and scripts. |
 | [`katacontainers.io`](https://github.com/kata-containers/www.katacontainers.io) | Source for the [`katacontainers.io`](https://www.katacontainers.io) site. |
 
@@ -79,13 +140,9 @@ The table below lists the remaining parts of the project:
 
 Kata Containers is now
 [available natively for most distributions](docs/install/README.md#packaged-installation-methods).
-However, packaging scripts and metadata are still used to generate snap and GitHub releases. See
+However, packaging scripts and metadata are still used to generate [snap](snap/local) and GitHub releases. See
 the [components](#components) section for further details.
 
 ## Glossary of Terms
 
-See the [glossary of terms](Glossary.md) related to Kata Containers.
----
-
-[kernel]: https://www.kernel.org
-[github-katacontainers.io]: https://github.com/kata-containers/www.katacontainers.io
+See the [glossary of terms](https://github.com/kata-containers/kata-containers/wiki/Glossary) related to Kata Containers.

VERSION

@@ -1 +1 @@
-2.4.0-alpha2
+2.5.0-alpha2

ci/darwin-test.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2022 Apple Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+cidir=$(dirname "$0")
+runtimedir=$cidir/../src/runtime
+
+build_working_packages() {
+	# working packages:
+	device_api=$runtimedir/virtcontainers/device/api
+	device_config=$runtimedir/virtcontainers/device/config
+	device_drivers=$runtimedir/virtcontainers/device/drivers
+	device_manager=$runtimedir/virtcontainers/device/manager
+	rc_pkg_dir=$runtimedir/pkg/resourcecontrol/
+	utils_pkg_dir=$runtimedir/virtcontainers/utils
+
+	# broken packages :( :
+	#katautils=$runtimedir/pkg/katautils
+	#oci=$runtimedir/pkg/oci
+	#vc=$runtimedir/virtcontainers
+
+	pkgs=(
+		"$device_api"
+		"$device_config"
+		"$device_drivers"
+		"$device_manager"
+		"$utils_pkg_dir"
+		"$rc_pkg_dir")
+	for pkg in "${pkgs[@]}"; do
+		echo building "$pkg"
+		pushd "$pkg" &>/dev/null
+		go build
+		go test
+		popd &>/dev/null
+	done
+}
+
+build_working_packages

ci/docs-url-alive-check.sh

@@ -1,6 +1,6 @@
-#!/usr/bin/env bash
+#!/bin/bash
 #
-# Copyright (c) 2020 Intel Corporation
+# Copyright (c) 2021 Easystack Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 
@@ -9,4 +9,4 @@ set -e
 cidir=$(dirname "$0")
 source "${cidir}/lib.sh"
 
-run_go_test
+run_docs_url_alive_check

ci/install_libseccomp.sh

@@ -19,7 +19,7 @@ source "${tests_repo_dir}/.ci/lib.sh"
 # fail. So let's ensure they are unset here.
 unset PREFIX DESTDIR
 
-arch=$(uname -m)
+arch=${ARCH:-$(uname -m)}
 workdir="$(mktemp -d --tmpdir build-libseccomp.XXXXX)"
 
 # Variables for libseccomp
@@ -70,7 +70,8 @@ build_and_install_gperf() {
     curl -sLO "${gperf_tarball_url}"
     tar -xf "${gperf_tarball}"
     pushd "gperf-${gperf_version}"
-    ./configure --prefix="${gperf_install_dir}"
+    # Unset $CC for configure, we will always use native for gperf
+    CC= ./configure --prefix="${gperf_install_dir}"
     make
     make install
     export PATH=$PATH:"${gperf_install_dir}"/bin
@@ -84,7 +85,7 @@ build_and_install_libseccomp() {
     curl -sLO "${libseccomp_tarball_url}"
     tar -xf "${libseccomp_tarball}"
     pushd "libseccomp-${libseccomp_version}"
-    ./configure --prefix="${libseccomp_install_dir}" CFLAGS="${cflags}" --enable-static
+    ./configure --prefix="${libseccomp_install_dir}" CFLAGS="${cflags}" --enable-static --host="${arch}"
     make
     make install
     popd

ci/install_musl.sh

@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-# Copyright (c) 2020 Ant Group
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-set -e
-
-install_aarch64_musl() {
-	local arch=$(uname -m)
-	if [ "${arch}" == "aarch64" ]; then
-		local musl_tar="${arch}-linux-musl-native.tgz"
-		local musl_dir="${arch}-linux-musl-native"
-		pushd /tmp
-		if curl -sLO --fail https://musl.cc/${musl_tar}; then
-			tar -zxf ${musl_tar}
-			mkdir -p /usr/local/musl/
-			cp -r ${musl_dir}/* /usr/local/musl/
-		fi
-		popd
-	fi
-}
-
-install_aarch64_musl

ci/lib.sh

@@ -39,8 +39,11 @@ run_static_checks()
 	bash "$tests_repo_dir/.ci/static-checks.sh" "$@"
 }
 
-run_go_test()
+run_docs_url_alive_check()
 {
 	clone_tests_repo
-	bash "$tests_repo_dir/.ci/go-test.sh"
+	# Make sure we have the targeting branch
+	git remote set-branches --add origin "${branch}"
+	git fetch -a
+	bash "$tests_repo_dir/.ci/static-checks.sh" --docs --all "github.com/kata-containers/kata-containers"
 }

ci/openshift-ci/images/Dockerfile.buildroot

@@ -4,7 +4,7 @@
 #
 # This is the build root image for Kata Containers on OpenShift CI.
 #
-FROM registry.centos.org/centos:8
+FROM quay.io/centos/centos:stream8
 
 RUN yum -y update && \
     yum -y install \

docs/Developer-Guide.md

@@ -212,11 +212,13 @@ $ sudo systemctl restart systemd-journald
 >
 > - You should only do this step if you are testing with the latest version of the agent.
 
-The rust-agent is built with a static linked `musl.` To configure this:
+The agent is built with a statically linked `musl.` The default `libc` used is `musl`, but on `ppc64le` and `s390x`, `gnu` should be used. To configure this:
 
 ```
-rustup target add x86_64-unknown-linux-musl
-sudo ln -s /usr/bin/g++ /bin/musl-g++
+$ export ARCH=$(uname -m)
+$ if [ "$ARCH" = "ppc64le" -o "$ARCH" = "s390x" ]; then export LIBC=gnu; else export LIBC=musl; fi
+$ [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
+$ rustup target add ${ARCH}-unknown-linux-${LIBC}
 ```
 
 To build the agent:
@@ -463,7 +465,7 @@ script and paste its output directly into a
 > [runtime](../src/runtime) repository.
 
 To perform analysis on Kata logs, use the
-[`kata-log-parser`](https://github.com/kata-containers/tests/tree/main/cmd/log-parser)
+[`kata-log-parser`](../src/tools/log-parser)
 tool, which can convert the logs into formats (e.g. JSON, TOML, XML, and YAML).
 
 See [Set up a debug console](#set-up-a-debug-console).
@@ -698,11 +700,11 @@ options to have the kernel boot messages logged into the system journal.
 For generic information on enabling debug in the configuration file, see the
 [Enable full debug](#enable-full-debug) section.
 
-The kernel boot messages will appear in the `containerd` or `CRI-O` log appropriately,
+The kernel boot messages will appear in the `kata` logs (and in the `containerd` or `CRI-O` log appropriately).
 such as:
 
 ```bash
-$ sudo journalctl -t containerd
+$ sudo journalctl -t kata
 -- Logs begin at Thu 2020-02-13 16:20:40 UTC, end at Thu 2020-02-13 16:30:23 UTC. --
 ...
 time="2020-09-15T14:56:23.095113803+08:00" level=debug msg="reading guest console" console-protocol=unix console-url=/run/vc/vm/ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791/console.sock pid=107642 sandbox=ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791 source=virtcontainers subsystem=sandbox vmconsole="[    0.395399] brd: module loaded"
@@ -712,3 +714,4 @@ time="2020-09-15T14:56:23.105268162+08:00" level=debug msg="reading guest consol
 time="2020-09-15T14:56:23.121121598+08:00" level=debug msg="reading guest console" console-protocol=unix console-url=/run/vc/vm/ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791/console.sock pid=107642 sandbox=ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791 source=virtcontainers subsystem=sandbox vmconsole="[    0.421324] memmap_init_zone_device initialised 32768 pages in 12ms"
 ...
 ```
+Refer to the [kata-log-parser documentation](../src/tools/log-parser/README.md) which is useful to fetch these.

docs/Limitations.md

@@ -46,7 +46,7 @@ The following link shows the latest list of limitations:
 # Contributing
 
 If you would like to work on resolving a limitation, please refer to the
-[contributors guide](https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md).
+[contributors guide](https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md).
 If you wish to raise an issue for a new limitation, either
 [raise an issue directly on the runtime](https://github.com/kata-containers/kata-containers/issues/new)
 or see the
@@ -57,6 +57,13 @@ for advice on which repository to raise the issue against.
 
 This section lists items that might be possible to fix.
 
+## OCI CLI commands
+
+### Docker and Podman support
+Currently Kata Containers does not support Docker or Podman.
+
+See issue https://github.com/kata-containers/kata-containers/issues/722 for more information.
+
 ## Runtime commands
 
 ### checkpoint and restore
@@ -97,57 +104,12 @@ See issue https://github.com/clearcontainers/runtime/issues/341 and [the constra
 For CPUs resource management see
 [CPU constraints](design/vcpu-handling.md).
 
-### docker run and shared memory
-
-The runtime does not implement the `docker run --shm-size` command to
-set the size of the `/dev/shm tmpfs` within the container. It is possible to pass this configuration value into the VM container so the appropriate mount command happens at launch time.
-
-See issue https://github.com/kata-containers/kata-containers/issues/21 for more information.
-
 # Architectural limitations
 
 This section lists items that might not be fixed due to fundamental
 architectural differences between "soft containers" (i.e. traditional Linux*
 containers) and those based on VMs.
 
-## Networking limitations
-
-### Support for joining an existing VM network
-
-Docker supports the ability for containers to join another containers
-namespace with the `docker run --net=containers` syntax. This allows
-multiple containers to share a common network namespace and the network
-interfaces placed in the network namespace.  Kata Containers does not
-support network namespace sharing. If a Kata Container is setup to
-share the network namespace of a `runc` container, the runtime
-effectively takes over all the network interfaces assigned to the
-namespace and binds them to the VM. Consequently, the `runc` container loses
-its network connectivity.
-
-### docker --net=host
-
-Docker host network support (`docker --net=host run`) is not supported.
-It is not possible to directly access the host networking configuration
-from within the VM.
-
-The `--net=host` option can still be used with `runc` containers and
-inter-mixed with running Kata Containers, thus enabling use of `--net=host`
-when necessary.
-
-It should be noted, currently passing the `--net=host` option into a
-Kata Container may result in the Kata Container networking setup
-modifying, re-configuring and therefore possibly breaking the host
-networking setup. Do not use `--net=host` with Kata Containers.
-
-### docker run --link
-
-The runtime does not support the `docker run --link` command. This
-command is now deprecated by docker and we have no intention of adding support.
-Equivalent functionality can be achieved with the newer docker networking commands.
-
-See more documentation at
-[docs.docker.com](https://docs.docker.com/engine/userguide/networking/default_network/dockerlinks/).
-
 ## Storage limitations
 
 ### Kubernetes `volumeMounts.subPaths`
@@ -158,15 +120,11 @@ moment.
 See [this issue](https://github.com/kata-containers/runtime/issues/2812) for more details.
 [Another issue](https://github.com/kata-containers/kata-containers/issues/1728) focuses on the case of `emptyDir`.
 
-
 ## Host resource sharing
 
-### docker run --privileged
+### Privileged containers
 
 Privileged support in Kata is essentially different from `runc` containers.
-Kata does support `docker run --privileged` command, but in this case full access
-to the guest VM is provided in addition to some host access.
-
 The container runs with elevated capabilities within the guest and is granted
 access to guest devices instead of the host devices.
 This is also true with using `securityContext privileged=true` with Kubernetes.
@@ -176,17 +134,6 @@ The container may also be granted full access to a subset of host devices
 
 See [Privileged Kata Containers](how-to/privileged.md) for how to configure some of this behavior.
 
-# Miscellaneous
-
-This section lists limitations where the possible solutions are uncertain.
-
-## Docker --security-opt option partially supported
-
-The `--security-opt=` option used by Docker is partially supported.
-We only support `--security-opt=no-new-privileges` and `--security-opt seccomp=/path/to/seccomp/profile.json`
-option as of today.
-
-Note: The `--security-opt apparmor=your_profile` is not yet supported. See https://github.com/kata-containers/runtime/issues/707.
 # Appendices
 
 ## The constraints challenge

docs/README.md

@@ -21,17 +21,15 @@ See the [tracing documentation](tracing.md).
 * [Limitations](Limitations.md): differences and limitations compared with the default [Docker](https://www.docker.com/) runtime,
 [`runc`](https://github.com/opencontainers/runc).
 
-### Howto guides
+### How-to guides
 
-See the [howto documentation](how-to).
+See the [how-to documentation](how-to).
 
 ## Kata Use-Cases
 
 * [GPU Passthrough with Kata](./use-cases/GPU-passthrough-and-Kata.md)
-* [OpenStack Zun with Kata Containers](./use-cases/zun_kata.md)
 * [SR-IOV with Kata](./use-cases/using-SRIOV-and-kata.md)
 * [Intel QAT with Kata](./use-cases/using-Intel-QAT-and-kata.md)
-* [VPP with Kata](./use-cases/using-vpp-and-kata.md)
 * [SPDK vhost-user with Kata](./use-cases/using-SPDK-vhostuser-and-kata.md)
 * [Intel SGX with Kata](./use-cases/using-Intel-SGX-and-kata.md)
 
@@ -49,7 +47,7 @@ Documents that help to understand and contribute to Kata Containers.
 ### How to Contribute
 
 * [Developer Guide](Developer-Guide.md): Setup the Kata Containers developing environments
-* [How to contribute to Kata Containers](https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md)
+* [How to contribute to Kata Containers](https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md)
 * [Code of Conduct](../CODE_OF_CONDUCT.md)
 
 ## Help Writing a Code PR
@@ -59,6 +57,7 @@ Documents that help to understand and contribute to Kata Containers.
 ## Help Writing Unit Tests
 
 * [Unit Test Advice](Unit-Test-Advice.md)
+* [Unit testing presentation](presentations/unit-testing/kata-containers-unit-testing.md)
 
 ## Help Improving the Documents
 
@@ -73,6 +72,10 @@ Documents that help to understand and contribute to Kata Containers.
 * [Release strategy](Stable-Branch-Strategy.md)
 * [Release Process](Release-Process.md)
 
+## Presentations
+
+* [Presentations](presentations)
+
 ## Website Changes
 
 If you have a suggestion for how we can improve the

docs/Release-Process.md

@@ -4,11 +4,11 @@
 ## Requirements
 
 - [hub](https://github.com/github/hub)
-  * Using an [application token](https://github.com/settings/tokens) is required for hub.
+  * Using an [application token](https://github.com/settings/tokens) is required for hub (set to a GITHUB_TOKEN environment variable).
 
 - GitHub permissions to push tags and create releases in Kata repositories.
 
-- GPG configured to sign git tags. https://help.github.com/articles/generating-a-new-gpg-key/
+- GPG configured to sign git tags. https://docs.github.com/en/authentication/managing-commit-signature-verification/generating-a-new-gpg-key
 
 - You should configure your GitHub to use your ssh keys (to push to branches). See https://help.github.com/articles/adding-a-new-ssh-key-to-your-github-account/.
     * As an alternative, configure hub to push and fork with HTTPS, `git config --global hub.protocol https` (Not tested yet) *
@@ -48,6 +48,7 @@
 ### Merge all bump version Pull requests
 
   - The above step will create a GitHub pull request in the Kata projects. Trigger the CI using `/test` command on each bump Pull request.
+  - Trigger the `test-kata-deploy` workflow which is under the `Actions` tab on the repository GitHub page (make sure to select the correct branch and validate it passes).
   - Check any failures and fix if needed.
   - Work with the Kata approvers to verify that the CI works and the pull requests are merged.
 
@@ -64,7 +65,7 @@
 
 ### Check Git-hub Actions
 
-  We make use of [GitHub actions](https://github.com/features/actions) in this [file](https://github.com/kata-containers/kata-containers/blob/main/.github/workflows/release.yaml) in the `kata-containers/kata-containers` repository to build and upload release artifacts. This action is auto triggered with the above step when a new tag is pushed to the `kata-containers/kata-containers` repository.
+  We make use of [GitHub actions](https://github.com/features/actions) in this [file](../.github/workflows/release.yaml) in the `kata-containers/kata-containers` repository to build and upload release artifacts. This action is auto triggered with the above step when a new tag is pushed to the `kata-containers/kata-containers` repository.
 
   Check the [actions status page](https://github.com/kata-containers/kata-containers/actions) to verify all steps in the actions workflow have completed successfully. On success, a static tarball containing Kata release artifacts will be uploaded to the [Release page](https://github.com/kata-containers/kata-containers/releases).
 

docs/Unit-Test-Advice.md

@@ -277,7 +277,9 @@ mod tests {
 
 ## Temporary files
 
-Always delete temporary files on success.
+Use `t.TempDir()` to create temporary directory. The directory created by
+`t.TempDir()` is automatically removed when the test and all its subtests
+complete.
 
 ### Golang temporary files
 
@@ -286,11 +288,7 @@ func TestSomething(t *testing.T) {
     assert := assert.New(t)
 
     // Create a temporary directory
-    tmpdir, err := os.MkdirTemp("", "")
-    assert.NoError(err)
-
-    // Delete it at the end of the test
-    defer os.RemoveAll(tmpdir) 
+    tmpdir := t.TempDir()
 
     // Add test logic that will use the tmpdir here...
 }
@@ -337,7 +335,7 @@ will run if the correct type of user is detected and skipped if not.
 
 The main repository has the most comprehensive set of skip abilities. See:
 
-- https://github.com/kata-containers/kata-containers/tree/main/src/runtime/pkg/katatestutils
+- [`katatestutils`](../src/runtime/pkg/katatestutils)
 
 ### Run Rust tests as a different user
 

docs/code-pr-advice.md

@@ -154,7 +154,7 @@ func testFoo() error {
 ### Tracing
 
 Consider if the code needs to create a new
-[trace span](https://github.com/kata-containers/kata-containers/blob/main/docs/tracing.md).
+[trace span](./tracing.md).
 
 Ensure any new trace spans added to the code are completed.
 

docs/design/README.md

@@ -10,7 +10,9 @@ Kata Containers design documents:
 - [Host cgroups](host-cgroups.md)
 - [`Inotify` support](inotify.md)
 - [Metrics(Kata 2.0)](kata-2-0-metrics.md)
-
+- [Design for Kata Containers `Lazyload` ability with `nydus`](kata-nydus-design.md)
+- [Design for direct-assigned volume](direct-blk-device-assignment.md)
+- [Design for core-scheduling](core-scheduling.md)
 ---
 
 - [Design proposals](proposals)

docs/design/VSocks.md

@@ -67,22 +67,15 @@ Using a proxy for multiplexing the connections between the VM and the host uses
 4.5MB per [POD][2]. In a high density deployment this could add up to GBs of
 memory that could have been used to host more PODs. When we talk about density
 each kilobyte matters and it might be the decisive factor between run another
-POD or not. For example if you have 500 PODs running in a server, the same
-amount of [`kata-proxy`][3] processes will be running and consuming for around
-2250MB of RAM. Before making the decision not to use VSOCKs, you should ask
+POD or not. Before making the decision not to use VSOCKs, you should ask
 yourself, how many more containers can run with the memory RAM consumed by the
 Kata proxies?
 
 ### Reliability
 
-[`kata-proxy`][3] is in charge of multiplexing the connections between virtual
-machine and host processes, if it dies all connections get broken. For example
-if you have a [POD][2] with 10 containers running, if `kata-proxy` dies it would
-be impossible to contact your containers, though they would still be running.
 Since communication via VSOCKs is direct, the only way to lose communication
 with the containers is if the VM itself or the `containerd-shim-kata-v2` dies, if this happens
 the containers are removed automatically.
 
 [1]: https://wiki.qemu.org/Features/VirtioVsock
 [2]: ./vcpu-handling.md#virtual-cpus-and-kubernetes-pods
-[3]: https://github.com/kata-containers/proxy

docs/design/arch-images/kata-nydus.drawio

@@ -0,0 +1 @@
+<mxfile host="app.diagrams.net" modified="2022-01-18T14:06:01.890Z" agent="5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36" etag="nId-8OV6FDjWTDgzqDu-" version="15.8.9" type="device"><diagram id="bkF_ZONM9sPFCpIYoGFl" name="Page-1">5Vtbj6M2GP01eUyEbW55nGSmM2q70mqnUnf6UrnBCdYSnIIzSfbX14AJYDsLSSDZUWdHGvxhDD7n8F1sdoTm6/1zgjfhJxaQaAStYD9CjyMIgQ1d8SezHAqL504LwyqhgexUGV7pdyKNlrRuaUDSRkfOWMTppmlcsDgmC96w4SRhu2a3JYuad93gFdEMrwsc6dY/acDDwupDr7K/ELoKyzuDcn5rXHaWM0lDHLBdzYSeRmieMMaLo/V+TqIMvBKX4rpfTpw9PlhCYt7lgpd4/J1SSP9++bR8Cb6A1de/XsZ2Mco7jrZywiFLuXxgfihR4GQv7jEL+ToSBiAOU56wb2TOIpYIS8xi0XO2pFGkmHBEV7FoLsRTEmGfvZOEU4HvgzyxpkGQ3Wa2Cyknrxu8yO65E2oStoRt44BkE7BES5+xBCEbk+xrJonAM2FrwpOD6CLP+o5kQ8rRsZ2ivavItWWXsMZrSSKWclodR64QFwcSdDMB8ycYj+f+Hw/+b3jxvN57e/vXsa8RoIHfBMEEU42XJYu5fIugfeSplC5QSBpBtHSyf/LKmr340ZgWZ9z858iHBr6BopN8INDkAwGdj6llIMSxh2JkamDEjbhEqEGN+++WlSfGaY76g+gA3c2+OimOVtnf+BBs03Ea400aMp69DHJY8ZTFyEW/H/AP+uC/D9aQNbFAkzjDiwQ8A3H+ULyVSrqCOARNxInQwjGNSRIMzth0OMacCYJN14csnTFnOkG+Tpo3GGnAQJqCJomDhyySZ1EkwmlKFzlKOOG6uYZr023WUBYTRDOBW3L4mp2cOGXzTV6ZNx738sqidWjEIBJoWYMWlFK2TRakg2DFTFaEt3kkndoab47JQ0pbQiLM6XvzeU1Eyjt8ZjR/W0rluErELD10OUQxT3lVPf9QBrIVV2+7ykAFDtpAua6O075Cauh6x97iH8ZpSNfjb5jj8TscxFn04Aocx2n3A65BUMM5AT0L7c+lwqFcqg8UHKEeAVGJdSOXdAYD0rle4tOTucvw4W8wrhyvyZU7NWQr0KB5dzCq3OupMqaZufcRVWnOzwfNVnxbiTlTg4tCP4h5/dPlXZin1KA7phxjkT3DRtZhTbxj+0Tikbc+k4SKCWWFdGHcU/61HF4cv1UJjWhVI2WNITIYdM/MxIOKStSEomtmosrNVVOcoTOTDosAncWl5LNWm6ykgirVvNX0dCMFdciBC0ruJjWkKAReKjWnZaCBpQZNRfLFUmu6sFYPdmdn1bXcuq9Xc1WFqClIV6mpA3nWjaV2aWlfl9oFkql5QgvYTYkC95Ioexd/Z/9MoVWLiJ39HWiJ0UOLEBpEeF6aDXxTmr3akrRzhv0zbZ9cl5grcdBxJL732j6BpqWDM/k1llHFNthHordZifn9EA6A4gmQYZXjtozraxxzoFFyaU2bB4hBalpggROpX1tRO9gaBNTXILLt6GX6IeH0O8KJBoNTXyOg6+zzAhGOPw6sSi3sGTZkgWlDdjhYTdXxmS7eMbn4NBSwBDQZZJ2s9OwRWfJ+qJmq+bxxq/yGxKAOteStNzc0t2BC6aZeodx1/d/LV0kdfeve8jXtB95ZvtNpO0i3VW+Hrbm2Iv70RjysL0DWS/xbrQkVL+e9qmzfP8H2uVW2Fhrs21bZyLTv2K9KykWd4wJkvx9rtK7HFFnIvZQCLNiXVFxVKt7kxmLRq47yo7g8mpmL63Mrahm4TtbTqXDjNF79nnd7tCvLF0leZmLi8mWUazYUFxIxwmyT4ZIj5czEr0Bznq1IOuJZ56INqrb4zbonfM5i8fiY5pojOOW7bO0okzzHHP+Tz1Sv4HvLiFzHLJ2adD3DZwrDxZRet7vO24MIcBoe43mP7qEQ9f3cg6VwrC6/dHUP6kYXALA//yCa1efuRffqPw2gp/8A</diagram></mxfile>

docs/design/architecture/storage.md

@@ -20,7 +20,7 @@ For virtio-fs, the [runtime](README.md#runtime) starts one `virtiofsd` daemon
 ## Devicemapper
 
 The
-[devicemapper `snapshotter`](https://github.com/containerd/containerd/tree/master/snapshots/devmapper)
+[devicemapper `snapshotter`](https://github.com/containerd/containerd/tree/main/snapshots/devmapper)
 is a special case. The `snapshotter` uses dedicated block devices
 rather than formatted filesystems, and operates at the block level
 rather than the file level. This knowledge is used to directly use the

docs/design/core-scheduling.md

@@ -0,0 +1,12 @@
+# Core scheduling
+
+Core scheduling is a Linux kernel feature that allows only trusted tasks to run concurrently on
+CPUs sharing compute resources (for example, hyper-threads on a core).
+
+Containerd versions >= 1.6.4 leverage this to treat all of the processes associated with a
+given pod or container to be a single group of trusted tasks. To indicate this should be carried
+out, containerd sets the `SCHED_CORE` environment variable for each shim it spawns. When this is
+set, the Kata Containers shim implementation uses the `prctl` syscall to create a new core scheduling
+domain for the shim process itself as well as future VMM processes it will start.
+
+For more details on the core scheduling feature, see the [Linux documentation](https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/core-scheduling.html).

docs/design/direct-blk-device-assignment.md

@@ -0,0 +1,253 @@
+# Motivation
+Today, there exist a few gaps between Container Storage Interface (CSI) and virtual machine (VM) based runtimes such as Kata Containers 
+that prevent them from working together smoothly.
+
+First, it’s cumbersome to use a persistent volume (PV) with Kata Containers. Today, for a PV with Filesystem volume mode, Virtio-fs
+is the only way to surface it inside a Kata Container guest VM. But often mounting the filesystem (FS) within the guest operating system (OS) is 
+desired due to performance benefits, availability of native FS features and security benefits over the Virtio-fs mechanism.
+
+Second, it’s difficult if not impossible to resize a PV online with Kata Containers. While a PV can be expanded on the host OS, 
+the updated metadata needs to be propagated to the guest OS in order for the application container to use the expanded volume. 
+Currently, there is not a way to propagate the PV metadata from the host OS to the guest OS without restarting the Pod sandbox.
+
+# Proposed Solution
+
+Because of the OS boundary, these features cannot be implemented in the CSI node driver plugin running on the host OS 
+as is normally done in the runc container. Instead, they can be done by the Kata Containers agent inside the guest OS, 
+but it requires the CSI driver to pass the relevant information to the Kata Containers runtime. 
+An ideal long term solution would be to have the `kubelet` coordinating the communication between the CSI driver and 
+the container runtime, as described in [KEP-2857](https://github.com/kubernetes/enhancements/pull/2893/files). 
+However, as the KEP is still under review, we would like to propose a short/medium term solution to unblock our use case.
+
+The proposed solution is built on top of a previous [proposal](https://github.com/egernst/kata-containers/blob/da-proposal/docs/design/direct-assign-volume.md) 
+described by Eric Ernst. The previous proposal has two gaps:
+
+1. Writing a `csiPlugin.json` file to the volume root path introduced a security risk. A malicious user can gain unauthorized 
+access to a block device by writing their own `csiPlugin.json` to the above location through an ephemeral CSI plugin.  
+
+2. The proposal didn't describe how to establish a mapping between a volume and a kata sandbox, which is needed for 
+implementing CSI volume resize and volume stat collection APIs.
+
+This document particularly focuses on how to address these two gaps.
+
+## Assumptions and Limitations
+1. The proposal assumes that a block device volume will only be used by one Pod on a node at a time, which we believe 
+is the most common pattern in Kata Containers use cases. It’s also unsafe to have the same block device attached to more than 
+one Kata pod. In the context of Kubernetes, the `PersistentVolumeClaim` (PVC) needs to have the `accessMode` as `ReadWriteOncePod`. 
+2. More advanced Kubernetes volume features such as, `fsGroup`, `fsGroupChangePolicy`, and `subPath` are not supported. 
+
+## End User Interface
+
+1. The user specifies a PV as a direct-assigned volume. How a PV is specified as a direct-assigned volume is left for each CSI implementation to decide.
+There are a few options for reference:
+   1. A storage class parameter specifies whether it's a direct-assigned volume. This avoids any lookups of PVC 
+   or Pod information from the CSI plugin (as external provisioner takes care of these). However, all PVs in the storage class with the parameter set 
+   will have host mounts skipped.
+   2. Use a PVC annotation. This approach requires the CSI plugins have `--extra-create-metadata` [set](https://kubernetes-csi.github.io/docs/external-provisioner.html#persistentvolumeclaim-and-persistentvolume-parameters)
+   to be able to perform a lookup of the PVC annotations from the API server. Pro: API server lookup of annotations only required during creation of PV. 
+   Con: The CSI plugin will always skip host mounting of the PV.
+   3. The CSI plugin can also lookup pod `runtimeclass` during `NodePublish`. This approach can be found in the [ALIBABA CSI plugin](https://github.com/kubernetes-sigs/alibaba-cloud-csi-driver/blob/master/pkg/disk/nodeserver.go#L248).
+2. The CSI node driver delegates the direct assigned volume to the Kata Containers runtime. The CSI node driver APIs need to 
+   be modified to pass the volume mount information and collect volume information to/from the Kata Containers runtime by invoking `kata-runtime` command line commands.
+   * **NodePublishVolume** -- It invokes `kata-runtime direct-volume add --volume-path [volumePath] --mount-info [mountInfo]` 
+   to propagate the volume mount information to the Kata Containers runtime for it to carry out the filesystem mount operation.
+   The `volumePath` is the [target_path](https://github.com/container-storage-interface/spec/blob/master/csi.proto#L1364) in the CSI `NodePublishVolumeRequest`.
+   The `mountInfo` is a serialized JSON string. 
+   * **NodeGetVolumeStats** -- It invokes `kata-runtime direct-volume stats --volume-path [volumePath]` to retrieve the filesystem stats of direct-assigned volume.
+   * **NodeExpandVolume** -- It invokes `kata-runtime direct-volume resize --volume-path [volumePath] --size [size]` to send a resize request to the Kata Containers runtime to
+   resize the direct-assigned volume.
+   * **NodeStageVolume/NodeUnStageVolume** -- It invokes `kata-runtime direct-volume remove --volume-path [volumePath]` to remove the persisted metadata of a direct-assigned volume.
+
+The `mountInfo` object is defined as follows:
+```Golang
+type MountInfo struct {
+    // The type of the volume (ie. block)
+    VolumeType string `json:"volume-type"`
+    // The device backing the volume.
+    Device string `json:"device"`
+    // The filesystem type to be mounted on the volume.
+    FsType string `json:"fstype"`
+    // Additional metadata to pass to the agent regarding this volume.
+    Metadata map[string]string `json:"metadata,omitempty"`
+    // Additional mount options.
+    Options []string `json:"options,omitempty"`
+}
+```
+Notes: given that the `mountInfo` is persisted to the disk by the Kata runtime, it shouldn't container any secrets (such as SMB mount password).
+
+## Implementation Details
+
+### Kata runtime
+Instead of the CSI node driver writing the mount info into a `csiPlugin.json` file under the volume root, 
+as described in the original proposal, here we propose that the CSI node driver passes the mount information to 
+the Kata Containers runtime through a new `kata-runtime` commandline command. The `kata-runtime` then writes the mount 
+information to a `mount-info.json` file in a predefined location (`/run/kata-containers/shared/direct-volumes/[volume_path]/`).
+
+When the Kata Containers runtime starts a container, it verifies whether a volume mount is a direct-assigned volume by checking 
+whether there is a `mountInfo` file under the computed Kata `direct-volumes` directory. If it is, the runtime parses the `mountInfo` file, 
+updates the mount spec with the data in `mountInfo`. The updated mount spec is then passed to the Kata agent in the guest VM together
+with other mounts. The Kata Containers runtime also creates a file named by the sandbox id under the `direct-volumes/[volume_path]/` 
+directory. The reason for adding a sandbox id file is to establish a mapping between the volume and the sandbox using it. 
+Later, when the Kata Containers runtime handles the `get-stats` and `resize` commands, it uses the sandbox id to identify 
+the endpoint of the corresponding `containerd-shim-kata-v2`.
+
+### containerd-shim-kata-v2 changes
+`containerd-shim-kata-v2` provides an API for sandbox management through a Unix domain socket. Two new handlers are proposed: `/direct-volume/stats` and `/direct-volume/resize`:
+
+Example:
+
+```bash
+$ curl --unix-socket "$shim_socket_path" -I -X GET 'http://localhost/direct-volume/stats/[urlSafeVolumePath]'
+$ curl --unix-socket "$shim_socket_path" -I -X POST 'http://localhost/direct-volume/resize' -d '{ "volumePath"": [volumePath], "Size": "123123" }'
+```
+
+The shim then forwards the corresponding request to the `kata-agent` to carry out the operations inside the guest VM. For `resize` operation, 
+the Kata runtime also needs to notify the hypervisor to resize the block device (e.g. call `block_resize` in QEMU). 
+
+### Kata agent changes
+
+The mount spec of a direct-assigned volume is passed to `kata-agent` through the existing `Storage` GRPC object. 
+Two new APIs and three new GRPC objects are added to GRPC protocol between the shim and agent for resizing and getting volume stats:
+```protobuf
+
+rpc GetVolumeStats(VolumeStatsRequest) returns (VolumeStatsResponse);
+rpc ResizeVolume(ResizeVolumeRequest) returns (google.protobuf.Empty);
+
+message VolumeStatsRequest {
+// The volume path on the guest outside the container
+    string volume_guest_path = 1;
+}
+
+message ResizeVolumeRequest {
+// Full VM guest path of the volume (outside the container)
+    string volume_guest_path = 1;
+    uint64 size = 2;
+}
+
+// This should be kept in sync with CSI NodeGetVolumeStatsResponse (https://github.com/container-storage-interface/spec/blob/v1.5.0/csi.proto)
+message VolumeStatsResponse {
+   // This field is OPTIONAL.
+   repeated VolumeUsage usage = 1;
+   // Information about the current condition of the volume.
+   // This field is OPTIONAL.
+   // This field MUST be specified if the VOLUME_CONDITION node
+   // capability is supported.
+   VolumeCondition volume_condition = 2;
+}
+message VolumeUsage {
+   enum Unit {
+      UNKNOWN = 0;
+      BYTES = 1;
+      INODES = 2;
+   }
+   // The available capacity in specified Unit. This field is OPTIONAL.
+   // The value of this field MUST NOT be negative.
+   uint64 available = 1;
+
+   // The total capacity in specified Unit. This field is REQUIRED.
+   // The value of this field MUST NOT be negative.
+   uint64 total = 2;
+
+   // The used capacity in specified Unit. This field is OPTIONAL.
+   // The value of this field MUST NOT be negative.
+   uint64 used = 3;
+
+   // Units by which values are measured. This field is REQUIRED.
+   Unit unit = 4;
+}
+
+// VolumeCondition represents the current condition of a volume.
+message VolumeCondition {
+
+   // Normal volumes are available for use and operating optimally.
+   // An abnormal volume does not meet these criteria.
+   // This field is REQUIRED.
+   bool abnormal = 1;
+
+   // The message describing the condition of the volume.
+   // This field is REQUIRED.
+   string message = 2;
+}
+
+```
+
+### Step by step walk-through
+
+Given the following definition:
+```YAML
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: app
+spec:
+  runtime-class: kata-qemu
+  containers:
+  - name: app
+    image: centos
+    command: ["/bin/sh"]
+    args: ["-c", "while true; do echo $(date -u) >> /data/out.txt; sleep 5; done"]
+    volumeMounts:
+    - name: persistent-storage
+      mountPath: /data
+  volumes:
+  - name: persistent-storage
+    persistentVolumeClaim:
+      claimName: ebs-claim
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  annotations:
+    skip-hostmount: "true"
+  name: ebs-claim
+spec:
+  accessModes:
+    - ReadWriteOncePod
+  volumeMode: Filesystem
+  storageClassName: ebs-sc
+  resources:
+    requests:
+      storage: 4Gi
+---
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: ebs-sc
+provisioner: ebs.csi.aws.com
+volumeBindingMode: WaitForFirstConsumer
+parameters:
+  csi.storage.k8s.io/fstype: ext4
+
+```
+Let’s assume that changes have been made in the `aws-ebs-csi-driver` node driver.
+
+**Node publish volume**
+1. In the node CSI driver, the `NodePublishVolume` API invokes: `kata-runtime direct-volume add --volume-path "/kubelet/a/b/c/d/sdf" --mount-info "{\"Device\": \"/dev/sdf\", \"fstype\": \"ext4\"}"`.
+2. The `Kata-runtime` writes the mount-info JSON to a file called `mountInfo.json` under `/run/kata-containers/shared/direct-volumes/kubelet/a/b/c/d/sdf`.
+
+**Node unstage volume**
+1. In the node CSI driver, the `NodeUnstageVolume` API invokes: `kata-runtime direct-volume remove --volume-path "/kubelet/a/b/c/d/sdf"`.
+2. Kata-runtime deletes the directory `/run/kata-containers/shared/direct-volumes/kubelet/a/b/c/d/sdf`.
+
+**Use the volume in sandbox**
+1. Upon the request to start a container, the `containerd-shim-kata-v2` examines the container spec,
+and iterates through the mounts. For each mount, if there is a `mountInfo.json` file under `/run/kata-containers/shared/direct-volumes/[mount source path]`,
+it generates a `storage` GRPC object after overwriting the mount spec with the information in `mountInfo.json`.
+2. The shim sends the storage objects to kata-agent through TTRPC.
+3. The shim writes a file with the sandbox id as the name under `/run/kata-containers/shared/direct-volumes/[mount source path]`.
+4. The kata-agent mounts the storage objects for the container.
+
+**Node expand volume**
+1. In the node CSI driver, the `NodeExpandVolume` API invokes: `kata-runtime direct-volume resize –-volume-path "/kubelet/a/b/c/d/sdf" –-size 8Gi`.
+2. The Kata runtime checks whether there is a sandbox id file under the directory `/run/kata-containers/shared/direct-volumes/kubelet/a/b/c/d/sdf`.
+3. The Kata runtime identifies the shim instance through the sandbox id, and sends a GRPC request to resize the volume.
+4. The shim handles the request, asks the hypervisor to resize the block device and sends a GRPC request to Kata agent to resize the filesystem.
+5. Kata agent receives the request and resizes the filesystem.
+
+**Node get volume stats**
+1. In the node CSI driver, the `NodeGetVolumeStats` API invokes: `kata-runtime direct-volume stats –-volume-path "/kubelet/a/b/c/d/sdf"`.
+2. The Kata runtime checks whether there is a sandbox id file under the directory `/run/kata-containers/shared/direct-volumes/kubelet/a/b/c/d/sdf`.
+3. The Kata runtime identifies the shim instance through the sandbox id, and sends a GRPC request to get the volume stats.
+4. The shim handles the request and forwards it to the Kata agent.
+5. Kata agent receives the request and returns the filesystem stats.

docs/design/kata-2-0-metrics.md

@@ -51,6 +51,7 @@ The `kata-monitor` management agent should be started on each node where the Kat
 > **Note**: a *node* running Kata containers will be either a single host system or a worker node belonging to a K8s cluster capable of running Kata pods.
 
 - Aggregate sandbox metrics running on the node, adding the `sandbox_id` label to them.
+- Attach the additional `cri_uid`, `cri_name` and `cri_namespace` labels to the sandbox metrics, tracking the `uid`, `name` and `namespace` Kubernetes pod metadata.
 - Expose a new Prometheus target, allowing all node metrics coming from the Kata shim to be collected by Prometheus indirectly. This simplifies the targets count in Prometheus and avoids exposing shim's metrics by `ip:port`.
 
 Only one `kata-monitor` process runs in each node.

docs/design/kata-nydus-design.md

@@ -0,0 +1,93 @@
+# Background
+
+[Research](https://www.usenix.org/conference/fast16/technical-sessions/presentation/harter) shows that time to take for pull operation accounts for 76% of container startup time but only 6.4% of that data is read. So if we can get data on demand (lazy load), it will speed up the container start. [`Nydus`](https://github.com/dragonflyoss/image-service) is a project which build image with new format and can get data on demand when container start.
+
+The following benchmarking result shows the performance improvement compared with the OCI image for the container cold startup elapsed time on containerd. As the OCI image size increases, the container startup time of using `nydus` image remains very short. [Click here](https://github.com/dragonflyoss/image-service/blob/master/docs/nydus-design.md) to see `nydus` design.
+
+![`nydus`-performance](arch-images/nydus-performance.png)
+
+## Proposal - Bring `lazyload` ability to Kata Containers
+
+`Nydusd` is a fuse/`virtiofs` daemon which is provided by `nydus` project and it supports `PassthroughFS` and [RAFS](https://github.com/dragonflyoss/image-service/blob/master/docs/nydus-design.md) (Registry Acceleration File System) natively, so in Kata Containers, we can use `nydusd` in place of `virtiofsd` and mount `nydus` image to guest in the meanwhile.
+
+The process of creating/starting Kata Containers with `virtiofsd`,
+
+1. When creating sandbox, the Kata Containers Containerd v2 [shim](https://github.com/kata-containers/kata-containers/blob/main/docs/design/architecture/README.md#runtime) will launch `virtiofsd` before VM starts and share directories with VM.
+2. When creating container, the Kata Containers Containerd v2 shim will mount rootfs to `kataShared`(/run/kata-containers/shared/sandboxes/\<SANDBOX\>/mounts/\<CONTAINER\>/rootfs), so it can be seen at the path `/run/kata-containers/shared/containers/shared/\<CONTAINER\>/rootfs` in the guest and used as container's rootfs.
+
+The process of creating/starting Kata Containers with `nydusd`,
+
+![kata-`nydus`](arch-images/kata-nydus.png)
+
+1. When creating sandbox, the Kata Containers Containerd v2 shim will launch `nydusd` daemon before VM starts.
+After VM starts, `kata-agent` will mount `virtiofs` at the path `/run/kata-containers/shared` and Kata Containers Containerd v2 shim mount `passthroughfs` filesystem to path `/run/kata-containers/shared/containers` when the VM starts.
+
+```bash
+# start nydusd
+$ sandbox_id=my-test-sandbox
+$ sudo /usr/local/bin/nydusd --log-level info --sock /run/vc/vm/${sandbox_id}/vhost-user-fs.sock --apisock /run/vc/vm/${sandbox_id}/api.sock
+```
+
+```bash
+# source: the host sharedir which will pass through to guest
+$ sudo curl -v --unix-socket /run/vc/vm/${sandbox_id}/api.sock \
+    -X POST "http://localhost/api/v1/mount?mountpoint=/containers" -H "accept: */*" \
+    -H "Content-Type: application/json" \
+    -d '{
+            "source":"/path/to/sharedir",
+            "fs_type":"passthrough_fs",
+            "config":""
+    }'
+```
+
+2. When creating normal container, the Kata Containers Containerd v2 shim send request to `nydusd` to mount `rafs` at the path `/run/kata-containers/shared/rafs/<container_id>/lowerdir` in guest.
+
+```bash
+# source: the metafile of nydus image
+# config: the config of this image
+$ sudo curl --unix-socket /run/vc/vm/${sandbox_id}/api.sock \
+    -X POST "http://localhost/api/v1/mount?mountpoint=/rafs/<container_id>/lowerdir" -H "accept: */*" \
+    -H "Content-Type: application/json" \
+    -d '{
+            "source":"/path/to/bootstrap",
+            "fs_type":"rafs",
+            "config":"config":"{\"device\":{\"backend\":{\"type\":\"localfs\",\"config\":{\"dir\":\"blobs\"}},\"cache\":{\"type\":\"blobcache\",\"config\":{\"work_dir\":\"cache\"}}},\"mode\":\"direct\",\"digest_validate\":true}",
+    }'
+```
+
+The Kata Containers Containerd v2 shim will also bind mount `snapshotdir` which `nydus-snapshotter` assigns to `sharedir`。
+So in guest, container rootfs=overlay(`lowerdir=rafs`, `upperdir=snapshotdir/fs`, `workdir=snapshotdir/work`)
+
+> how to transfer the `rafs` info from `nydus-snapshotter` to the Kata Containers Containerd v2 shim?
+
+By default, when creating `OCI` image container, `nydus-snapshotter` will return [`struct` Mount slice](https://github.com/containerd/containerd/blob/main/mount/mount.go#L21) below to containerd and containerd use them to mount rootfs
+
+```
+[
+    {
+        Type: "overlay",
+        Source: "overlay",
+        Options: [lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/<snapshot_A>/mnt,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/<snapshot_B>/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/<snapshot_B>/work],
+    }
+]
+```
+
+Then, we can append `rafs` info into `Options`, but if do this, containerd will mount failed, as containerd can not identify `rafs` info. Here, we can refer to [containerd mount helper](https://github.com/containerd/containerd/blob/main/mount/mount_linux.go#L42) and provide a binary called `nydus-overlayfs`. The `Mount` slice which `nydus-snapshotter` returned becomes
+
+```
+[
+    {
+        Type: "fuse.nydus-overlayfs",
+        Source: "overlay",
+        Options: [lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/<snapshot_A>/mnt,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/<snapshot_B>/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/<snapshot_B>/work,extraoption=base64({source:xxx,config:xxx,snapshotdir:xxx})],
+    }
+]
+```
+
+When containerd find `Type` is `fuse.nydus-overlayfs`,
+
+1. containerd will call `mount.fuse` command;
+2. in `mount.fuse`, it will call `nydus-overlayfs`.
+3. in `nydus-overlayfs`, it will ignore the `extraoption` and do the overlay mount.
+
+Finally, in the Kata Containers Containerd v2 shim, it parse `extraoption` and get the `rafs` info to mount the image in guest.

docs/design/vcpu-handling.md

@@ -2,24 +2,15 @@
 
 ## Default number of virtual CPUs
 
-Before starting a container, the [runtime][6] reads the `default_vcpus` option
-from the [configuration file][7] to determine the number of virtual CPUs
+Before starting a container, the [runtime][4] reads the `default_vcpus` option
+from the [configuration file][5] to determine the number of virtual CPUs
 (vCPUs) needed to start the virtual machine. By default, `default_vcpus` is
 equal to 1 for fast boot time and a small memory footprint per virtual machine.
 Be aware that increasing this value negatively impacts the virtual machine's
 boot time and memory footprint.
 In general, we recommend that you do not edit this variable, unless you know
 what are you doing. If your container needs more than one vCPU, use
-[docker `--cpus`][1], [docker update][4], or [Kubernetes `cpu` limits][2] to
-assign more resources.
-
-*Docker*
-
-```sh
-$ docker run --name foo -ti --cpus 2 debian bash
-$ docker update --cpus 4 foo
-```
-
+[Kubernetes `cpu` limits][1] to assign more resources.
 
 *Kubernetes*
 
@@ -49,7 +40,7 @@ $ sudo -E kubectl create -f ~/cpu-demo.yaml
 ## Virtual CPUs and Kubernetes pods
 
 A Kubernetes pod is a group of one or more containers, with shared storage and
-network, and a specification for how to run the containers [[specification][3]].
+network, and a specification for how to run the containers [[specification][2]].
 In Kata Containers this group of containers, which is called a sandbox, runs inside
 the same virtual machine. If you do not specify a CPU constraint, the runtime does
 not add more vCPUs and the container is not placed inside a CPU cgroup.
@@ -73,13 +64,7 @@ constraints with each container trying to consume 100% of vCPU, the resources
 divide in two parts, 50% of vCPU for each container because your virtual
 machine does not have enough resources to satisfy containers needs. If you want
 to give access to a greater or lesser portion of vCPUs to a specific container,
-use [`docker --cpu-shares`][1] or [Kubernetes `cpu` requests][2].
-
-*Docker*
-
-```sh
-$ docker run -ti --cpus-shares=512 debian bash
-```
+use [Kubernetes `cpu` requests][1].
 
 *Kubernetes*
 
@@ -109,10 +94,9 @@ $ sudo -E kubectl create -f ~/cpu-demo.yaml
 Before running containers without CPU constraint, consider that your containers
 are not running alone. Since your containers run inside a virtual machine other
 processes use the vCPUs as well (e.g. `systemd` and the Kata Containers
-[agent][5]). In general, we recommend setting `default_vcpus` equal to 1 to
+[agent][3]). In general, we recommend setting `default_vcpus` equal to 1 to
 allow non-container processes to run on this vCPU and to specify a CPU
-constraint for each container. If your container is already running and needs
-more vCPUs, you can add more using [docker update][4].
+constraint for each container.
 
 ## Container with CPU constraint
 
@@ -121,7 +105,7 @@ constraints using the following formula: `vCPUs = ceiling( quota / period )`, wh
 `quota` specifies the number of microseconds per CPU Period that the container is
 guaranteed CPU access and `period` specifies the CPU CFS scheduler period of time
 in microseconds. The result determines the number of vCPU to hot plug into the
-virtual machine. Once the vCPUs have been added, the [agent][5] places the
+virtual machine. Once the vCPUs have been added, the [agent][3] places the
 container inside a CPU cgroup. This placement allows the container to use only
 its assigned resources.
 
@@ -138,25 +122,6 @@ the virtual machine starts with 8 vCPUs and 1 vCPUs is added and assigned
 to the container. Non-container processes might be able to use 8 vCPUs but they
 use a maximum 1 vCPU, hence 7 vCPUs might not be used.
 
-
-*Container without CPU constraint*
-
-```sh
-$ docker run -ti debian bash -c "nproc; cat /sys/fs/cgroup/cpu,cpuacct/cpu.cfs_*"
-1       # number of vCPUs
-100000  # cfs period
--1      # cfs quota
-```
-
-*Container with CPU constraint*
-
-```sh
-docker run --cpus 4 -ti debian bash -c "nproc; cat /sys/fs/cgroup/cpu,cpuacct/cpu.cfs_*"
-5       # number of vCPUs
-100000  # cfs period
-400000  # cfs quota
-```
-
 ## Virtual CPU handling without hotplug
 
 In some cases, the hardware and/or software architecture being utilized does not support
@@ -183,11 +148,8 @@ the container's `spec` will provide the sizing information directly. If these ar
 calculate the number of CPUs required for the workload and augment this by `default_vcpus`
 configuration option, and use this for the virtual machine size.
 
-
-[1]: https://docs.docker.com/config/containers/resource_constraints/#cpu
-[2]: https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource
-[3]: https://kubernetes.io/docs/concepts/workloads/pods/pod/
-[4]: https://docs.docker.com/engine/reference/commandline/update/
-[5]: ../../src/agent
-[6]: ../../src/runtime
-[7]: ../../src/runtime/README.md#configuration
+[1]: https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource
+[2]: https://kubernetes.io/docs/concepts/workloads/pods/pod/
+[3]: ../../src/agent
+[4]: ../../src/runtime
+[5]: ../../src/runtime/README.md#configuration

docs/design/virtualization.md

@@ -39,7 +39,7 @@ Details of each solution and a summary are provided below.
 Kata Containers with QEMU has complete compatibility with Kubernetes.
 
 Depending on the host architecture, Kata Containers supports various machine types,
-for example `pc` and `q35` on x86 systems, `virt` on ARM systems and `pseries` on IBM Power systems. The default Kata Containers
+for example `q35` on x86 systems, `virt` on ARM systems and `pseries` on IBM Power systems. The default Kata Containers
 machine type is `q35`. The machine type and its [`Machine accelerators`](#machine-accelerators) can
 be changed by editing the runtime [`configuration`](architecture/README.md#configuration) file.
 
@@ -60,9 +60,8 @@ Machine accelerators are architecture specific and can be used to improve the pe
 and enable specific features of the machine types. The following machine accelerators
 are used in Kata Containers:
 
-- NVDIMM: This machine accelerator is x86 specific and only supported by `pc` and
-`q35` machine types. `nvdimm` is used to provide the root filesystem as a persistent
-memory device to the Virtual Machine.
+- NVDIMM: This machine accelerator is x86 specific and only supported by `q35` machine types.
+`nvdimm` is used to provide the root filesystem as a persistent memory device to the Virtual Machine.
 
 #### Hotplug devices
 

docs/how-to/README.md

@@ -15,6 +15,11 @@
 - `qemu`
 - `cloud-hypervisor`
 - `firecracker`
+
+   In the case of `firecracker` the use of a block device `snapshotter` is needed
+   for the VM rootfs. Refer to the following guide for additional configuration
+   steps:
+   - [Setup Kata containers with `firecracker`](how-to-use-kata-containers-with-firecracker.md)
 - `ACRN`
 
   While `qemu` , `cloud-hypervisor` and `firecracker` work out of the box with installation of Kata,
@@ -37,3 +42,4 @@
 - [How to setup swap devices in guest kernel](how-to-setup-swap-devices-in-guest-kernel.md)
 - [How to run rootless vmm](how-to-run-rootless-vmm.md)
 - [How to run Docker with Kata Containers](how-to-run-docker-with-kata.md)
+- [How to run Kata Containers with `nydus`](how-to-use-virtio-fs-nydus-with-kata.md)

docs/how-to/containerd-kata.md

@@ -72,7 +72,6 @@ $ command -v containerd
 
 ### Install CNI plugins
 
-> **Note:** You do not need to install CNI plugins if you do not want to use containerd with Kubernetes.
 > If you have installed Kubernetes with `kubeadm`, you might have already installed the CNI plugins.
 
 You can manually install CNI plugins as follows:
@@ -131,74 +130,42 @@ For
 
 The `RuntimeClass` is suggested.
 
-The following configuration includes three runtime classes:
+The following configuration includes two runtime classes:
 - `plugins.cri.containerd.runtimes.runc`: the runc, and it is the default runtime.
 - `plugins.cri.containerd.runtimes.kata`: The function in containerd (reference [the document here](https://github.com/containerd/containerd/tree/master/runtime/v2#binary-naming)) 
   where the dot-connected string `io.containerd.kata.v2` is translated to `containerd-shim-kata-v2` (i.e. the 
   binary name of the Kata implementation of [Containerd Runtime V2 (Shim API)](https://github.com/containerd/containerd/tree/master/runtime/v2)).
-- `plugins.cri.containerd.runtimes.katacli`: the `containerd-shim-runc-v1` calls `kata-runtime`, which is the legacy process.
 
 ```toml
     [plugins.cri.containerd]
       no_pivot = false
     [plugins.cri.containerd.runtimes]
-      [plugins.cri.containerd.runtimes.runc]
-         runtime_type = "io.containerd.runc.v1"
-         [plugins.cri.containerd.runtimes.runc.options]
-           NoPivotRoot = false
-           NoNewKeyring = false
-           ShimCgroup = ""
-           IoUid = 0
-           IoGid = 0
-           BinaryName = "runc"
-           Root = ""
-           CriuPath = ""
-           SystemdCgroup = false
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+         privileged_without_host_devices = false
+         runtime_type = "io.containerd.runc.v2"
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+            BinaryName = ""
+            CriuImagePath = ""
+            CriuPath = ""
+            CriuWorkPath = ""
+            IoGid = 0
       [plugins.cri.containerd.runtimes.kata]
          runtime_type = "io.containerd.kata.v2"
-      [plugins.cri.containerd.runtimes.katacli]
-         runtime_type = "io.containerd.runc.v1"
-         [plugins.cri.containerd.runtimes.katacli.options]
-           NoPivotRoot = false
-           NoNewKeyring = false
-           ShimCgroup = ""
-           IoUid = 0
-           IoGid = 0
-           BinaryName = "/usr/bin/kata-runtime"
-           Root = ""
-           CriuPath = ""
-           SystemdCgroup = false
-```
-
-From Containerd v1.2.4 and Kata v1.6.0, there is a new runtime option supported, which allows you to specify a specific Kata configuration file as follows:
-
-```toml
-      [plugins.cri.containerd.runtimes.kata]
-         runtime_type = "io.containerd.kata.v2"
-	 privileged_without_host_devices = true
-	 [plugins.cri.containerd.runtimes.kata.options]
-	   ConfigPath = "/etc/kata-containers/config.toml"
+         privileged_without_host_devices = true
+         pod_annotations = ["io.katacontainers.*"]
+         container_annotations = ["io.katacontainers.*"]
+         [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options]
+            ConfigPath = "/opt/kata/share/defaults/kata-containers/configuration.toml"
 ```
 
 `privileged_without_host_devices` tells containerd that a privileged Kata container should not have direct access to all host devices. If unset, containerd will pass all host devices to Kata container, which may cause security issues.
 
+`pod_annotations` is the list of pod annotations passed to both the pod sandbox as well as container through the OCI config.
+
+`container_annotations` is the list of container annotations passed through to the OCI config of the containers.
+
 This `ConfigPath` option is optional. If you do not specify it, shimv2 first tries to get the configuration file from the environment variable `KATA_CONF_FILE`. If neither are set, shimv2 will use the default Kata configuration file paths (`/etc/kata-containers/configuration.toml` and `/usr/share/defaults/kata-containers/configuration.toml`).
 
-If you use Containerd older than v1.2.4 or a version of Kata older than v1.6.0  and also want to specify a configuration file, you can use the following workaround, since the shimv2 accepts an environment variable, `KATA_CONF_FILE` for the configuration file path. Then, you can create a
-shell script with the following:
-
-```bash
-#!/usr/bin/env bash
-KATA_CONF_FILE=/etc/kata-containers/firecracker.toml containerd-shim-kata-v2 $@
-```
-
-Name it as `/usr/local/bin/containerd-shim-katafc-v2` and reference it in the configuration of containerd:
-
-```toml
-      [plugins.cri.containerd.runtimes.kata-firecracker]
-         runtime_type = "io.containerd.katafc.v2"
-```
-
 #### Kata Containers as the runtime for untrusted workload
 
 For cases without `RuntimeClass` support, we can use the legacy annotation method to support using Kata Containers 
@@ -218,28 +185,8 @@ and then, run an untrusted workload with Kata Containers:
       runtime_type = "io.containerd.kata.v2"
 ```
 
-For the earlier versions of Kata Containers and containerd that do not support Runtime V2 (Shim API), you can use the following alternative configuration:
-
-```toml
-    [plugins.cri.containerd]
-  
-    # "plugins.cri.containerd.default_runtime" is the runtime to use in containerd.
-    [plugins.cri.containerd.default_runtime]
-      # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
-      runtime_type = "io.containerd.runtime.v1.linux"
-
-    # "plugins.cri.containerd.untrusted_workload_runtime" is a runtime to run untrusted workloads on it.
-    [plugins.cri.containerd.untrusted_workload_runtime]
-      # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
-      runtime_type = "io.containerd.runtime.v1.linux"
-
-      # runtime_engine is the name of the runtime engine used by containerd.
-      runtime_engine = "/usr/bin/kata-runtime"
-```
-
 You can find more information on the [Containerd config documentation](https://github.com/containerd/cri/blob/master/docs/config.md)
 
-
 #### Kata Containers as the default runtime
 
 If you want to set Kata Containers as the only runtime in the deployment, you can simply configure as follows:
@@ -250,15 +197,6 @@ If you want to set Kata Containers as the only runtime in the deployment, you ca
       runtime_type = "io.containerd.kata.v2"
 ```
 
-Alternatively, for the earlier versions of Kata Containers and containerd that do not support Runtime V2 (Shim API), you can use the following alternative configuration:
-
-```toml
-    [plugins.cri.containerd]
-    [plugins.cri.containerd.default_runtime]
-      runtime_type = "io.containerd.runtime.v1.linux"
-      runtime_engine = "/usr/bin/kata-runtime"
-```
-
 ### Configuration for `cri-tools`
 
 > **Note:** If you skipped the [Install `cri-tools`](#install-cri-tools) section, you can skip this section too.
@@ -312,10 +250,12 @@ To run a container with Kata Containers through the containerd command line, you
 
 ```bash
 $ sudo ctr image pull docker.io/library/busybox:latest
-$ sudo ctr run --runtime io.containerd.run.kata.v2 -t --rm docker.io/library/busybox:latest hello sh
+$ sudo ctr run --cni --runtime io.containerd.run.kata.v2 -t --rm docker.io/library/busybox:latest hello sh
 ```
 
 This launches a BusyBox container named `hello`, and it will be removed by `--rm` after it quits.
+The `--cni` flag enables CNI networking for the container. Without this flag, a container with just a
+loopback interface is created.
 
 ### Launch Pods with `crictl` command line
 

docs/how-to/how-to-import-kata-logs-with-fluentd.md

@@ -4,7 +4,7 @@
 
 This document describes how to import Kata Containers logs into [Fluentd](https://www.fluentd.org/),
 typically for importing into an
-Elastic/Fluentd/Kibana([EFK](https://github.com/kubernetes/kubernetes/tree/master/cluster/addons/fluentd-elasticsearch#running-efk-stack-in-production))
+Elastic/Fluentd/Kibana([EFK](https://github.com/kubernetes-sigs/instrumentation-addons/tree/master/fluentd-elasticsearch#running-efk-stack-in-production))
 or Elastic/Logstash/Kibana([ELK](https://www.elastic.co/elastic-stack)) stack.
 
 The majority of this document focusses on CRI-O based (classic) Kata runtime. Much of that information
@@ -257,7 +257,7 @@ go directly to a full Kata specific JSON format logfile test.
 
 Kata runtime has the ability to generate JSON logs directly, rather than its default `logfmt` format. Passing
 the `--log-format=json` argument to the Kata runtime enables this. The easiest way to pass in this extra
-parameter from a [Kata deploy](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy) installation
+parameter from a [Kata deploy](../../tools/packaging/kata-deploy) installation
 is to edit the `/opt/kata/bin/kata-qemu` shell script.
 
 At the same time, we will add the `--log=/var/log/kata-runtime.log` argument to store the Kata logs in their

docs/how-to/how-to-run-docker-with-kata.md

@@ -22,7 +22,7 @@ You can learn more about about Docker-in-Docker at the following links:
 - [`docker` image Docker Hub page](https://hub.docker.com/_/docker/) (this page lists the `-dind` releases)
 
 While normally DinD refers to running `docker` from inside a Docker container,
-Kata Containers 2.x allows only supported runtimes (such as [`containerd`](../install/container-manager/containerd/containerd-install.md)).
+Kata Containers 2.x allows only [supported runtimes][kata-2.x-supported-runtimes] (such as [`containerd`](../install/container-manager/containerd/containerd-install.md)).
 
 Running `docker` in a Kata Container implies creating Docker containers from inside a container managed by `containerd` (or another supported container manager), as illustrated below:
 
@@ -37,7 +37,7 @@ container manager -> Kata Containers shim     -> Docker Daemon -> Docker contain
 
 [OverlayFS]: https://www.kernel.org/doc/html/latest/filesystems/overlayfs.html
 [v2.0.0]: https://github.com/kata-containers/kata-containers/releases/tag/2.0.0
-[kata-2.x-supported-runtimes]: https://github.com/kata-containers/kata-containers/blob/5737b36a3513f4da11a9dc7301b0c97ea22a51cf/docs/install/container-manager/containerd/containerd-install.md
+[kata-2.x-supported-runtimes]: ../install/container-manager/containerd/containerd-install.md
 
 ## Why Docker in Kata Containers 2.x requires special measures
 
@@ -48,9 +48,9 @@ Running Docker containers Kata Containers requires care because `VOLUME`s specif
 kataShared on / type virtiofs (rw,relatime,dax)
 ```
 
-`kataShared` mount types are powered by [`virtio-fs`][virtio-fs], a marked improvement over `virtio-9p`, thanks to [PR #1016](https://github.com/kata-containers/runtime/pull/1016). While `virtio-fs` is normally an excellent choice, in the case of DinD workloads `virtio-fs` causes an issue -- [it *cannot* be used as a "upper layer" of `overlayfs` without a custom patch](http://lists.katacontainers.io/pipermail/kata-dev/2020-January/001216.html).
+`kataShared` mount types are powered by [`virtio-fs`](https://virtio-fs.gitlab.io/), a marked improvement over `virtio-9p`, thanks to [PR #1016](https://github.com/kata-containers/runtime/pull/1016). While `virtio-fs` is normally an excellent choice, in the case of DinD workloads `virtio-fs` causes an issue -- [it *cannot* be used as a "upper layer" of `overlayfs` without a custom patch](http://lists.katacontainers.io/pipermail/kata-dev/2020-January/001216.html).
 
-As `/var/lib/docker` is a `VOLUME` specified by DinD (i.e. the `docker` images tagged `*-dind`/`*-dind-rootless`), `docker` fill fail to start (or even worse, silently pick a worse storage driver like `vfs`) when started in a Kata Container. Special measures must be taken when running DinD-powered workloads in Kata Containers.
+As `/var/lib/docker` is a `VOLUME` specified by DinD (i.e. the `docker` images tagged `*-dind`/`*-dind-rootless`), `docker` will fail to start (or even worse, silently pick a worse storage driver like `vfs`) when started in a Kata Container. Special measures must be taken when running DinD-powered workloads in Kata Containers.
 
 ## Workarounds/Solutions
 
@@ -58,7 +58,7 @@ Thanks to various community contributions (see [issue references below](#referen
 
 ### Use a memory backed volume
 
-For small workloads (small container images, without much generated filesystem load), a memory-backed volume is sufficient. Kubernetes supports a variant of  [the `EmptyDir` volume][k8s-emptydir], which allows for memdisk-backed storage -- the [the `medium: Memory` ][k8s-memory-volume-type]. An example of a `Pod` using such a setup [was contributed](https://github.com/kata-containers/runtime/issues/1429#issuecomment-477385283), and is reproduced below:
+For small workloads (small container images, without much generated filesystem load), a memory-backed volume is sufficient. Kubernetes supports a variant of  [the `EmptyDir` volume](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir), which allows for memdisk-backed storage -- the the `medium: Memory`. An example of a `Pod` using such a setup [was contributed](https://github.com/kata-containers/runtime/issues/1429#issuecomment-477385283), and is reproduced below:
 
 ```yaml
 apiVersion: v1

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -62,6 +62,8 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.file_mem_backend` (R) | string | file based memory backend root directory |
 | `io.katacontainers.config.hypervisor.firmware_hash` | string | container firmware SHA-512 hash value |
 | `io.katacontainers.config.hypervisor.firmware` | string | the guest firmware that will run the container VM |
+| `io.katacontainers.config.hypervisor.firmware_volume_hash` | string | container firmware volume SHA-512 hash value |
+| `io.katacontainers.config.hypervisor.firmware_volume` | string | the guest firmware volume that will be passed to the container VM |
 | `io.katacontainers.config.hypervisor.guest_hook_path` | string | the path within the VM that will be used for drop in hooks |
 | `io.katacontainers.config.hypervisor.hotplug_vfio_on_root_bus` | `boolean` | indicate if devices need to be hotplugged on the root bus instead of a bridge|
 | `io.katacontainers.config.hypervisor.hypervisor_hash` | string | container hypervisor binary SHA-512 hash value |
@@ -89,6 +91,7 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.virtio_fs_daemon` | string | virtio-fs `vhost-user` daemon path |
 | `io.katacontainers.config.hypervisor.virtio_fs_extra_args` | string | extra options passed to `virtiofs` daemon |
 | `io.katacontainers.config.hypervisor.enable_guest_swap` | `boolean` | enable swap in the guest |
+| `io.katacontainers.config.hypervisor.use_legacy_serial` | `boolean` | uses legacy serial device for guest's console (QEMU) |
 
 ## Container Options
 | Key | Value Type | Comments |
@@ -170,7 +173,7 @@ kind: Pod
 metadata:
   name: pod2
   annotations:
-    io.katacontainers.config.runtime.disable_guest_seccomp: false
+    io.katacontainers.config.runtime.disable_guest_seccomp: "false"
 spec:
   runtimeClassName: kata
   containers:

docs/how-to/how-to-use-kata-containers-with-acrn.md

@@ -101,7 +101,7 @@ Start an ACRN based Kata Container,
 $ sudo docker run -ti --runtime=kata-runtime busybox sh
 ```
 
-You will see ACRN(`acrn-dm`) is now running on your system, as well as a `kata-shim`, `kata-proxy`. You should obtain an interactive shell prompt. Verify that all the Kata processes terminate once you exit the container.
+You will see ACRN(`acrn-dm`) is now running on your system, as well as a `kata-shim`. You should obtain an interactive shell prompt. Verify that all the Kata processes terminate once you exit the container.
 
 ```bash
 $ ps -ef | grep -E "kata|acrn"

docs/how-to/how-to-use-kata-containers-with-firecracker.md

@@ -0,0 +1,254 @@
+# Configure Kata Containers to use Firecracker
+
+This document provides an overview on how to run Kata Containers with the AWS Firecracker hypervisor.
+
+## Introduction
+
+AWS Firecracker is an open source virtualization technology that is purpose-built for creating and managing secure, multi-tenant container and function-based services that provide serverless operational models. AWS Firecracker runs workloads in lightweight virtual machines, called `microVMs`, which combine the security and isolation properties provided by hardware virtualization technology with the speed and flexibility of Containers.
+
+Please refer to AWS Firecracker [documentation](https://github.com/firecracker-microvm/firecracker/blob/main/docs/getting-started.md) for more details.
+
+## Pre-requisites
+
+This document requires the presence of Kata Containers on your system. Install using the instructions available through the following links:
+
+- Kata Containers [automated installation](../install/README.md)
+
+- Kata Containers manual installation: Automated installation does not seem to be supported for Clear Linux, so please use [manual installation](../Developer-Guide.md) steps.
+> **Note:** Create rootfs image and not initrd image.
+
+## Install AWS Firecracker
+
+Kata Containers only support AWS Firecracker v0.23.4 ([yet](https://github.com/kata-containers/kata-containers/pull/1519)).
+To install Firecracker we need to get the `firecracker` and `jailer` binaries:
+
+```bash
+$ release_url="https://github.com/firecracker-microvm/firecracker/releases"
+$ version="v0.23.1"
+$ arch=`uname -m`
+$ curl ${release_url}/download/${version}/firecracker-${version}-${arch} -o firecracker
+$ curl ${release_url}/download/${version}/jailer-${version}-${arch} -o jailer
+$ chmod +x jailer firecracker
+```
+
+To make the binaries available from the default system `PATH` it is recommended to move them to `/usr/local/bin` or add a symbolic link:
+
+```bash
+$ sudo ln -s $(pwd)/firecracker /usr/local/bin
+$ sudo ln -s $(pwd)/jailer /usr/local/bin
+```
+
+More details can be found in [AWS Firecracker docs](https://github.com/firecracker-microvm/firecracker/blob/main/docs/getting-started.md)
+
+In order to run Kata with AWS Firecracker a block device as the backing store for a VM is required. To interact with `containerd` and Kata we use the `devmapper` `snapshotter`.
+
+## Configure `devmapper`
+
+To check support for your `containerd` installation, you can run:
+
+```
+$ ctr plugins ls |grep devmapper
+```
+
+if the output of the above command is:
+
+```
+io.containerd.snapshotter.v1    devmapper                linux/amd64    ok
+```
+then you can skip this section and move on to `Configure Kata Containers with AWS Firecracker`
+
+If the output of the above command is:
+
+```
+io.containerd.snapshotter.v1    devmapper                linux/amd64    error
+```
+
+then we need to setup `devmapper` `snapshotter`. Based on a [very useful
+guide](https://docs.docker.com/storage/storagedriver/device-mapper-driver/)
+from docker, we can set it up using the following scripts:
+
+> **Note:** The following scripts assume a 100G sparse file for storing container images, a 10G sparse file for the thin-provisioning pool and 10G base image files for any sandboxed container created. This means that we will need at least 10GB free space.
+
+```
+#!/bin/bash
+set -ex
+
+DATA_DIR=/var/lib/containerd/devmapper
+POOL_NAME=devpool
+
+mkdir -p ${DATA_DIR}
+
+# Create data file
+sudo touch "${DATA_DIR}/data"
+sudo truncate -s 100G "${DATA_DIR}/data"
+
+# Create metadata file
+sudo touch "${DATA_DIR}/meta"
+sudo truncate -s 10G "${DATA_DIR}/meta"
+
+# Allocate loop devices
+DATA_DEV=$(sudo losetup --find --show "${DATA_DIR}/data")
+META_DEV=$(sudo losetup --find --show "${DATA_DIR}/meta")
+
+# Define thin-pool parameters.
+# See https://www.kernel.org/doc/Documentation/device-mapper/thin-provisioning.txt for details.
+SECTOR_SIZE=512
+DATA_SIZE="$(sudo blockdev --getsize64 -q ${DATA_DEV})"
+LENGTH_IN_SECTORS=$(bc <<< "${DATA_SIZE}/${SECTOR_SIZE}")
+DATA_BLOCK_SIZE=128
+LOW_WATER_MARK=32768
+
+# Create a thin-pool device
+sudo dmsetup create "${POOL_NAME}" \
+    --table "0 ${LENGTH_IN_SECTORS} thin-pool ${META_DEV} ${DATA_DEV} ${DATA_BLOCK_SIZE} ${LOW_WATER_MARK}"
+
+cat << EOF
+#
+# Add this to your config.toml configuration file and restart `containerd` daemon
+#
+[plugins]
+  [plugins.devmapper]
+    pool_name = "${POOL_NAME}"
+    root_path = "${DATA_DIR}"
+    base_image_size = "10GB"
+    discard_blocks = true
+EOF
+```
+
+Make it executable and run it:
+
+```bash
+$ sudo chmod +x ~/scripts/devmapper/create.sh
+$ cd ~/scripts/devmapper/
+$ sudo ./create.sh
+```
+
+Now, we can add the `devmapper` configuration provided from the script to `/etc/containerd/config.toml`.
+> **Note:** If you are using the default `containerd` configuration (`containerd config default >> /etc/containerd/config.toml`), you may need to edit the existing `[plugins."io.containerd.snapshotter.v1.devmapper"]`configuration.
+Save and restart `containerd`:
+
+
+```bash
+$ sudo systemctl restart containerd
+```
+
+We can use `dmsetup` to verify that the thin-pool was created successfully.
+
+```bash
+$ sudo dmsetup ls
+```
+
+ We should also check that `devmapper` is registered and running:
+
+```bash
+$ sudo ctr plugins ls | grep devmapper
+```
+
+This script needs to be run only once, while setting up the `devmapper` `snapshotter` for `containerd`. Afterwards, make sure that on each reboot, the thin-pool is initialized from the same data directory. Otherwise, all the fetched containers (or the ones that you have created) will be re-initialized. A simple script that re-creates the thin-pool from the same data directory is shown below:
+
+```
+#!/bin/bash
+set -ex
+
+DATA_DIR=/var/lib/containerd/devmapper
+POOL_NAME=devpool
+
+# Allocate loop devices
+DATA_DEV=$(sudo losetup --find --show "${DATA_DIR}/data")
+META_DEV=$(sudo losetup --find --show "${DATA_DIR}/meta")
+
+# Define thin-pool parameters.
+# See https://www.kernel.org/doc/Documentation/device-mapper/thin-provisioning.txt for details.
+SECTOR_SIZE=512
+DATA_SIZE="$(sudo blockdev --getsize64 -q ${DATA_DEV})"
+LENGTH_IN_SECTORS=$(bc <<< "${DATA_SIZE}/${SECTOR_SIZE}")
+DATA_BLOCK_SIZE=128
+LOW_WATER_MARK=32768
+
+# Create a thin-pool device
+sudo dmsetup create "${POOL_NAME}" \
+    --table "0 ${LENGTH_IN_SECTORS} thin-pool ${META_DEV} ${DATA_DEV} ${DATA_BLOCK_SIZE} ${LOW_WATER_MARK}"
+```
+
+We can create a systemd service to run the above script on each reboot:
+
+```bash
+$ sudo nano /lib/systemd/system/devmapper_reload.service
+```
+
+The service file:
+
+```
+[Unit]
+Description=Devmapper reload script
+
+[Service]
+ExecStart=/path/to/script/reload.sh
+
+[Install]
+WantedBy=multi-user.target
+```
+
+Enable the newly created service:
+
+```bash
+$ sudo systemctl daemon-reload
+$ sudo systemctl enable devmapper_reload.service
+$ sudo systemctl start devmapper_reload.service
+```
+
+## Configure Kata Containers with AWS Firecracker
+
+To configure Kata Containers with AWS Firecracker, copy the generated `configuration-fc.toml` file when building the `kata-runtime` to either `/etc/kata-containers/configuration-fc.toml` or `/usr/share/defaults/kata-containers/configuration-fc.toml`.
+
+The following command shows full paths to the `configuration.toml` files that the runtime loads. It will use the first path that exists. (Please make sure the kernel and image paths are set correctly in the `configuration.toml` file)
+
+```bash
+$ sudo kata-runtime --show-default-config-paths
+```
+
+## Configure `containerd`
+Next, we need to configure containerd. Add a file in your path (e.g. `/usr/local/bin/containerd-shim-kata-fc-v2`) with the following contents:
+
+```
+#!/bin/bash
+KATA_CONF_FILE=/etc/containers/configuration-fc.toml /usr/local/bin/containerd-shim-kata-v2 $@
+```
+> **Note:** You may need to edit the paths of the configuration file and the `containerd-shim-kata-v2` to correspond to your setup.
+
+Make it executable:
+
+```bash
+$ sudo chmod +x /usr/local/bin/containerd-shim-kata-fc-v2
+```
+
+Add the relevant section in `containerd`’s `config.toml` file (`/etc/containerd/config.toml`):
+
+```
+[plugins.cri.containerd.runtimes]
+  [plugins.cri.containerd.runtimes.kata-fc]
+    runtime_type = "io.containerd.kata-fc.v2"
+```
+
+> **Note:** If you are using the default `containerd` configuration (`containerd config default >> /etc/containerd/config.toml`),
+> the configuration should change to :
+```
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-fc]
+    runtime_type = "io.containerd.kata-fc.v2"
+```
+
+Restart `containerd`:
+
+```bash
+$ sudo systemctl restart containerd
+```
+
+## Verify the installation
+
+We are now ready to launch a container using Kata with Firecracker to verify that everything worked:
+
+```bash
+$ sudo ctr images pull --snapshotter devmapper docker.io/library/ubuntu:latest
+$ sudo ctr run --snapshotter devmapper --runtime io.containerd.run.kata-fc.v2 -t --rm docker.io/library/ubuntu
+```

docs/how-to/how-to-use-virtio-fs-nydus-with-kata.md

@@ -0,0 +1,57 @@
+# Kata Containers with virtio-fs-nydus
+
+## Introduction
+
+Refer to [kata-`nydus`-design](../design/kata-nydus-design.md) for introduction and `nydus` has supported Kata Containers with hypervisor `QEMU` and `CLH` currently.
+
+## How to
+
+You can use Kata Containers with `nydus` as follows,
+
+1. Use [`nydus` latest branch](https://github.com/dragonflyoss/image-service);
+
+2. Deploy `nydus` environment as [`Nydus` Setup for Containerd Environment](https://github.com/dragonflyoss/image-service/blob/master/docs/containerd-env-setup.md);
+
+3. Start `nydus-snapshotter` with `enable_nydus_overlayfs` enabled;
+
+4. Use [kata-containers](https://github.com/kata-containers/kata-containers) `latest` branch to compile and build `kata-containers.img`;
+
+5. Update `configuration-qemu.toml` or `configuration-clh.toml`to include:
+
+```toml
+shared_fs = "virtio-fs-nydus"
+virtio_fs_daemon = "<nydusd binary path>"
+virtio_fs_extra_args = []
+```
+
+6. run `crictl run -r kata nydus-container.yaml nydus-sandbox.yaml`;
+
+The `nydus-sandbox.yaml` looks like below:
+
+```yaml
+metadata:
+  attempt: 1
+  name: nydus-sandbox
+  namespace: default
+log_directory: /tmp
+linux:
+  security_context:
+    namespace_options:
+      network: 2
+annotations:
+  "io.containerd.osfeature": "nydus.remoteimage.v1"
+```
+
+The `nydus-container.yaml` looks like below:
+
+```yaml
+metadata:
+  name: nydus-container
+image:
+  image: localhost:5000/ubuntu-nydus:latest
+command:
+  - /bin/sleep
+args:
+  - 600
+log_path: container.1.log
+```

docs/how-to/how-to-use-virtio-fs-with-kata.md

@@ -6,4 +6,4 @@ Container deployments utilize explicit or implicit file sharing between host fil
 
 As of the 2.0 release of Kata Containers, [virtio-fs](https://virtio-fs.gitlab.io/) is the default filesystem sharing mechanism.
 
-virtio-fs support works out of the box for `cloud-hypervisor` and `qemu`, when Kata Containers is deployed using `kata-deploy`. Learn more about `kata-deploy` and how to use `kata-deploy` in Kubernetes [here](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy#kubernetes-quick-start).
+virtio-fs support works out of the box for `cloud-hypervisor` and `qemu`, when Kata Containers is deployed using `kata-deploy`. Learn more about `kata-deploy` and how to use `kata-deploy` in Kubernetes [here](../../tools/packaging/kata-deploy/README.md#kubernetes-quick-start).

docs/how-to/privileged.md

@@ -31,7 +31,7 @@ See below example config:
   [plugins.cri]
     [plugins.cri.containerd]
        [plugins.cri.containerd.runtimes.runc]
-         runtime_type = "io.containerd.runc.v1"
+         runtime_type = "io.containerd.runc.v2"
          privileged_without_host_devices = false
        [plugins.cri.containerd.runtimes.kata]
          runtime_type = "io.containerd.kata.v2"

docs/how-to/run-kata-with-k8s.md

@@ -104,26 +104,69 @@ $ sudo kubeadm init --ignore-preflight-errors=all --cri-socket /run/containerd/c
 $ export KUBECONFIG=/etc/kubernetes/admin.conf
 ```
 
-You can force Kubelet to use Kata Containers by adding some `untrusted`
-annotation to your pod configuration. In our case, this ensures Kata
-Containers is the selected runtime to run the described workload.
+### Allow pods to run in the master node
 
-`nginx-untrusted.yaml`
-```yaml
-apiVersion: v1
-kind: Pod
+By default, the cluster will not schedule pods in the master node. To enable master node scheduling:
+```bash
+$ sudo -E kubectl taint nodes --all node-role.kubernetes.io/master-
+```
+
+### Create runtime class for Kata Containers
+
+Users can use [`RuntimeClass`](https://kubernetes.io/docs/concepts/containers/runtime-class/#runtime-class) to specify a different runtime for Pods.
+
+```bash
+$ cat > runtime.yaml <<EOF
+apiVersion: node.k8s.io/v1
+kind: RuntimeClass
 metadata:
-  name: nginx-untrusted
-  annotations:
-    io.kubernetes.cri.untrusted-workload: "true"
-spec:
-  containers:
+  name: kata
+handler: kata
+EOF
+
+$ sudo -E kubectl apply -f runtime.yaml
+```
+
+### Run pod in Kata Containers
+
+If a pod has the `runtimeClassName` set to `kata`, the CRI plugin runs the pod with the
+[Kata Containers runtime](../../src/runtime/README.md).
+
+- Create an pod configuration that using Kata Containers runtime
+
+  ```bash
+  $ cat << EOF | tee nginx-kata.yaml
+  apiVersion: v1
+  kind: Pod
+  metadata:
+    name: nginx-kata
+  spec:
+    runtimeClassName: kata
+    containers:
     - name: nginx
       image: nginx
-```
 
-Next, you run your pod:
-```
-$ sudo -E kubectl apply -f nginx-untrusted.yaml
-```
+  EOF
+  ```
 
+- Create the pod
+  ```bash
+  $ sudo -E kubectl apply -f nginx-kata.yaml
+  ```
+
+- Check pod is running
+
+  ```bash
+  $ sudo -E kubectl get pods
+  ```
+
+- Check hypervisor is running
+  ```bash
+  $ ps aux | grep qemu
+  ```
+
+### Delete created pod
+
+```bash
+$ sudo -E kubectl delete -f nginx-kata.yaml
+```

docs/install/container-manager/containerd/containerd-install.md

@@ -81,7 +81,7 @@
   - Download the standard `systemd(1)` service file and install to
     `/etc/systemd/system/`:
 
-    - https://raw.githubusercontent.com/containerd/containerd/master/containerd.service
+    - https://raw.githubusercontent.com/containerd/containerd/main/containerd.service
 
     > **Notes:**
     >

docs/install/minikube-installation-guide.md

@@ -6,7 +6,7 @@
 cluster locally. It creates a single node Kubernetes stack in a local VM.
 
 [Kata Containers](https://github.com/kata-containers) can be installed into a Minikube cluster using
-[`kata-deploy`](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy).
+[`kata-deploy`](../../tools/packaging/kata-deploy).
 
 This document details the pre-requisites, installation steps, and how to check
 the installation has been successful.
@@ -123,7 +123,7 @@ $ kubectl apply -f kata-deploy/base/kata-deploy.yaml
 This installs the Kata Containers components into `/opt/kata` inside the Minikube node. It can take
 a few minutes for the operation to complete. You can check the installation has worked by checking
 the status of the `kata-deploy` pod, which will be executing
-[this script](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy/scripts/kata-deploy.sh),
+[this script](../../tools/packaging/kata-deploy/scripts/kata-deploy.sh),
 and will be executing a `sleep infinity` once it has successfully completed its work.
 You can accomplish this by running the following:
 

docs/install/snap-installation-guide.md

@@ -39,8 +39,8 @@ can be used as runtime.
 
 Read the following documents to know how to run Kata Containers 2.x with `containerd`.
 
-* [How to use Kata Containers and Containerd](https://github.com/kata-containers/kata-containers/blob/main/docs/how-to/containerd-kata.md)
-* [Install Kata Containers with containerd](https://github.com/kata-containers/kata-containers/blob/main/docs/install/container-manager/containerd/containerd-install.md)
+* [How to use Kata Containers and Containerd](../how-to/containerd-kata.md)
+* [Install Kata Containers with containerd](./container-manager/containerd/containerd-install.md)
 
 
 ## Remove Kata Containers snap package

docs/presentations/README.md

@@ -0,0 +1,3 @@
+# Kata Containers presentations
+
+* [Unit testing](unit-testing)

docs/presentations/unit-testing/README.md

@@ -0,0 +1,14 @@
+# Kata Containers unit testing presentation
+
+## Markdown version
+
+See [the Kata Containers unit testing presentation](kata-containers-unit-testing.md).
+
+### To view as an HTML presentation
+
+```bash
+$ infile="kata-containers-unit-testing.md"
+$ outfile="/tmp/kata-containers-unit-testing.html"
+$ pandoc -s --metadata title="Kata Containers unit testing" -f markdown -t revealjs --highlight-style="zenburn" -i -o "$outfile" "$infile"
+$ xdg-open "file://$outfile"
+```

docs/presentations/unit-testing/kata-containers-unit-testing.md

@@ -0,0 +1,335 @@
+## Why write unit tests?
+
+- Catch regressions
+
+- Improve the code being tested
+
+  Structure, quality, security, performance, "shakes out" implicit
+  assumptions, _etc_
+
+- Extremely instructive
+
+  Once you've fully tested a single function, you'll understand that
+  code very well indeed.
+
+## Why write unit tests? (continued)
+
+- Fun!
+
+  Yes, really! Don't believe me? Try it! ;)
+
+## Run all Kata Containers agent unit tests
+
+As an example, to run all agent unit tests:
+
+```bash
+$ cd $GOPATH/src/github.com/kata-containers/kata-containers
+$ cd src/agent
+$ make test
+```
+
+## List all unit tests
+
+- Identify the full name of all the tests _in the current package_:
+
+  ```bash
+  $ cargo test -- --list
+  ```
+
+- Identify the full name of all tests in the `foo` "local crate"
+  (sub-directory containing another `Cargo.toml` file):
+
+  ```bash
+  $ cargo test -p "foo" -- --list
+  ```
+
+## Run a single unit test
+
+- Run a test in the current package in verbose mode:
+
+  ```bash
+  # Example 
+  $ test="config::tests::test_get_log_level"
+
+  $ cargo test "$test" -vv -- --exact --nocapture
+  ```
+
+## Test coverage setup
+
+```bash
+$ cargo install cargo-tarpaulin
+```
+
+## Show test coverage
+
+```bash
+$ cd $GOPATH/src/github.com/kata-containers/kata-containers/src/agent
+$ cargo -v tarpaulin --all-features --run-types AllTargets --count --force-clean -o Html
+$ xdg-open "file://$PWD/tarpaulin-report.html"
+```
+
+## Testability (part 1)
+
+- To be testable, a function should:
+  - Not be "too long" (say >100 lines).
+  - Not be "too complex" (say >3 levels of indentation).
+  - Should return a `Result` or an `Option` so error paths
+    can be tested.
+
+- If functions don't conform, they need to be reworked (refactored)
+  before writing tests.
+
+## Testability (part 2)
+
+- Some functions can't be fully tested.
+- However, you _can_ test the initial code that checks
+  the parameter values (test error paths only).
+
+## Writing new tests: General advice (part 1)
+
+- KISS: Keep It Simple Stupid
+
+  You don't get extra points for cryptic code.
+
+- DRY: Don't Repeat Yourself
+
+  Make use of existing facilities (don't "re-invert the wheel").
+
+- Read the [unit test advice document](https://github.com/kata-containers/kata-containers/blob/main/docs/Unit-Test-Advice.md)
+
+## Writing new tests: General advice (part 2)
+
+- Attack the function in all possible ways
+
+- Use the _table driven_ approach:
+  - Simple
+  - Compact
+  - Easy to debug
+  - Makes boundary analysis easy
+  - Encourages functions to be testable
+
+## Writing new tests: Specific advice (part 1)
+
+- Create a new "`tests`" module if necessary.
+- Give each test function a "`test_`" prefix.
+- Add the "`#[test]`" annotation on each test function.
+
+## Writing new tests: Specific advice (part 2)
+
+- If you need to `use` (import) packages for the tests,
+  _only do it in the `tests` module_:
+  ```rust
+  use some_test_pkg::{foo, bar}; // <-- Not here
+
+  #[cfg(test)]
+  mod tests {
+    use super::*;
+    use some_test_pkg:{foo, bar}; // <-- Put it here
+  }
+  ```
+
+## Writing new tests: Specific advice (part 3)
+
+- You can add test-specific dependencies in `Cargo.toml`:
+  ```toml
+  [dev-dependencies]
+  serial_test = "0.5.1"
+  ```
+
+## Writing new tests: Specific advice (part 4)
+
+- Don't add in lots of error handling code: let the test panic!
+  ```rust
+  // This will panic if the unwrap fails.
+  // - NOT acceptable generally for production code.
+  // - PERFECTLY acceptable for test code since:
+  //   - Keeps the test code simple.
+  //   - Rust will detect the panic and fail the test.
+  let result = func().unwrap();
+  ```
+
+## Debugging tests (part 1)
+
+- Comment out all tests in your `TestData` array apart from the failing test.
+
+- Add temporary `println!("FIXME: ...")` statements in the code.
+
+- Set `RUST_BACKTRACE=full` before running `cargo test`.
+
+## Debugging tests (part 2)
+
+- Use a debugger (not normally necessary though):
+  ```bash
+  # Disable optimisation
+  $ RUSTFLAGS="-C opt-level=0" cargo test --no-run
+
+  # Find the test binary
+  $ test_binary=$(find target/debug/deps | grep "kata_agent-[a-z0-9][a-z0-9]*$" | tail -1)
+
+  $ rust-gdb "$test_binary"
+  ```
+
+## Useful tips
+
+- Always start a test with a "clean environment":
+
+  Create new set of objects / files / directories / _etc_
+  for each test.
+
+- Mounts
+  - Linux allows mounts on top of existing mounts.
+  - Bind mounts and read-only mounts can be useful.
+
+## Gotchas (part 1)
+
+If a test runs successfully _most of the time_:
+
+- Review the test logic.
+
+- Add a `#[serial]` annotation on the test function
+  Requires the `serial_test` package in the `[dev-dependencies]`
+  section of `Cargo.toml`.
+
+  If this makes it work the test is probably sharing resources with
+  another task (thread).
+
+## Gotchas (part 2)
+
+If a test works locally but fails in the CI, consider the following
+attributes of each environment (local and CI):
+
+- The version of rust being used.
+- The hardware architecture.
+- Number (and spec) of the CPUs.
+
+## Gotchas (part 3)
+
+If in doubt, look at the
+["test artifacts" attached to the failing CI test](http://jenkins.katacontainers.io).
+
+## Before raising a PR
+
+- Remember to check that the test runs locally:
+  - As a non-privileged user.
+  - As the `root` user (carefully!)
+
+- Run the [static checker](https://github.com/kata-containers/tests/blob/main/.ci/static-checks.sh)
+  on your changes.
+
+  Checks formatting and many other things.
+
+## If in doubt
+
+- Ask for help! ;)
+
+## Quiz 1
+
+What's wrong with this function?
+
+```rust
+fn foo(config: &Config, path_prefix: String, container_id: String, pid: String) -> Result<()> {
+    let mut full_path = format!("{}/{}", path_prefix, container_id);
+
+    let _ = remove_recursively(&mut full_path);
+
+    write_number_to_file(pid, full_path);
+
+    Ok(())
+}
+```
+
+## Quiz 1: Answers (part 1)
+
+- No check that `path_prefix`, `container_id` and `pid` are not `""`.
+- No check that `path_prefix` is absolute.
+- No check that `container_id` does not contain slashes / contains only valid characters.
+- Result of `remove_recursively()` discarded.
+- `remove_recursively()` _may_ modify `full_path` without `foo()` knowing!
+
+## Quiz 1: Answers (part 2)
+
+- Why is `pid` not a numeric?
+- No check to ensure the PID is positive.
+- No check to recreate any directories in the original `path_prefix`.
+- `write_number_to_file()` could fail so why doesn't it return a value?
+- The `config` parameter is unused.
+
+## Quiz 1: What if...
+
+Imagine if the caller managed to do this:
+
+```rust
+foo(config, "", "sbin/init", r#"#!/bin/sh\n/sbin/reboot"#);
+```
+
+## Quiz 2
+
+What makes this function difficult to test?
+
+```rust
+fn get_user_id(username: String) -> i32 {
+    let line = grep_file(username, "/etc/passwd").unwrap();
+    let fields = line.split(':');
+
+    let uid = fields.nth(2).ok_or("failed").unwrap();
+
+    uid.parse::<i32>()
+}
+```
+
+## Quiz 2: Answers (part 1)
+
+- Unhelpful error message ("failed").
+
+- Panics on error! Return a `Result` instead!
+
+- UID's cannot be negative so function should return an unsigned
+  value.
+
+## Quiz 2: Answers (part 2)
+
+- Hard-coded filename.
+
+  This would be better:
+
+  ```rust
+  const PASSWD_DB: &str = "/etc/passwd";
+
+  // Test code can now pass valid and invalid files!
+  fn get_user_id(filename: String, username: String) -> i32 {
+    // ...
+  }
+
+  let id = get_user_id(PASSWD_DB, username);
+  ```
+
+## Quiz 3
+
+What's wrong with this test code?
+
+```rust
+let mut obj = Object::new();
+
+// Sanity check
+assert_eq!(obj.num, 0);
+assert_eq!(obj.wibble, false);
+
+// Test 1
+obj->foo_method(7);
+assert_eq!(obj.num, 7);
+
+// Test 2
+obj->bar_method(true);
+assert_eq!(obj.wibble, true);
+```
+
+## Quiz 3: Answers
+
+- The test code is "fragile":
+  - The 2nd test re-uses the object created in the first test.
+
+## Finally
+
+- [We need a GH action to run the unit tests](https://github.com/kata-containers/kata-containers/issues/2934)
+
+  Needs to fail PRs that decrease test coverage<br/> by "x%".

docs/tracing.md

@@ -203,11 +203,11 @@ is highly recommended. For working with the agent, you may also wish to
 [enable a debug console][setup-debug-console]
 to allow you to access the VM environment.
 
-[enable-full-debug]: https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md#enable-full-debug
+[enable-full-debug]: ./Developer-Guide.md#enable-full-debug
 [jaeger-all-in-one]: https://www.jaegertracing.io/docs/getting-started/
 [jaeger-tracing]: https://www.jaegertracing.io
 [opentelemetry]: https://opentelemetry.io
-[osbuilder]: https://github.com/kata-containers/kata-containers/blob/main/tools/osbuilder
-[setup-debug-console]: https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md#set-up-a-debug-console
+[osbuilder]: ../tools/osbuilder
+[setup-debug-console]: ./Developer-Guide.md#set-up-a-debug-console
 [trace-forwarder]: /src/tools/trace-forwarder
 [vsock]: https://wiki.qemu.org/Features/VirtioVsock

docs/use-cases/GPU-passthrough-and-Kata.md

@@ -3,4 +3,4 @@
 Kata Containers supports passing certain GPUs from the host into the container. Select the GPU vendor for detailed information:
 
 - [Intel](Intel-GPU-passthrough-and-Kata.md)
-- [Nvidia](Nvidia-GPU-passthrough-and-Kata.md)
+- [NVIDIA](NVIDIA-GPU-passthrough-and-Kata.md)

docs/use-cases/NVIDIA-GPU-passthrough-and-Kata.md

@@ -0,0 +1,592 @@
+# Using NVIDIA GPU device with Kata Containers
+
+An NVIDIA GPU device can be passed to a Kata Containers container using GPU
+passthrough (NVIDIA GPU pass-through mode) as well as GPU mediated passthrough
+(NVIDIA `vGPU` mode).
+
+NVIDIA GPU pass-through mode, an entire physical GPU is directly assigned to one
+VM, bypassing the NVIDIA Virtual GPU Manager. In this mode of operation, the GPU
+is accessed exclusively by the NVIDIA driver running in the VM to which it is
+assigned. The GPU is not shared among VMs.
+
+NVIDIA Virtual GPU (`vGPU`) enables multiple virtual machines (VMs) to have
+simultaneous, direct access to a single physical GPU, using the same NVIDIA
+graphics drivers that are deployed on non-virtualized operating systems. By
+doing this, NVIDIA `vGPU` provides VMs with unparalleled graphics performance,
+compute performance, and application compatibility, together with the
+cost-effectiveness and scalability brought about by sharing a GPU among multiple
+workloads. A `vGPU` can be either time-sliced or Multi-Instance GPU (MIG)-backed
+with [MIG-slices](https://docs.nvidia.com/datacenter/tesla/mig-user-guide/).
+
+| Technology | Description | Behavior | Detail |
+| --- | --- | --- | --- |
+| NVIDIA GPU pass-through mode | GPU passthrough | Physical GPU assigned to a single VM | Direct GPU assignment to VM without limitation |
+| NVIDIA vGPU time-sliced | GPU time-sliced | Physical GPU time-sliced for multiple VMs | Mediated passthrough |
+| NVIDIA vGPU MIG-backed | GPU with MIG-slices | Physical GPU MIG-sliced for multiple VMs | Mediated passthrough |
+
+## Hardware Requirements
+
+NVIDIA GPUs Recommended for Virtualization:
+
+- NVIDIA Tesla (T4, M10, P6, V100 or newer)
+- NVIDIA Quadro RTX 6000/8000
+
+## Host BIOS Requirements
+
+Some hardware requires a larger PCI BARs window, for example, NVIDIA Tesla P100,
+K40m
+
+```sh
+$ lspci -s d0:00.0 -vv | grep Region
+        Region 0: Memory at e7000000 (32-bit, non-prefetchable) [size=16M]
+        Region 1: Memory at 222800000000 (64-bit, prefetchable) [size=32G] # Above 4G
+        Region 3: Memory at 223810000000 (64-bit, prefetchable) [size=32M]
+```
+
+For large BARs devices, MMIO mapping above 4G address space should be `enabled`
+in the PCI configuration of the BIOS.
+
+Some hardware vendors use a different name in BIOS, such as:
+
+- Above 4G Decoding
+- Memory Hole for PCI MMIO
+- Memory Mapped I/O above 4GB
+
+If one is using a GPU based on the Ampere architecture and later additionally
+SR-IOV needs to be enabled for the `vGPU` use-case.
+
+The following steps outline the workflow for using an NVIDIA GPU with Kata.
+
+## Host Kernel Requirements
+
+The following configurations need to be enabled on your host kernel:
+
+- `CONFIG_VFIO`
+- `CONFIG_VFIO_IOMMU_TYPE1`
+- `CONFIG_VFIO_MDEV`
+- `CONFIG_VFIO_MDEV_DEVICE`
+- `CONFIG_VFIO_PCI`
+
+Your host kernel needs to be booted with `intel_iommu=on` on the kernel command
+line.
+
+## Install and configure Kata Containers
+
+To use non-large BARs devices (for example, NVIDIA Tesla T4), you need Kata
+version 1.3.0 or above. Follow the [Kata Containers setup
+instructions](../install/README.md) to install the latest version of Kata.
+
+To use large BARs devices (for example, NVIDIA Tesla P100), you need Kata
+version 1.11.0 or above.
+
+The following configuration in the Kata `configuration.toml` file as shown below
+can work:
+
+Hotplug for PCI devices with small BARs by `acpi_pcihp` (Linux's ACPI PCI
+Hotplug driver):
+
+```sh
+machine_type = "q35"
+
+hotplug_vfio_on_root_bus = false
+```
+
+Hotplug for PCIe devices with large BARs by `pciehp` (Linux's PCIe Hotplug
+driver):
+
+```sh
+machine_type = "q35"
+
+hotplug_vfio_on_root_bus = true
+pcie_root_port = 1
+```
+
+## Build Kata Containers kernel with GPU support
+
+The default guest kernel installed with Kata Containers does not provide GPU
+support. To use an NVIDIA GPU with Kata Containers, you need to build a kernel
+with the necessary GPU support.
+
+The following kernel config options need to be enabled:
+
+```sh
+# Support PCI/PCIe device hotplug (Required for large BARs device)
+CONFIG_HOTPLUG_PCI_PCIE=y
+
+# Support for loading modules (Required for load NVIDIA drivers)
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+
+# Enable the MMIO access method for PCIe devices (Required for large BARs device)
+ CONFIG_PCI_MMCONFIG=y
+```
+
+The following kernel config options need to be disabled:
+
+```sh
+# Disable Open Source NVIDIA driver nouveau
+# It conflicts with NVIDIA official driver
+CONFIG_DRM_NOUVEAU=n
+```
+
+> **Note**: `CONFIG_DRM_NOUVEAU` is normally disabled by default.
+It is worth checking that it is not enabled in your kernel configuration to
+prevent any conflicts.
+
+Build the Kata Containers kernel with the previous config options, using the
+instructions described in [Building Kata Containers
+kernel](../../tools/packaging/kernel). For further details on building and
+installing guest kernels, see [the developer
+guide](../Developer-Guide.md#install-guest-kernel-images).
+
+There is an easy way to build a guest kernel that supports NVIDIA GPU:
+
+```sh
+## Build guest kernel with ../../tools/packaging/kernel
+
+# Prepare (download guest kernel source, generate .config)
+$ ./build-kernel.sh -v 5.15.23 -g nvidia -f setup
+
+# Build guest kernel
+$ ./build-kernel.sh -v 5.15.23 -g nvidia build
+
+# Install guest kernel
+$ sudo -E ./build-kernel.sh -v 5.15.23 -g nvidia install
+```
+
+To build NVIDIA Driver in Kata container, `linux-headers` are required.
+This is a way to generate deb packages for `linux-headers`:
+
+> **Note**:
+> Run `make rpm-pkg` to build the rpm package.
+> Run `make deb-pkg` to build the deb package.
+>
+
+```sh
+$ cd kata-linux-5.15.23-89
+$ make deb-pkg
+```
+Before using the new guest kernel, please update the `kernel` parameters in
+ `configuration.toml`.
+
+```sh
+kernel = "/usr/share/kata-containers/vmlinuz-nvidia-gpu.container"
+```
+
+## NVIDIA GPU pass-through mode with Kata Containers
+
+Use the following steps to pass an NVIDIA GPU device in pass-through mode with Kata:
+
+1. Find the Bus-Device-Function (BDF) for the GPU device on the host:
+
+   ```sh
+   $ sudo lspci -nn -D | grep -i nvidia
+   0000:d0:00.0 3D controller [0302]: NVIDIA Corporation Device [10de:20b9] (rev a1)
+   ```
+
+   > PCI address `0000:d0:00.0` is assigned to the hardware GPU device.
+   > `10de:20b9` is the device ID of the hardware GPU device.
+
+2. Find the IOMMU group for the GPU device:
+
+   ```sh
+   $ BDF="0000:d0:00.0"
+   $ readlink -e /sys/bus/pci/devices/$BDF/iommu_group
+   ```
+
+   The previous output shows that the GPU belongs to IOMMU group 192. The next
+   step is to bind the GPU to the VFIO-PCI driver.
+
+   ```sh
+   $ BDF="0000:d0:00.0"
+   $ DEV="/sys/bus/pci/devices/$BDF"
+   $ echo "vfio-pci" > $DEV/driver_override
+   $ echo $BDF > $DEV/driver/unbind
+   $ echo $BDF > /sys/bus/pci/drivers_probe
+   # To return the device to the standard driver, we simply clear the
+   # driver_override and reprobe the device, ex:
+   $ echo > $DEV/preferred_driver
+   $ echo $BDF > $DEV/driver/unbind
+   $ echo $BDF > /sys/bus/pci/drivers_probe
+   ```
+
+3. Check the IOMMU group number under `/dev/vfio`:
+
+   ```sh
+   $ ls -l /dev/vfio
+   total 0
+   crw------- 1 zvonkok zvonkok 243,   0 Mar 18 03:06 192
+   crw-rw-rw- 1 root    root     10, 196 Mar 18 02:27 vfio
+   ```
+
+4. Start a Kata container with the GPU device:
+
+   ```sh
+   # You may need to `modprobe vhost-vsock` if you get
+   # host system doesn't support vsock: stat /dev/vhost-vsock
+   $ sudo ctr --debug run --runtime "io.containerd.kata.v2"  --device /dev/vfio/192  --rm -t  "docker.io/library/archlinux:latest" arch uname -r
+   ```
+
+5. Run `lspci` within the container to verify the GPU device is seen in the list
+   of the PCI devices. Note the vendor-device id of the GPU (`10de:20b9`) in the `lspci` output.
+
+   ```sh
+   $ sudo ctr --debug run --runtime "io.containerd.kata.v2"  --device /dev/vfio/192  --rm -t  "docker.io/library/archlinux:latest" arch sh -c "lspci -nn | grep '10de:20b9'"
+   ```
+
+6. Additionally, you can check the PCI BARs space of ​​the NVIDIA GPU device in the container:
+
+   ```sh
+   $ sudo ctr --debug run --runtime "io.containerd.kata.v2"  --device /dev/vfio/192  --rm -t  "docker.io/library/archlinux:latest" arch sh -c "lspci -s 02:00.0 -vv | grep Region"
+   ```
+
+   > **Note**: If you see a message similar to the above, the BAR space of the NVIDIA
+   > GPU has been successfully allocated.
+
+## NVIDIA vGPU mode with Kata Containers
+
+NVIDIA vGPU is a licensed product on all supported GPU boards. A software license
+is required to enable all vGPU features within the guest VM. NVIDIA vGPU manager
+needs to be installed on the host to configure GPUs in vGPU mode. See [NVIDIA Virtual GPU Software Documentation v14.0 through 14.1](https://docs.nvidia.com/grid/14.0/) for more details.
+
+### NVIDIA vGPU time-sliced
+
+In the time-sliced mode, the GPU is not partitioned and the workload uses the
+whole GPU and shares access to the GPU engines. Processes are scheduled in
+series. The best effort scheduler is the default one and can be exchanged by
+other scheduling policies see the documentation above how to do that.
+
+Beware if you had `MIG` enabled before to disable `MIG` on the GPU if you want
+to use `time-sliced` `vGPU`.
+
+```sh
+$ sudo nvidia-smi -mig 0
+```
+
+Enable the virtual functions for the physical GPU in the `sysfs` file system.
+
+```sh
+$ sudo /usr/lib/nvidia/sriov-manage -e 0000:41:00.0
+```
+
+Get the `BDF` of the available virtual function on the GPU, and choose one for the
+following steps.
+
+```sh
+$ cd /sys/bus/pci/devices/0000:41:00.0/
+$ ls -l |  grep virtfn
+```
+
+#### List all available vGPU instances
+
+The following shell snippet will walk the `sysfs` and only print instances
+that are available, that can be created.
+
+```sh
+# The 00.0 is often the PF of the device the VFs will have the funciont in the
+# BDF incremented by some values so e.g. the very first VF is 0000:41:00.4
+
+cd /sys/bus/pci/devices/0000:41:00.0/
+
+for vf in $(ls -d virtfn*)
+do
+        BDF=$(basename $(readlink -f $vf))
+        for md in $(ls -d $vf/mdev_supported_types/*)
+        do
+                AVAIL=$(cat $md/available_instances)
+                NAME=$(cat $md/name)
+                DIR=$(basename $md)
+
+                if [ $AVAIL -gt 0 ]; then
+                        echo "| BDF          | INSTANCES | NAME           | DIR        |"
+                        echo "+--------------+-----------+----------------+------------+"
+                        printf "| %12s |%10d |%15s | %10s |\n\n" "$BDF" "$AVAIL" "$NAME" "$DIR"
+                fi
+
+        done
+done
+```
+
+If there are available instances you get something like this (for the first VF),
+beware that the output is highly dependent on the GPU you have, if there is no
+output check again if `MIG` is really disabled.
+
+```sh
+| BDF          | INSTANCES | NAME           | DIR        |
++--------------+-----------+----------------+------------+
+| 0000:41:00.4 |         1 |  GRID A100D-4C | nvidia-692 |
+
+| BDF          | INSTANCES | NAME           | DIR        |
++--------------+-----------+----------------+------------+
+| 0000:41:00.4 |         1 |  GRID A100D-8C | nvidia-693 |
+
+| BDF          | INSTANCES | NAME           | DIR        |
++--------------+-----------+----------------+------------+
+| 0000:41:00.4 |         1 | GRID A100D-10C | nvidia-694 |
+
+| BDF          | INSTANCES | NAME           | DIR        |
++--------------+-----------+----------------+------------+
+| 0000:41:00.4 |         1 | GRID A100D-16C | nvidia-695 |
+
+| BDF          | INSTANCES | NAME           | DIR        |
++--------------+-----------+----------------+------------+
+| 0000:41:00.4 |         1 | GRID A100D-20C | nvidia-696 |
+
+| BDF          | INSTANCES | NAME           | DIR        |
++--------------+-----------+----------------+------------+
+| 0000:41:00.4 |         1 | GRID A100D-40C | nvidia-697 |
+
+| BDF          | INSTANCES | NAME           | DIR        |
++--------------+-----------+----------------+------------+
+| 0000:41:00.4 |         1 | GRID A100D-80C | nvidia-698 |
+
+```
+
+Change to the `mdev_supported_types` directory for the virtual function on which
+you want to create the `vGPU`. Taking the first output as an example:
+
+```sh
+$ cd virtfn0/mdev_supported_types/nvidia-692
+$ UUIDGEN=$(uuidgen)
+$ sudo bash -c "echo $UUIDGEN > create"
+```
+
+Confirm that the `vGPU` was created. You should see the `UUID` pointing to a
+subdirectory of the `sysfs` space.
+
+```sh
+$ ls -l /sys/bus/mdev/devices/
+```
+
+Get the `IOMMU` group number and verify there is a `VFIO` device created to use
+with Kata.
+
+```sh
+$ ls -l /sys/bus/mdev/devices/*/
+$ ls -l /dev/vfio
+```
+
+Use the `VFIO` device created in the same way as in the pass-through use-case.
+Beware that the guest needs the NVIDIA guest drivers, so one would need to build
+a new guest `OS` image.
+
+### NVIDIA vGPU MIG-backed
+
+We're not going into detail what `MIG` is but briefly it is a technology to
+partition the hardware into independent instances with guaranteed quality of
+service. For more details see [NVIDIA Multi-Instance GPU User Guide](https://docs.nvidia.com/datacenter/tesla/mig-user-guide/).
+
+First enable `MIG` mode for a GPU, depending on the platform you're running
+a reboot would be necessary. Some platforms support GPU reset.
+
+```sh
+$ sudo nvidia-smi -mig 1
+```
+
+If the platform supports a GPU reset one can run, otherwise you will get a
+warning to reboot the server.
+
+```sh
+$ sudo nvidia-smi --gpu-reset
+```
+
+The driver per default provides a number of profiles that users can opt-in when
+configuring the MIG feature.
+
+```sh
+$ sudo nvidia-smi mig -lgip
++-----------------------------------------------------------------------------+
+| GPU instance profiles:                                                      |
+| GPU   Name             ID    Instances   Memory     P2P    SM    DEC   ENC  |
+|                              Free/Total   GiB              CE    JPEG  OFA  |
+|=============================================================================|
+|   0  MIG 1g.10gb       19     7/7        9.50       No     14     0     0   |
+|                                                             1     0     0   |
++-----------------------------------------------------------------------------+
+|   0  MIG 1g.10gb+me    20     1/1        9.50       No     14     1     0   |
+|                                                             1     1     1   |
++-----------------------------------------------------------------------------+
+|   0  MIG 2g.20gb       14     3/3        19.50      No     28     1     0   |
+|                                                             2     0     0   |
++-----------------------------------------------------------------------------+
+                              ...
+```
+
+Create the GPU instances that correspond to the `vGPU` types of the `MIG-backed`
+`vGPUs` that you will create [NVIDIA A100 PCIe 80GB Virtual GPU Types](https://docs.nvidia.com/grid/13.0/grid-vgpu-user-guide/index.html#vgpu-types-nvidia-a100-pcie-80gb).
+
+```sh
+# MIG 1g.10gb --> vGPU A100D-1-10C
+$ sudo nvidia-smi mig -cgi 19
+```
+
+List the GPU instances and get the GPU instance id to create the compute
+instance.
+
+```sh
+$ sudo nvidia-smi mig -lgi # list the created GPU instances
+$ sudo nvidia-smi mig -cci -gi 9 # each GPU instance can have several compute
+                                 # instances. Instance -> Workload
+```
+
+Verify that the compute instances were created within the GPU instance
+
+```sh
+$ nvidia-smi
+                              ... snip ...
++-----------------------------------------------------------------------------+
+| MIG devices:                                                                |
++------------------+----------------------+-----------+-----------------------+
+| GPU  GI  CI  MIG |         Memory-Usage |        Vol|         Shared        |
+|      ID  ID  Dev |           BAR1-Usage | SM     Unc| CE  ENC  DEC  OFA  JPG|
+|                  |                      |        ECC|                       |
+|==================+======================+===========+=======================|
+|  0    9   0   0  |      0MiB /  9728MiB | 14      0 |  1   0    0    0    0 |
+|                  |      0MiB /  4095MiB |           |                       |
++------------------+----------------------+-----------+-----------------------+
+                              ... snip ...
+```
+
+We can use the [snippet](#list-all-available-vgpu-instances) from before to list
+the available `vGPU` instances, this time `MIG-backed`.
+
+```sh
+| BDF          | INSTANCES | NAME           | DIR        |
++--------------+-----------+----------------+------------+
+| 0000:41:00.4 |         1 |GRID A100D-1-10C | nvidia-699 |
+
+| BDF          | INSTANCES | NAME           | DIR        |
++--------------+-----------+----------------+------------+
+| 0000:41:00.5 |         1 |GRID A100D-1-10C | nvidia-699 |
+
+| BDF          | INSTANCES | NAME           | DIR        |
++--------------+-----------+----------------+------------+
+| 0000:41:01.6 |         1 |GRID A100D-1-10C | nvidia-699 |
+                       ... snip ...
+```
+
+Repeat the steps after the [snippet](#list-all-available-vgpu-instances) listing
+to create the corresponding `mdev` device and use the guest `OS` created in the
+previous section with `time-sliced` `vGPUs`.
+
+## Install NVIDIA Driver + Toolkit in Kata Containers Guest OS
+
+Consult the [Developer-Guide](https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md#create-a-rootfs-image) on how to create a
+rootfs base image for a distribution of your choice. This is going to be used as
+a base for a NVIDIA enabled guest OS. Use the `EXTRA_PKGS` variable to install
+all the needed packages to compile the drivers. Also copy the kernel development
+packages from the previous `make deb-pkg` into `$ROOTFS_DIR`.
+
+```sh
+export EXTRA_PKGS="gcc make curl gnupg"
+```
+
+Having the `$ROOTFS_DIR` exported in the previous step we can now install all the
+needed parts in the guest OS. In this case, we have an Ubuntu based rootfs.
+
+First off all mount the special filesystems into the rootfs
+
+```sh
+$ sudo mount -t sysfs -o ro none ${ROOTFS_DIR}/sys
+$ sudo mount -t proc -o ro none ${ROOTFS_DIR}/proc
+$ sudo mount -t tmpfs none ${ROOTFS_DIR}/tmp
+$ sudo mount -o bind,ro /dev ${ROOTFS_DIR}/dev
+$ sudo mount -t devpts none ${ROOTFS_DIR}/dev/pts
+```
+
+Now we can enter `chroot`
+
+```sh
+$ sudo chroot ${ROOTFS_DIR}
+```
+
+Inside the rootfs one is going to install the drivers and toolkit to enable the
+easy creation of GPU containers with Kata. We can also use this rootfs for any
+other container not specifically only for GPUs.
+
+As a prerequisite install the copied kernel development packages
+
+```sh
+$ sudo dpkg -i *.deb
+```
+
+Get the driver run file, since we need to build the driver against a kernel that
+is not running on the host we need the ability to specify the exact version we
+want the driver to build against. Take the kernel version one used for building
+the NVIDIA kernel (`5.15.23-nvidia-gpu`).
+
+```sh
+$ wget https://us.download.nvidia.com/XFree86/Linux-x86_64/510.54/NVIDIA-Linux-x86_64-510.54.run
+$ chmod +x NVIDIA-Linux-x86_64-510.54.run
+# Extract the source files so we can run the installer with arguments
+$ ./NVIDIA-Linux-x86_64-510.54.run -x
+$ cd NVIDIA-Linux-x86_64-510.54
+$ ./nvidia-installer -k 5.15.23-nvidia-gpu
+```
+
+Having the drivers installed we need to install the toolkit which will take care
+of providing the right bits into the container.
+
+```sh
+$ distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
+$ curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
+$ curl -s -L https://nvidia.github.io/libnvidia-container/$distribution/libnvidia-container.list | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
+$ apt update
+$ apt install nvidia-container-toolkit
+```
+
+Create the hook execution file for Kata:
+
+```
+# Content of $ROOTFS_DIR/usr/share/oci/hooks/prestart/nvidia-container-toolkit.sh
+
+#!/bin/bash -x
+
+/usr/bin/nvidia-container-toolkit -debug $@
+```
+
+As the last step one can do some cleanup of files or package caches. Build the
+rootfs and configure it for use with Kata according to the development guide.
+
+Enable the `guest_hook_path` in Kata's `configuration.toml`
+
+```sh
+guest_hook_path = "/usr/share/oci/hooks"
+```
+
+One has built a NVIDIA rootfs, kernel and now we can run any GPU container
+without installing the drivers into the container. Check NVIDIA device status
+with `nvidia-smi`
+
+```sh
+$  sudo ctr --debug run --runtime "io.containerd.kata.v2"  --device /dev/vfio/192  --rm -t "docker.io/nvidia/cuda:11.6.0-base-ubuntu20.04" cuda nvidia-smi
+Fri Mar 18 10:36:59 2022
++-----------------------------------------------------------------------------+
+| NVIDIA-SMI 510.54       Driver Version: 510.54       CUDA Version: 11.6     |
+|-------------------------------+----------------------+----------------------+
+| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
+| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
+|                               |                      |               MIG M. |
+|===============================+======================+======================|
+|   0  NVIDIA A30X         Off  | 00000000:02:00.0 Off |                    0 |
+| N/A   38C    P0    67W / 230W |      0MiB / 24576MiB |      0%      Default |
+|                               |                      |             Disabled |
++-------------------------------+----------------------+----------------------+
+
++-----------------------------------------------------------------------------+
+| Processes:                                                                  |
+|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
+|        ID   ID                                                   Usage      |
+|=============================================================================|
+|  No running processes found                                                 |
++-----------------------------------------------------------------------------+
+```
+
+As the last step one can remove the additional packages and files that were added
+to the `$ROOTFS_DIR` to keep it as small as possible.
+
+## References
+
+- [Configuring a VM for GPU Pass-Through by Using the QEMU Command Line](https://docs.nvidia.com/grid/latest/grid-vgpu-user-guide/index.html#using-gpu-pass-through-red-hat-el-qemu-cli)
+- https://gitlab.com/nvidia/container-images/driver/-/tree/master
+- https://github.com/NVIDIA/nvidia-docker/wiki/Driver-containers

docs/use-cases/Nvidia-GPU-passthrough-and-Kata.md

@@ -1,293 +0,0 @@
-# Using Nvidia GPU device with Kata Containers
-
-An Nvidia GPU device can be passed to a Kata Containers container using GPU passthrough
-(Nvidia GPU pass-through mode) as well as GPU mediated passthrough (Nvidia vGPU mode). 
-
-Nvidia GPU pass-through mode, an entire physical GPU is directly assigned to one VM,
-bypassing the Nvidia Virtual GPU Manager. In this mode of operation, the GPU is accessed
-exclusively by the Nvidia driver running in the VM to which it is assigned.
-The GPU is not shared among VMs.
-
-Nvidia Virtual GPU (vGPU) enables multiple virtual machines (VMs) to have simultaneous,
-direct access to a single physical GPU, using the same Nvidia graphics drivers that are
-deployed on non-virtualized operating systems. By doing this, Nvidia vGPU provides VMs
-with unparalleled graphics performance, compute performance, and application compatibility,
-together with the cost-effectiveness and scalability brought about by sharing a GPU
-among multiple workloads.
-
-| Technology | Description | Behaviour | Detail |
-| --- | --- | --- | --- |
-| Nvidia GPU pass-through mode | GPU passthrough | Physical GPU assigned to a single VM | Direct GPU assignment to VM without limitation |
-| Nvidia vGPU mode | GPU sharing | Physical GPU shared by multiple VMs | Mediated passthrough |
-
-## Hardware Requirements
-Nvidia GPUs Recommended for Virtualization:
-
-- Nvidia Tesla (T4, M10, P6, V100 or newer)
-- Nvidia Quadro RTX 6000/8000
-
-## Host BIOS Requirements
-
-Some hardware requires a larger PCI BARs window, for example, Nvidia Tesla P100, K40m
-```
-$ lspci -s 04:00.0 -vv | grep Region
-      Region 0: Memory at c6000000 (32-bit, non-prefetchable) [size=16M]
-      Region 1: Memory at 383800000000 (64-bit, prefetchable) [size=16G] #above 4G
-      Region 3: Memory at 383c00000000 (64-bit, prefetchable) [size=32M]
-```
-
-For large BARs devices, MMIO mapping above 4G address space should be `enabled`
-in the PCI configuration of the BIOS.
-
-Some hardware vendors use different name in BIOS, such as:
-
-- Above 4G Decoding
-- Memory Hole for PCI MMIO
-- Memory Mapped I/O above 4GB
-
-The following steps outline the workflow for using an Nvidia GPU with Kata.
-
-## Host Kernel Requirements
-The following configurations need to be enabled on your host kernel:
-
-- `CONFIG_VFIO`
-- `CONFIG_VFIO_IOMMU_TYPE1`
-- `CONFIG_VFIO_MDEV`
-- `CONFIG_VFIO_MDEV_DEVICE`
-- `CONFIG_VFIO_PCI`
-
-Your host kernel needs to be booted with `intel_iommu=on` on the kernel command line.
-
-## Install and configure Kata Containers
-To use non-large BARs devices (for example, Nvidia Tesla T4), you need Kata version 1.3.0 or above.
-Follow the [Kata Containers setup instructions](../install/README.md)
-to install the latest version of Kata.
-
-To use large BARs devices (for example, Nvidia Tesla P100), you need Kata version 1.11.0 or above.
-
-The following configuration in the Kata `configuration.toml` file as shown below can work:
-
-Hotplug for PCI devices by `acpi_pcihp` (Linux's ACPI PCI Hotplug driver):
-```
-machine_type = "q35"
-
-hotplug_vfio_on_root_bus = false
-```
-
-Hotplug for PCIe devices by `pciehp` (Linux's PCIe Hotplug driver):
-```
-machine_type = "q35"
-
-hotplug_vfio_on_root_bus = true
-pcie_root_port = 1
-```
-
-## Build Kata Containers kernel with GPU support
-The default guest kernel installed with Kata Containers does not provide GPU support.
-To use an Nvidia GPU with Kata Containers, you need to build a kernel with the
-necessary GPU support.
-
-The following kernel config options need to be enabled:
-```
-# Support PCI/PCIe device hotplug (Required for large BARs device)
-CONFIG_HOTPLUG_PCI_PCIE=y
-
-# Support for loading modules (Required for load Nvidia drivers)
-CONFIG_MODULES=y
-CONFIG_MODULE_UNLOAD=y
-
-# Enable the MMIO access method for PCIe devices (Required for large BARs device)
-CONFIG_PCI_MMCONFIG=y
-```
-
-The following kernel config options need to be disabled:
-```
-# Disable Open Source Nvidia driver nouveau
-# It conflicts with Nvidia official driver
-CONFIG_DRM_NOUVEAU=n
-```
-> **Note**: `CONFIG_DRM_NOUVEAU` is normally disabled by default.
-It is worth checking that it is not enabled in your kernel configuration to prevent any conflicts.
-
-
-Build the Kata Containers kernel with the previous config options,
-using the instructions described in [Building Kata Containers kernel](../../tools/packaging/kernel).
-For further details on building and installing guest kernels,
-see [the developer guide](../Developer-Guide.md#install-guest-kernel-images).
-
-There is an easy way to build a guest kernel that supports Nvidia GPU:
-```
-## Build guest kernel with ../../tools/packaging/kernel
-
-# Prepare (download guest kernel source, generate .config)
-$ ./build-kernel.sh -v 4.19.86 -g nvidia -f setup
-
-# Build guest kernel
-$ ./build-kernel.sh -v 4.19.86 -g nvidia build
-
-# Install guest kernel
-$ sudo -E ./build-kernel.sh -v 4.19.86 -g nvidia install
-/usr/share/kata-containers/vmlinux-nvidia-gpu.container -> vmlinux-4.19.86-70-nvidia-gpu
-/usr/share/kata-containers/vmlinuz-nvidia-gpu.container -> vmlinuz-4.19.86-70-nvidia-gpu
-```
-
-To build Nvidia Driver in Kata container, `kernel-devel` is required.  
-This is a way to generate rpm packages for `kernel-devel`:
-```
-$ cd kata-linux-4.19.86-68
-$ make rpm-pkg
-Output RPMs:
-~/rpmbuild/RPMS/x86_64/kernel-devel-4.19.86_nvidia_gpu-1.x86_64.rpm
-```
-> **Note**:
-> - `kernel-devel` should be installed in Kata container before run Nvidia driver installer.
-> - Run `make deb-pkg` to build the deb package.
-
-Before using the new guest kernel, please update the `kernel` parameters in `configuration.toml`.
-```
-kernel = "/usr/share/kata-containers/vmlinuz-nvidia-gpu.container"
-```
-
-## Nvidia GPU pass-through mode with Kata Containers
-Use the following steps to pass an Nvidia GPU device in pass-through mode with Kata:
-
-1. Find the Bus-Device-Function (BDF) for GPU device on host:
-   ```
-   $ sudo lspci -nn -D | grep -i nvidia
-   0000:04:00.0 3D controller [0302]: NVIDIA Corporation Device [10de:15f8] (rev a1)
-   0000:84:00.0 3D controller [0302]: NVIDIA Corporation Device [10de:15f8] (rev a1)
-   ```
-   > PCI address `0000:04:00.0` is assigned to the hardware GPU device.
-   > `10de:15f8` is the device ID of the hardware GPU device.
-
-2. Find the IOMMU group for the GPU device:
-   ```
-   $ BDF="0000:04:00.0"
-   $ readlink -e /sys/bus/pci/devices/$BDF/iommu_group
-   /sys/kernel/iommu_groups/45
-   ```
-   The previous output shows that the GPU belongs to IOMMU group 45.
-
-3. Check the IOMMU group number under `/dev/vfio`:
-   ```
-   $ ls -l /dev/vfio
-   total 0
-   crw------- 1 root root 248,   0 Feb 28 09:57 45
-   crw------- 1 root root 248,   1 Feb 28 09:57 54
-   crw-rw-rw- 1 root root  10, 196 Feb 28 09:57 vfio
-   ```
-
-4. Start a Kata container with GPU device:
-   ```
-   $ sudo docker run -it --runtime=kata-runtime --cap-add=ALL --device /dev/vfio/45 centos /bin/bash
-   ```
-
-5. Run `lspci` within the container to verify the GPU device is seen in the list
-   of the PCI devices. Note the vendor-device id of the GPU (`10de:15f8`) in the `lspci` output.
-   ```
-   $ lspci -nn -D | grep '10de:15f8'
-   0000:01:01.0 3D controller [0302]: NVIDIA Corporation GP100GL [Tesla P100 PCIe 16GB] [10de:15f8] (rev a1)
-   ```
-
-6. Additionally, you can check the PCI BARs space of ​​the Nvidia GPU device in the container:
-   ```
-   $ lspci -s 01:01.0 -vv | grep Region
-		Region 0: Memory at c0000000 (32-bit, non-prefetchable) [disabled] [size=16M]
-		Region 1: Memory at 4400000000 (64-bit, prefetchable) [disabled] [size=16G]
-		Region 3: Memory at 4800000000 (64-bit, prefetchable) [disabled] [size=32M]
-   ```
-   > **Note**: If you see a message similar to the above, the BAR space of the Nvidia
-   > GPU has been successfully allocated.
-
-## Nvidia vGPU mode with Kata Containers
-
-Nvidia vGPU is a licensed product on all supported GPU boards. A software license
-is required to enable all vGPU features within the guest VM.
-
-> **Note**: There is no suitable test environment, so it is not written here.
-
-
-## Install Nvidia Driver in Kata Containers
-Download the official Nvidia driver from
-[https://www.nvidia.com/Download/index.aspx](https://www.nvidia.com/Download/index.aspx),
-for example `NVIDIA-Linux-x86_64-418.87.01.run`.
-
-Install the `kernel-devel`(generated in the previous steps) for guest kernel:
-```
-$ sudo rpm -ivh kernel-devel-4.19.86_gpu-1.x86_64.rpm
-```
-
-Here is an example to extract, compile and install Nvidia driver:
-```
-## Extract
-$ sh ./NVIDIA-Linux-x86_64-418.87.01.run -x
-
-## Compile and install (It will take some time)
-$ cd NVIDIA-Linux-x86_64-418.87.01
-$ sudo ./nvidia-installer -a -q --ui=none \
- --no-cc-version-check \
- --no-opengl-files --no-install-libglvnd \
- --kernel-source-path=/usr/src/kernels/`uname -r`
-```
-
-Or just run one command line:
-```
-$ sudo sh ./NVIDIA-Linux-x86_64-418.87.01.run -a -q --ui=none \
- --no-cc-version-check \
- --no-opengl-files --no-install-libglvnd \
- --kernel-source-path=/usr/src/kernels/`uname -r`
-```
-
-To view detailed logs of the installer:
-```
-$ tail -f /var/log/nvidia-installer.log
-```
-
-Load Nvidia driver module manually
-```
-# Optional（generate modules.dep and map files for Nvidia driver）
-$ sudo depmod
-
-# Load module
-$ sudo modprobe nvidia-drm
-
-# Check module
-$ lsmod | grep nvidia
-nvidia_drm             45056  0
-nvidia_modeset       1093632  1 nvidia_drm
-nvidia              18202624  1 nvidia_modeset
-drm_kms_helper        159744  1 nvidia_drm
-drm                   364544  3 nvidia_drm,drm_kms_helper
-i2c_core               65536  3 nvidia,drm_kms_helper,drm
-ipmi_msghandler        49152  1 nvidia
-```
-
-
-Check Nvidia device status with `nvidia-smi`
-```
-$ nvidia-smi
-Tue Mar  3 00:03:49 2020
-+-----------------------------------------------------------------------------+
-| NVIDIA-SMI 418.87.01    Driver Version: 418.87.01    CUDA Version: 10.1     |
-|-------------------------------+----------------------+----------------------+
-| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
-| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
-|===============================+======================+======================|
-|   0  Tesla P100-PCIE...  Off  | 00000000:01:01.0 Off |                    0 |
-| N/A   27C    P0    25W / 250W |      0MiB / 16280MiB |      0%      Default |
-+-------------------------------+----------------------+----------------------+
-
-+-----------------------------------------------------------------------------+
-| Processes:                                                       GPU Memory |
-|  GPU       PID   Type   Process name                             Usage      |
-|=============================================================================|
-|  No running processes found                                                 |
-+-----------------------------------------------------------------------------+
-
-```
-
-## References
-
-- [Configuring a VM for GPU Pass-Through by Using the QEMU Command Line](https://docs.nvidia.com/grid/latest/grid-vgpu-user-guide/index.html#using-gpu-pass-through-red-hat-el-qemu-cli)
-- https://gitlab.com/nvidia/container-images/driver/-/tree/master
-- https://github.com/NVIDIA/nvidia-docker/wiki/Driver-containers

docs/use-cases/using-Intel-QAT-and-kata.md

@@ -231,7 +231,7 @@ $ cp ${GOPATH}/${LINUX_VER}/vmlinux ${KATA_KERNEL_LOCATION}/${KATA_KERNEL_NAME}
 These instructions build upon the OS builder instructions located in the 
 [Developer Guide](../Developer-Guide.md). At this point it is recommended that
 [Docker](https://docs.docker.com/engine/install/ubuntu/) is installed first, and
-then [Kata-deploy](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy)
+then [Kata-deploy](../../tools/packaging/kata-deploy)
 is use to install Kata. This will make sure that the correct `agent` version 
 is installed into the rootfs in the steps below.
 
@@ -312,7 +312,7 @@ working properly with the Kata Containers VM.
 
 ### Build OpenSSL Intel® QAT engine container
 
-Use the OpenSSL Intel® QAT [Dockerfile](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/master/demo/openssl-qat-engine) 
+Use the OpenSSL Intel® QAT [Dockerfile](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/main/demo/openssl-qat-engine) 
 to build a container image with an optimized OpenSSL engine for 
 Intel® QAT. Using `docker build` with the Kata Containers runtime can sometimes
 have issues. Therefore, make sure that `runc` is the default Docker container 
@@ -419,11 +419,11 @@ You might need to disable Docker before initializing Kubernetes. Be aware
 that the OpenSSL container image built above will need to be exported from
 Docker and imported into containerd.
 
-If Kata is installed through [`kata-deploy`](https://github.com/kata-containers/kata-containers/blob/stable-2.0/tools/packaging/kata-deploy/README.md)
+If Kata is installed through [`kata-deploy`](../../tools/packaging/kata-deploy/README.md)
 there will be multiple `configuration.toml` files associated with different 
 hypervisors. Rather than add in the custom Kata kernel, Kata rootfs, and 
 kernel modules to each `configuration.toml` as the default, instead use
-[annotations](https://github.com/kata-containers/kata-containers/blob/stable-2.0/docs/how-to/how-to-load-kernel-modules-with-kata.md)
+[annotations](../how-to/how-to-load-kernel-modules-with-kata.md)
 in the Kubernetes YAML file to tell Kata which kernel and rootfs to use. The 
 easy way to do this is to use `kata-deploy` which will install the Kata binaries
 to `/opt` and properly configure the `/etc/containerd/config.toml` with annotation 
@@ -444,7 +444,7 @@ $ sudo docker save -o openssl-qat-engine.tar openssl-qat-engine:latest
 $ sudo ctr -n=k8s.io images import openssl-qat-engine.tar
 ```
 
-The [Intel® QAT Plugin](https://github.com/intel/intel-device-plugins-for-kubernetes/blob/master/cmd/qat_plugin/README.md)
+The [Intel® QAT Plugin](https://github.com/intel/intel-device-plugins-for-kubernetes/blob/main/cmd/qat_plugin/README.md)
 needs to be started so that the virtual functions can be discovered and
 used by Kubernetes. 
 

docs/use-cases/using-Intel-SGX-and-kata.md

@@ -17,24 +17,11 @@ CONFIG_X86_SGX_KVM=y
 ```
 
 * Kubernetes cluster configured with:
-   * [`kata-deploy`](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy) based Kata Containers installation
+   * [`kata-deploy`](../../tools/packaging/kata-deploy) based Kata Containers installation
    * [Intel SGX Kubernetes device plugin](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/main/cmd/sgx_plugin#deploying-with-pre-built-images)
 
 > Note: Kata Containers supports creating VM sandboxes with Intel® SGX enabled
-> using [cloud-hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor/) VMM only. QEMU support is waiting to get the
-> Intel SGX enabled QEMU upstream release.
-
-## Installation
-
-### Kata Containers Guest Kernel
-
-Follow the instructions to [setup](../../tools/packaging/kernel/README.md#setup-kernel-source-code) and [build](../../tools/packaging/kernel/README.md#build-the-kernel) the experimental guest kernel. Then, install as:
-
-```sh
-$ sudo cp kata-linux-experimental-*/vmlinux /opt/kata/share/kata-containers/vmlinux.sgx
-$ sudo sed -i 's|vmlinux.container|vmlinux.sgx|g' \
-  /opt/kata/share/defaults/kata-containers/configuration-clh.toml
-```
+> using [cloud-hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor/) and [QEMU](https://www.qemu.org/) VMMs only.
 
 ### Kata Containers Configuration
 
@@ -48,6 +35,8 @@ to the `sandbox` are: `["io.katacontainers.*", "sgx.intel.com/epc"]`.
 
 With the following sample job deployed using `kubectl apply -f`:
 
+> Note: Change the `runtimeClassName` option accordingly, only `kata-clh` and `kata-qemu` support Intel® SGX.
+
 ```yaml
 apiVersion: batch/v1
 kind: Job

docs/use-cases/using-vpp-and-kata.md

@@ -1,76 +0,0 @@
-# Setup to run VPP
-
-The Data Plane Development Kit (DPDK) is a set of libraries and drivers for
-fast packet processing. Vector Packet Processing (VPP) is a platform
-extensible framework that provides out-of-the-box production quality
-switch and router functionality. VPP is a high performance packet-processing
-stack that can run on commodity CPUs. Enabling VPP with DPDK support can
-yield significant performance improvements over a Linux\* bridge providing a
-switch with DPDK VHOST-USER ports.
-
-For more information about VPP visit their [wiki](https://wiki.fd.io/view/VPP).
-
-## Install and configure Kata Containers
-
-Follow  the [Kata Containers setup instructions](../Developer-Guide.md).
-
-In order to make use of VHOST-USER based interfaces, the container needs to be backed
-by huge pages. `HugePages` support is required for the large memory pool allocation used for
-DPDK packet buffers.  This is a feature which must be configured within the Linux Kernel. See
-[the DPDK documentation](https://doc.dpdk.org/guides/linux_gsg/sys_reqs.html#use-of-hugepages-in-the-linux-environment)
-for details on how to enable for the host. After enabling huge pages support on the host system,
-update the Kata configuration to enable huge page support in the guest kernel:
-
-```
-$ sudo sed -i -e 's/^# *\(enable_hugepages\).*=.*$/\1 = true/g' /usr/share/defaults/kata-containers/configuration.toml
-```
-
-
-## Install VPP
-
-Follow the [VPP installation instructions](https://wiki.fd.io/view/VPP/Installing_VPP_binaries_from_packages).
-
-After a successful installation, your host system is ready to start
-connecting Kata Containers with VPP bridges.
-
-### Install the VPP Docker\* plugin
-
-To create a Docker network and connect Kata Containers easily to that network through
-Docker, install a VPP Docker plugin.
-
-To install the plugin, follow the [plugin installation instructions](https://github.com/clearcontainers/vpp).
-
-This VPP plugin allows the creation of a VPP network. Every container added
-to this network is connected through an L2 bridge-domain provided by VPP.
-
-## Example: Launch two Kata Containers using VPP
-
-To use VPP, use Docker to create a network that makes use of VPP.
-For example:
-
-```
-$ sudo docker network create -d=vpp --ipam-driver=vpp --subnet=192.168.1.0/24 --gateway=192.168.1.1  vpp_net
-```
-
-Test connectivity by launching two containers:
-```
-$ sudo docker run --runtime=kata-runtime --net=vpp_net --ip=192.168.1.2 --mac-address=CA:FE:CA:FE:01:02 -it busybox bash -c "ip a; ip route; sleep 300"
-
-$ sudo docker run --runtime=kata-runtime --net=vpp_net --ip=192.168.1.3 --mac-address=CA:FE:CA:FE:01:03 -it busybox bash -c "ip a; ip route; ping 192.168.1.2"
-```
-
-These commands setup two Kata Containers connected via a VPP L2 bridge
-domain. The first of the two VMs displays the networking details and then
-sleeps providing a period of time for it to be pinged. The second
-VM displays its networking details and then pings the first VM, verifying
-connectivity between them.
-
-After verifying connectivity, cleanup with the following commands:
-
-```
-$ sudo docker kill $(sudo docker ps --no-trunc -aq)
-$ sudo docker rm $(sudo docker ps --no-trunc -aq)
-$ sudo docker network rm vpp_net
-$ sudo service vpp stop
-```
-

docs/use-cases/zun_kata.md

@@ -1,121 +0,0 @@
-# OpenStack Zun DevStack working with Kata Containers
-
-## Introduction
-
-This guide describes how to get Kata Containers to work with OpenStack Zun
-using DevStack on Ubuntu 16.04. Running DevStack with this guide will setup
-Docker and Clear Containers 2.0, which you replace with Kata Containers.
-Currently, the instructions are based on the following links:
-
-- https://docs.openstack.org/zun/latest/contributor/quickstart.html
-
-- https://docs.openstack.org/zun/latest/admin/clear-containers.html
-
-## Install Git to use with DevStack
-
-```sh
-$ sudo apt install git
-```
-
-## Setup OpenStack DevStack
-The following commands will sync DevStack from GitHub, create your
-`local.conf` file, assign your host IP to this file, enable Clear
-Containers, start DevStack, and set the environment variables to use
-`zun` on the command line.
-
-```sh
-$ sudo mkdir -p /opt/stack
-$ sudo chown $USER /opt/stack
-$ git clone https://github.com/openstack-dev/devstack /opt/stack/devstack
-$ HOST_IP="$(ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1 -d'/')"
-$ git clone https://github.com/openstack/zun /opt/stack/zun
-$ cat /opt/stack/zun/devstack/local.conf.sample \
-$     | sed "s/HOST_IP=.*/HOST_IP=$HOST_IP/" \
-$     > /opt/stack/devstack/local.conf
-$ sed -i "s/KURYR_CAPABILITY_SCOPE=.*/KURYR_CAPABILITY_SCOPE=local/" /opt/stack/devstack/local.conf
-$ echo "ENABLE_CLEAR_CONTAINER=true" >> /opt/stack/devstack/local.conf
-$ echo "enable_plugin zun-ui https://git.openstack.org/openstack/zun-ui" >> /opt/stack/devstack/local.conf
-$ /opt/stack/devstack/stack.sh
-$ source /opt/stack/devstack/openrc admin admin
-```
-
-The previous commands start OpenStack DevStack with Zun support. You can test
-it using `runc` as shown by the following commands to make sure everything
-installed correctly and is working.
-
-```sh
-$ zun run --name test cirros ping -c 4 8.8.8.8
-$ zun list
-$ zun logs test
-$ zun delete test
-```
-
-## Install Kata Containers
-
-Follow [these instructions](../install/README.md)
-to install the Kata Containers components.
-
-## Update Docker with new Kata Containers runtime
-
-The following commands replace the Clear Containers 2.x runtime setup with
-DevStack, with Kata Containers:
-
-```sh
-$ sudo sed -i 's/"cor"/"kata-runtime"/' /etc/docker/daemon.json
-$ sudo sed -i 's/"\/usr\/bin\/cc-oci-runtime"/"\/usr\/bin\/kata-runtime"/' /etc/docker/daemon.json
-$ sudo systemctl daemon-reload
-$ sudo systemctl restart docker
-```
-
-## Test that everything works in both Docker and OpenStack Zun
-
-```sh
-$ sudo docker run -ti --runtime kata-runtime busybox sh
-$ zun run --name kata --runtime kata-runtime cirros ping -c 4 8.8.8.8
-$ zun list
-$ zun logs kata
-$ zun delete kata
-```
-
-## Stop DevStack and clean up system (Optional)
-
-```sh
-$ /opt/stack/devstack/unstack.sh
-$ /opt/stack/devstack/clean.sh
-```
-
-## Restart DevStack and reset CC 2.x runtime to `kata-runtime`
-
-Run the following commands if you already setup Kata Containers and want to
-restart DevStack:
-
-```sh
-$ /opt/stack/devstack/unstack.sh
-$ /opt/stack/devstack/clean.sh
-$ /opt/stack/devstack/stack.sh
-$ source /opt/stack/devstack/openrc admin admin
-$ sudo sed -i 's/"cor"/"kata-runtime"/' /etc/docker/daemon.json
-$ sudo sed -i 's/"\/usr\/bin\/cc-oci-runtime"/"\/usr\/bin\/kata-runtime"/' /etc/docker/daemon.json
-$ sudo systemctl daemon-reload
-$ sudo systemctl restart docker
-```
-
-![Kata Zun image 1](./images/kata-zun1.png)
-
-Figure 1: Create a BusyBox container image
-
-![Kata Zun image 2](./images/kata-zun2.png)
-
-Figure 2: Select `kata-runtime` to use
-
-![Kata Zun image 3](./images/kata-zun3.png)
-
-Figure 3: Two BusyBox containers successfully launched
-
-![Kata Zun image 4](./images/kata-zun4.png)
-
-Figure 4: Test connectivity between Kata Containers
-
-![Kata Zun image 5](./images/kata-zun5.png)
-
-Figure 5: CLI for Zun

snap/local/README.md

@@ -22,21 +22,35 @@ $ sudo snap install kata-containers --classic
 
 ## Build and install snap image
 
-Run next command at the root directory of the packaging repository.
+Run the command below which will use the packaging Makefile to build the snap image:
 
 ```sh
-$ make snap
+$ make -C tools/packaging snap
 ```
 
+> **Warning:**
+>
+> By default, `snapcraft` will create a clean virtual machine
+> environment to build the snap in using the `multipass` tool.
+>
+> However, `multipass` is silently disabled when `--destructive-mode` is
+> used.
+>
+> Since building the Kata Containers package currently requires
+> `--destructive-mode`, the snap will be built using the host
+> environment. To avoid parts of the build auto-detecting additional
+> features to enable (for example for QEMU), we recommend that you
+> only run the snap build in a minimal host environment.
+
 To install the resulting snap image, snap must be put in [classic mode][3] and the
-security confinement must be disabled (*--classic*). Also since the resulting snap
-has not been signed the verification of signature must be omitted (*--dangerous*).
+security confinement must be disabled (`--classic`). Also since the resulting snap
+has not been signed the verification of signature must be omitted (`--dangerous`).
 
 ```sh
-$ sudo snap install --classic --dangerous kata-containers_[VERSION]_[ARCH].snap
+$ sudo snap install --classic --dangerous "kata-containers_${version}_${arch}.snap"
 ```
 
-Replace `VERSION` with the current version of Kata Containers and `ARCH` with
+Replace `${version}` with the current version of Kata Containers and `${arch}` with
 the system architecture.
 
 ## Configure Kata Containers
@@ -76,12 +90,12 @@ then a new configuration file can be [created](#configure-kata-containers)
 and [configured][7].
 
 [1]: https://docs.snapcraft.io/snaps/intro
-[2]: ../docs/design/architecture/README.md#root-filesystem-image
+[2]: ../../docs/design/architecture/README.md#root-filesystem-image
 [3]: https://docs.snapcraft.io/reference/confinement#classic
-[4]: https://github.com/kata-containers/runtime#configuration
+[4]: https://github.com/kata-containers/kata-containers/tree/main/src/runtime#configuration
 [5]: https://docs.docker.com/engine/reference/commandline/dockerd
-[6]: ../docs/install/docker/ubuntu-docker-install.md
-[7]: ../docs/Developer-Guide.md#configure-to-use-initrd-or-rootfs-image
+[6]: ../../docs/install/docker/ubuntu-docker-install.md
+[7]: ../../docs/Developer-Guide.md#configure-to-use-initrd-or-rootfs-image
 [8]: https://snapcraft.io/kata-containers
-[9]: ../docs/Developer-Guide.md#run-kata-containers-with-docker
-[10]: ../docs/Developer-Guide.md#run-kata-containers-with-kubernetes
+[9]: ../../docs/Developer-Guide.md#run-kata-containers-with-docker
+[10]: ../../docs/Developer-Guide.md#run-kata-containers-with-kubernetes

snap/local/snap-common.sh

@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2022 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Description: Idempotent script to be sourced by all parts in a
+#   snapcraft config file.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# XXX: Bash-specific code. zsh doesn't support this option and that *does*
+# matter if this script is run sourced... since it'll be using zsh! ;)
+[ -n "$BASH_VERSION" ] && set -o errtrace
+
+[ -n "${DEBUG:-}" ] && set -o xtrace
+
+die()
+{
+	echo >&2 "ERROR: $0: $*"
+}
+
+[ -n "${SNAPCRAFT_STAGE:-}" ] ||\
+	die "must be sourced from a snapcraft config file"
+
+snap_yq_version=3.4.1
+
+snap_common_install_yq()
+{
+      export yq="${SNAPCRAFT_STAGE}/bin/yq"
+
+      local yq_pkg
+      yq_pkg="github.com/mikefarah/yq"
+
+      local yq_url
+      yq_url="https://${yq_pkg}/releases/download/${snap_yq_version}/yq_${goos}_${goarch}"
+      curl -o "${yq}" -L "${yq_url}"
+      chmod +x "${yq}"
+}
+
+# Function that should be called for each snap "part" in
+# snapcraft.yaml.
+snap_common_main()
+{
+	# Architecture
+	arch="$(uname -m)"
+
+	case "${arch}" in
+		aarch64)
+			goarch="arm64"
+			qemu_arch="${arch}"
+			;;
+
+		ppc64le)
+			goarch="ppc64le"
+			qemu_arch="ppc64"
+			;;
+
+		s390x)
+			goarch="${arch}"
+			qemu_arch="${arch}"
+			;;
+
+		x86_64)
+			goarch="amd64"
+			qemu_arch="${arch}"
+			;;
+
+		*) die "unsupported architecture: ${arch}" ;;
+	esac
+
+	dpkg_arch=$(dpkg --print-architecture)
+
+	# golang
+	#
+	# We need the O/S name in golang format, but since we don't
+	# know if the godeps part has run, we don't know if golang is
+	# available yet, hence fall back to a standard system command.
+	goos="$(go env GOOS &>/dev/null || true)"
+	[ -z "$goos" ] && goos=$(uname -s|tr '[A-Z]' '[a-z]')
+
+	export GOROOT="${SNAPCRAFT_STAGE}"
+	export GOPATH="${GOROOT}/gopath"
+	export GO111MODULE="auto"
+
+	mkdir -p "${GOPATH}/bin"
+	export PATH="${GOPATH}/bin:${PATH}"
+
+	# Proxy
+	export http_proxy="${http_proxy:-}"
+	export https_proxy="${https_proxy:-}"
+
+	# Binaries
+	mkdir -p "${SNAPCRAFT_STAGE}/bin"
+
+	export PATH="$PATH:${SNAPCRAFT_STAGE}/bin"
+
+	# YAML query tool
+	export yq="${SNAPCRAFT_STAGE}/bin/yq"
+
+	# Kata paths
+	export kata_dir=$(printf "%s/src/github.com/%s/%s" \
+		"${GOPATH}" \
+		"${SNAPCRAFT_PROJECT_NAME}" \
+		"${SNAPCRAFT_PROJECT_NAME}")
+
+	export versions_file="${kata_dir}/versions.yaml"
+
+	[ -n "${yq:-}" ] && [ -x "${yq:-}" ] || snap_common_install_yq
+}
+
+snap_common_main

snap/snapcraft.yaml

@@ -1,4 +1,5 @@
 name: kata-containers
+website: https://github.com/kata-containers/kata-containers
 summary: Build lightweight VMs that seamlessly plug into the containers ecosystem
 description: |
   Kata Containers is an open source project and community working to build a
@@ -18,20 +19,18 @@ parts:
       - git
       - git-extras
     override-pull: |
-      version="9999"
-      kata_url="https://github.com/kata-containers/kata-containers"
+      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
 
-      if echo "${GITHUB_REF}" | grep -q -E "^refs/tags"; then
-        version=$(echo ${GITHUB_REF} | cut -d/ -f3)
+      version="9999"
+
+      if echo "${GITHUB_REF:-}" | grep -q -E "^refs/tags"; then
+        version=$(echo ${GITHUB_REF:-} | cut -d/ -f3)
         git checkout ${version}
       fi
 
       snapcraftctl set-grade "stable"
       snapcraftctl set-version "${version}"
 
-      # setup GOPATH - this repo dir should be there
-      export GOPATH=${SNAPCRAFT_STAGE}/gopath
-      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
       mkdir -p $(dirname ${kata_dir})
       ln -sf $(realpath "${SNAPCRAFT_STAGE}/..") ${kata_dir}
 
@@ -43,27 +42,12 @@ parts:
     build-packages:
       - curl
     override-build: |
+      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
+
       # put everything in stage
-      cd ${SNAPCRAFT_STAGE}
+      cd "${SNAPCRAFT_STAGE}"
 
-      yq_path="./yq"
-      yq_pkg="github.com/mikefarah/yq"
-      goos="linux"
-      case "$(uname -m)" in
-        aarch64) goarch="arm64";;
-        ppc64le) goarch="ppc64le";;
-        x86_64) goarch="amd64";;
-        s390x) goarch="s390x";;
-        *) echo "unsupported architecture: $(uname -m)"; exit 1;;
-      esac
-
-      yq_version=3.4.1
-      yq_url="https://${yq_pkg}/releases/download/${yq_version}/yq_${goos}_${goarch}"
-      curl -o "${yq_path}" -L "${yq_url}"
-      chmod +x "${yq_path}"
-
-      kata_dir=gopath/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
-      version="$(${yq_path} r ${kata_dir}/versions.yaml languages.golang.meta.newest-version)"
+      version="$(${yq} r ${kata_dir}/versions.yaml languages.golang.meta.newest-version)"
       tarfile="go${version}.${goos}-${goarch}.tar.gz"
       curl -LO https://golang.org/dl/${tarfile}
       tar -xf ${tarfile} --strip-components=1
@@ -80,28 +64,17 @@ parts:
       - uidmap
       - gnupg2
     override-build: |
-      [ "$(uname -m)" = "ppc64le" ] || [ "$(uname -m)" = "s390x" ] && sudo apt-get --no-install-recommends install -y protobuf-compiler
+      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
 
-      yq=${SNAPCRAFT_STAGE}/yq
+      [ "${arch}" = "ppc64le" ] || [ "${arch}" = "s390x" ] && sudo apt-get --no-install-recommends install -y protobuf-compiler
 
-      # set GOPATH
-      export GOPATH=${SNAPCRAFT_STAGE}/gopath
-      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
-
-      export GOROOT=${SNAPCRAFT_STAGE}
-      export PATH="${GOROOT}/bin:${PATH}"
-      export GO111MODULE="auto"
-
-      http_proxy=${http_proxy:-""}
-      https_proxy=${https_proxy:-""}
       if [ -n "$http_proxy" ]; then
         echo "Setting proxy $http_proxy"
-        sudo -E systemctl set-environment http_proxy=$http_proxy || true
-        sudo -E systemctl set-environment https_proxy=$https_proxy || true
+        sudo -E systemctl set-environment http_proxy="$http_proxy" || true
+        sudo -E systemctl set-environment https_proxy="$https_proxy" || true
       fi
 
       # Copy yq binary. It's used in the container
-      mkdir -p "${GOPATH}/bin/"
       cp -a "${yq}" "${GOPATH}/bin/"
 
       echo "Unmasking docker service"
@@ -112,63 +85,54 @@ parts:
       echo "Starting docker"
       sudo -E systemctl start docker || true
 
-      cd ${kata_dir}/tools/osbuilder
+      cd "${kata_dir}/tools/osbuilder"
 
       # build image
       export AGENT_INIT=yes
       export USE_DOCKER=1
       export DEBUG=1
-      arch="$(uname -m)"
       initrd_distro=$(${yq} r -X ${kata_dir}/versions.yaml assets.initrd.architecture.${arch}.name)
       image_distro=$(${yq} r -X ${kata_dir}/versions.yaml assets.image.architecture.${arch}.name)
       case "$arch" in
         x86_64)
           # In some build systems it's impossible to build a rootfs image, try with the initrd image
-          sudo -E PATH=$PATH make image DISTRO=${image_distro} || sudo -E PATH=$PATH make initrd DISTRO=${initrd_distro}
+          sudo -E PATH=$PATH make image DISTRO="${image_distro}" || sudo -E PATH="$PATH" make initrd DISTRO="${initrd_distro}"
         ;;
 
         aarch64|ppc64le|s390x)
-          sudo -E PATH=$PATH make initrd DISTRO=${initrd_distro}
+          sudo -E PATH="$PATH" make initrd DISTRO="${initrd_distro}"
         ;;
 
-        *) echo "unsupported architecture: $(uname -m)"; exit 1;;
+        *) die "unsupported architecture: ${arch}" ;;
       esac
 
       # Install image
-      kata_image_dir=${SNAPCRAFT_PART_INSTALL}/usr/share/kata-containers
-      mkdir -p ${kata_image_dir}
-      cp kata-containers*.img ${kata_image_dir}
+      kata_image_dir="${SNAPCRAFT_PART_INSTALL}/usr/share/kata-containers"
+      mkdir -p "${kata_image_dir}"
+      cp kata-containers*.img "${kata_image_dir}"
 
   runtime:
     after: [godeps, image, cloud-hypervisor]
     plugin: nil
     build-attributes: [no-patchelf]
     override-build: |
-      # set GOPATH
-      export GOPATH=${SNAPCRAFT_STAGE}/gopath
-      export GOROOT=${SNAPCRAFT_STAGE}
-      export PATH="${GOROOT}/bin:${PATH}"
-      export GO111MODULE="auto"
-      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
+      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
 
-      cd ${kata_dir}/src/runtime
+      cd "${kata_dir}/src/runtime"
 
-      # setup arch
-      arch=$(uname -m)
-      if [ ${arch} = "ppc64le" ]; then
-        arch="ppc64"
-      fi
+      qemu_cmd="qemu-system-${qemu_arch}"
 
       # build and install runtime
       make \
-        PREFIX=/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr \
+        PREFIX="/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr" \
         SKIP_GO_VERSION_CHECK=1 \
-        QEMUCMD=qemu-system-$arch
+        QEMUCMD="${qemu_cmd}"
+
       make install \
         PREFIX=/usr \
-        DESTDIR=${SNAPCRAFT_PART_INSTALL} \
+        DESTDIR="${SNAPCRAFT_PART_INSTALL}" \
         SKIP_GO_VERSION_CHECK=1 \
-        QEMUCMD=qemu-system-$arch
+        QEMUCMD="${qemu_cmd}"
 
       if [ ! -f ${SNAPCRAFT_PART_INSTALL}/../../image/install/usr/share/kata-containers/kata-containers.img ]; then
         sed -i -e "s|^image =.*|initrd = \"/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr/share/kata-containers/kata-containers-initrd.img\"|" \
@@ -185,51 +149,37 @@ parts:
       - bison
       - flex
     override-build: |
-      yq=${SNAPCRAFT_STAGE}/yq
-      export PATH="${PATH}:${SNAPCRAFT_STAGE}"
-      export GOPATH=${SNAPCRAFT_STAGE}/gopath
-      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
-      versions_file="${kata_dir}/versions.yaml"
+      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
+
       kernel_version="$(${yq} r $versions_file assets.kernel.version)"
       #Remove extra 'v'
-      kernel_version=${kernel_version#v}
+      kernel_version="${kernel_version#v}"
 
-      [ "$(uname -m)" = "s390x" ] && sudo apt-get --no-install-recommends install -y libssl-dev
+      [ "${arch}" = "s390x" ] && sudo apt-get --no-install-recommends install -y libssl-dev
 
-      export GOPATH=${SNAPCRAFT_STAGE}/gopath
-      export GO111MODULE="auto"
-      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
-
-      cd ${kata_dir}/tools/packaging/kernel
+      cd "${kata_dir}/tools/packaging/kernel"
       kernel_dir_prefix="kata-linux-"
 
       # Setup and build kernel
-      if [ "$(uname -m)" = "x86_64" ]; then
-        kernel_version="$(${yq} r $versions_file assets.kernel-experimental.tag)"
-        kernel_version=${kernel_version#v}
-        kernel_dir_prefix="kata-linux-experimental-"
-        ./build-kernel.sh -e -v ${kernel_version} -d setup
-      else
-        ./build-kernel.sh -v ${kernel_version} -d setup
-      fi
+      ./build-kernel.sh -v "${kernel_version}" -d setup
       cd ${kernel_dir_prefix}*
       make -j $(($(nproc)-1)) EXTRAVERSION=".container"
 
-      kernel_suffix=${kernel_version}.container
-      kata_kernel_dir=${SNAPCRAFT_PART_INSTALL}/usr/share/kata-containers
-      mkdir -p ${kata_kernel_dir}
+      kernel_suffix="${kernel_version}.container"
+      kata_kernel_dir="${SNAPCRAFT_PART_INSTALL}/usr/share/kata-containers"
+      mkdir -p "${kata_kernel_dir}"
 
       # Install bz kernel
-      make install INSTALL_PATH=${kata_kernel_dir} EXTRAVERSION=".container" || true
-      vmlinuz_name=vmlinuz-${kernel_suffix}
-      ln -sf ${vmlinuz_name} ${kata_kernel_dir}/vmlinuz.container
+      make install INSTALL_PATH="${kata_kernel_dir}" EXTRAVERSION=".container" || true
+      vmlinuz_name="vmlinuz-${kernel_suffix}"
+      ln -sf "${vmlinuz_name}" "${kata_kernel_dir}/vmlinuz.container"
 
       # Install raw kernel
-      vmlinux_path=vmlinux
-      [ "$(uname -m)" = "s390x" ] && vmlinux_path=arch/s390/boot/compressed/vmlinux
-      vmlinux_name=vmlinux-${kernel_suffix}
-      cp ${vmlinux_path} ${kata_kernel_dir}/${vmlinux_name}
-      ln -sf ${vmlinux_name} ${kata_kernel_dir}/vmlinux.container
+      vmlinux_path="vmlinux"
+      [ "${arch}" = "s390x" ] && vmlinux_path="arch/s390/boot/compressed/vmlinux"
+      vmlinux_name="vmlinux-${kernel_suffix}"
+      cp "${vmlinux_path}" "${kata_kernel_dir}/${vmlinux_name}"
+      ln -sf "${vmlinux_name}" "${kata_kernel_dir}/vmlinux.container"
 
   qemu:
     plugin: make
@@ -256,40 +206,23 @@ parts:
       - libselinux1-dev
       - ninja-build
     override-build: |
-      yq=${SNAPCRAFT_STAGE}/yq
-      export GOPATH=${SNAPCRAFT_STAGE}/gopath
-      export GO111MODULE="auto"
-      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
+      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
 
-      versions_file="${kata_dir}/versions.yaml"
-      # arch-specific definition
-      case "$(uname -m)" in
-        "aarch64")
-          branch="$(${yq} r ${versions_file} assets.hypervisor.qemu.architecture.aarch64.version)"
-          url="$(${yq} r ${versions_file} assets.hypervisor.qemu.url)"
-          commit="$(${yq} r ${versions_file} assets.hypervisor.qemu.architecture.aarch64.commit)"
-          patches_dir="${kata_dir}/tools/packaging/qemu/patches/$(echo ${branch} | sed -e 's/.[[:digit:]]*$//' -e 's/^v//').x"
-          patches_version_dir="${kata_dir}/tools/packaging/qemu/patches/tag_patches/${branch}"
-        ;;
-
-        *)
-          branch="$(${yq} r ${versions_file} assets.hypervisor.qemu.version)"
-          url="$(${yq} r ${versions_file} assets.hypervisor.qemu.url)"
-          commit=""
-          patches_dir="${kata_dir}/tools/packaging/qemu/patches/$(echo ${branch} | sed -e 's/.[[:digit:]]*$//' -e 's/^v//').x"
-          patches_version_dir="${kata_dir}/tools/packaging/qemu/patches/tag_patches/${branch}"
-        ;;
-      esac
+      branch="$(${yq} r ${versions_file} assets.hypervisor.qemu.version)"
+      url="$(${yq} r ${versions_file} assets.hypervisor.qemu.url)"
+      commit=""
+      patches_dir="${kata_dir}/tools/packaging/qemu/patches/$(echo ${branch} | sed -e 's/.[[:digit:]]*$//' -e 's/^v//').x"
+      patches_version_dir="${kata_dir}/tools/packaging/qemu/patches/tag_patches/${branch}"
 
       # download source
-      qemu_dir=${SNAPCRAFT_STAGE}/qemu
+      qemu_dir="${SNAPCRAFT_STAGE}/qemu"
       rm -rf "${qemu_dir}"
-      git clone --branch ${branch} --single-branch ${url} "${qemu_dir}"
-      cd ${qemu_dir}
-      [ -z "${commit}" ] || git checkout ${commit}
+      git clone --depth 1 --branch ${branch} --single-branch ${url} "${qemu_dir}"
+      cd "${qemu_dir}"
+      [ -z "${commit}" ] || git checkout "${commit}"
 
-      [ -n "$(ls -A ui/keycodemapdb)" ] || git clone https://github.com/qemu/keycodemapdb ui/keycodemapdb/
-      [ -n "$(ls -A capstone)" ] || git clone https://github.com/qemu/capstone capstone
+      [ -n "$(ls -A ui/keycodemapdb)" ] || git clone --depth 1 https://github.com/qemu/keycodemapdb ui/keycodemapdb/
+      [ -n "$(ls -A capstone)" ] || git clone --depth 1 https://github.com/qemu/capstone capstone
 
       # Apply branch patches
       [ -d "${patches_version_dir}" ] || mkdir "${patches_version_dir}"
@@ -297,10 +230,10 @@ parts:
       ${kata_dir}/tools/packaging/scripts/apply_patches.sh "${patches_version_dir}"
 
       # Only x86_64 supports libpmem
-      [ "$(uname -m)" = "x86_64" ] && sudo apt-get --no-install-recommends install -y apt-utils ca-certificates libpmem-dev
+      [ "${arch}" = "x86_64" ] && sudo apt-get --no-install-recommends install -y apt-utils ca-certificates libpmem-dev
 
-      configure_hypervisor=${kata_dir}/tools/packaging/scripts/configure-hypervisor.sh
-      chmod +x ${configure_hypervisor}
+      configure_hypervisor="${kata_dir}/tools/packaging/scripts/configure-hypervisor.sh"
+      chmod +x "${configure_hypervisor}"
       # static build. The --prefix, --libdir, --libexecdir, --datadir arguments are
       # based on PREFIX and set by configure-hypervisor.sh
       echo "$(PREFIX=/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr ${configure_hypervisor} -s kata-qemu) \
@@ -310,17 +243,17 @@ parts:
       # Copy QEMU configurations (Kconfigs)
       case "${branch}" in
       "v5.1.0")
-        cp -a ${kata_dir}/tools/packaging/qemu/default-configs/* default-configs
+        cp -a "${kata_dir}"/tools/packaging/qemu/default-configs/* default-configs
         ;;
 
       *)
-        cp -a ${kata_dir}/tools/packaging/qemu/default-configs/* configs/devices/
+        cp -a "${kata_dir}"/tools/packaging/qemu/default-configs/* configs/devices/
         ;;
       esac
 
       # build and install
       make -j $(($(nproc)-1))
-      make install DESTDIR=${SNAPCRAFT_PART_INSTALL}
+      make install DESTDIR="${SNAPCRAFT_PART_INSTALL}"
     prime:
       - -snap/
       - -usr/bin/qemu-ga
@@ -336,21 +269,67 @@ parts:
       # Hack: move qemu to /
       "snap/kata-containers/current/": "./"
 
+  virtiofsd:
+    plugin: nil
+    after: [godeps]
+    override-build: |
+      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
+
+      # Currently, only one platform uses the new rust virtiofsd. The
+      # others make use of QEMU's C implementation.
+      #
+      # See "tools/packaging/scripts/configure-hypervisor.sh".
+      if [ "${arch}" = 'x86_64' ]
+      then
+          echo "INFO: Building rust version of virtiofsd"
+      else
+          echo "INFO: Building QEMU's C version of virtiofsd"
+          # Handled by the 'qemu' part, so nothing more to do here.
+          exit 0
+      fi
+
+      cd "${kata_dir}"
+
+      # Download the rust implementation of virtiofsd
+      tools/packaging/static-build/virtiofsd/build-static-virtiofsd.sh
+      sudo install \
+        --owner='root' \
+        --group='root' \
+        --mode=0755 \
+        -D \
+        --target-directory="${SNAPCRAFT_PART_INSTALL}/usr/libexec/" \
+        virtiofsd/virtiofsd
+
   cloud-hypervisor:
     plugin: nil
     after: [godeps]
     override-build: |
-      export GOPATH=${SNAPCRAFT_STAGE}/gopath
-      yq=${SNAPCRAFT_STAGE}/yq
-      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
-      versions_file="${kata_dir}/versions.yaml"
-      version="$(${yq} r ${versions_file} assets.hypervisor.cloud_hypervisor.version)"
-      url="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/${version}"
-      curl -L ${url}/cloud-hypervisor-static -o cloud-hypervisor
-      curl -LO ${url}/clh-remote
+      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
 
-      install -D cloud-hypervisor ${SNAPCRAFT_PART_INSTALL}/usr/bin/cloud-hypervisor
-      install -D clh-remote ${SNAPCRAFT_PART_INSTALL}/usr/bin/clh-remote
+      if [ "${arch}" == "aarch64" ] || [ "${arch}" == "x86_64" ]; then
+          sudo apt-get -y update
+          sudo apt-get -y install ca-certificates curl gnupg lsb-release
+          curl -fsSL https://download.docker.com/linux/ubuntu/gpg |\
+              sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+          distro_codename=$(lsb_release -cs)
+          echo "deb [arch=${dpkg_arch} signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu ${distro_codename} stable" |\
+              sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get -y update
+          sudo apt-get -y install docker-ce docker-ce-cli containerd.io
+          sudo systemctl start docker.socket
+
+          cd "${SNAPCRAFT_PROJECT_DIR}"
+          sudo -E NO_TTY=true make cloud-hypervisor-tarball
+
+          tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-cloud-hypervisor.tar.xz"
+          tmpdir=$(mktemp -d)
+
+          tar -xvJpf "${tarfile}" -C "${tmpdir}"
+
+          install -D "${tmpdir}/opt/kata/bin/cloud-hypervisor" "${SNAPCRAFT_PART_INSTALL}/usr/bin/cloud-hypervisor"
+
+          rm -rf "${tmpdir}"
+      fi
 
 apps:
   runtime:

src/agent/Cargo.lock

@@ -214,6 +214,12 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
+
 [[package]]
 name = "crc32fast"
 version = "1.3.0"
@@ -233,6 +239,30 @@ dependencies = [
  "crossbeam-utils",
 ]
 
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
+dependencies = [
+ "cfg-if 1.0.0",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97242a70df9b89a65d0b6df3c4bf5b9ce03c5b7309019777fbde37e7537f8762"
+dependencies = [
+ "cfg-if 1.0.0",
+ "crossbeam-utils",
+ "lazy_static",
+ "memoffset",
+ "scopeguard",
+]
+
 [[package]]
 name = "crossbeam-utils"
 version = "0.8.5"
@@ -565,6 +595,7 @@ dependencies = [
  "slog",
  "slog-scope",
  "slog-stdlog",
+ "sysinfo",
  "tempfile",
  "thiserror",
  "tokio",
@@ -1238,6 +1269,31 @@ dependencies = [
  "rand_core",
 ]
 
+[[package]]
+name = "rayon"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90"
+dependencies = [
+ "autocfg",
+ "crossbeam-deque",
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e"
+dependencies = [
+ "crossbeam-channel",
+ "crossbeam-deque",
+ "crossbeam-utils",
+ "lazy_static",
+ "num_cpus",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.2.10"
@@ -1314,6 +1370,7 @@ dependencies = [
  "async-trait",
  "capctl",
  "caps",
+ "cfg-if 0.1.10",
  "cgroups-rs",
  "futures",
  "inotify",
@@ -1518,6 +1575,21 @@ dependencies = [
  "unicode-xid",
 ]
 
+[[package]]
+name = "sysinfo"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e757000a4bed2b1be9be65a3f418b9696adf30bb419214c73997422de73a591"
+dependencies = [
+ "cfg-if 1.0.0",
+ "core-foundation-sys",
+ "libc",
+ "ntapi",
+ "once_cell",
+ "rayon",
+ "winapi",
+]
+
 [[package]]
 name = "take_mut"
 version = "0.2.2"

src/agent/Cargo.toml

@@ -20,6 +20,7 @@ scopeguard = "1.0.0"
 thiserror = "1.0.26"
 regex = "1.5.4"
 serial_test = "0.5.1"
+sysinfo = "0.23.0"
 
 # Async helpers
 async-trait = "0.1.42"
@@ -75,3 +76,8 @@ lto = true
 
 [features]
 seccomp = ["rustjail/seccomp"]
+standard-oci-runtime = ["rustjail/standard-oci-runtime"]
+
+[[bin]]
+name = "kata-agent"
+path = "src/main.rs"

src/agent/Makefile

@@ -14,10 +14,6 @@ PROJECT_COMPONENT = kata-agent
 
 TARGET = $(PROJECT_COMPONENT)
 
-SOURCES := \
-  $(shell find . 2>&1 | grep -E '.*\.rs$$') \
-  Cargo.toml
-
 VERSION_FILE := ./VERSION
 VERSION := $(shell grep -v ^\# $(VERSION_FILE))
 COMMIT_NO := $(shell git rev-parse HEAD 2>/dev/null || true)
@@ -37,8 +33,16 @@ ifeq ($(SECCOMP),yes)
     override EXTRA_RUSTFEATURES += seccomp
 endif
 
+##VAR STANDARD_OCI_RUNTIME=yes|no define if agent enables standard oci runtime feature
+STANDARD_OCI_RUNTIME := no
+
+# Enable standard oci runtime feature of rust build
+ifeq ($(STANDARD_OCI_RUNTIME),yes)
+    override EXTRA_RUSTFEATURES += standard-oci-runtime
+endif
+
 ifneq ($(EXTRA_RUSTFEATURES),)
-    override EXTRA_RUSTFEATURES := --features $(EXTRA_RUSTFEATURES)
+    override EXTRA_RUSTFEATURES := --features "$(EXTRA_RUSTFEATURES)"
 endif
 
 include ../../utils.mk
@@ -98,6 +102,8 @@ define INSTALL_FILE
 	install -D -m 644 $1 $(DESTDIR)$2/$1 || exit 1;
 endef
 
+.DEFAULT_GOAL := default
+
 ##TARGET default: build code
 default: $(TARGET) show-header
 
@@ -106,27 +112,16 @@ $(TARGET): $(GENERATED_CODE) logging-crate-tests $(TARGET_PATH)
 logging-crate-tests:
 	make -C $(CWD)/../libs/logging
 
-$(TARGET_PATH): $(SOURCES) | show-summary
+$(TARGET_PATH): show-summary
 	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
 
 $(GENERATED_FILES): %: %.in
 	@sed $(foreach r,$(GENERATED_REPLACEMENTS),-e 's|@$r@|$($r)|g') "$<" > "$@"
 
 ##TARGET optimize: optimized  build
-optimize: $(SOURCES) | show-summary show-header
+optimize: show-summary show-header
 	@RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
 
-##TARGET clippy: run clippy linter
-clippy: $(GENERATED_CODE)
-	cargo clippy --all-targets --all-features --release \
-		-- \
-		-Aclippy::redundant_allocation \
-		-D warnings
-
-format:
-	cargo fmt -- --check
-
-
 ##TARGET install: install agent
 install: install-services
 	@install -D $(TARGET_PATH) $(DESTDIR)/$(BINDIR)/$(TARGET)
@@ -146,7 +141,7 @@ test:
 	@cargo test --all --target $(TRIPLE) $(EXTRA_RUSTFEATURES) -- --nocapture
 
 ##TARGET check: run test
-check: clippy format
+check: $(GENERATED_FILES) standard_rust_check
 
 ##TARGET run: build and run agent
 run:

src/agent/rustjail/Cargo.toml

@@ -25,6 +25,7 @@ path-absolutize = "1.2.0"
 anyhow = "1.0.32"
 cgroups = { package = "cgroups-rs", version = "0.2.8" }
 rlimit = "0.5.3"
+cfg-if = "0.1.0"
 
 tokio = { version = "1.2.0", features = ["sync", "io-util", "process", "time", "macros"] }
 futures = "0.3.17"
@@ -38,3 +39,4 @@ tempfile = "3.1.0"
 
 [features]
 seccomp = ["libseccomp"]
+standard-oci-runtime = []

src/agent/rustjail/src/cgroups/fs/mod.rs

@@ -391,7 +391,7 @@ fn set_memory_resources(cg: &cgroups::Cgroup, memory: &LinuxMemory, update: bool
 
     if let Some(swappiness) = memory.swappiness {
         if (0..=100).contains(&swappiness) {
-            mem_controller.set_swappiness(swappiness as u64)?;
+            mem_controller.set_swappiness(swappiness)?;
         } else {
             return Err(anyhow!(
                 "invalid value:{}. valid memory swappiness range is 0-100",
@@ -590,9 +590,9 @@ fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> SingularPtrField<CpuUsage> {
 
         let h = lines_to_map(&cpuacct.stat);
         let usage_in_usermode =
-            (((*h.get("user").unwrap() * NANO_PER_SECOND) as f64) / *CLOCK_TICKS) as u64;
+            (((*h.get("user").unwrap_or(&0) * NANO_PER_SECOND) as f64) / *CLOCK_TICKS) as u64;
         let usage_in_kernelmode =
-            (((*h.get("system").unwrap() * NANO_PER_SECOND) as f64) / *CLOCK_TICKS) as u64;
+            (((*h.get("system").unwrap_or(&0) * NANO_PER_SECOND) as f64) / *CLOCK_TICKS) as u64;
 
         let total_usage = cpuacct.usage;
 
@@ -623,9 +623,9 @@ fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> SingularPtrField<CpuUsage> {
     let cpu_controller: &CpuController = get_controller_or_return_singular_none!(cg);
     let stat = cpu_controller.cpu().stat;
     let h = lines_to_map(&stat);
-    let usage_in_usermode = *h.get("user_usec").unwrap();
-    let usage_in_kernelmode = *h.get("system_usec").unwrap();
-    let total_usage = *h.get("usage_usec").unwrap();
+    let usage_in_usermode = *h.get("user_usec").unwrap_or(&0);
+    let usage_in_kernelmode = *h.get("system_usec").unwrap_or(&0);
+    let total_usage = *h.get("usage_usec").unwrap_or(&0);
     let percpu_usage = vec![];
 
     SingularPtrField::some(CpuUsage {
@@ -905,13 +905,7 @@ pub fn get_paths() -> Result<HashMap<String, String>> {
 
         let keys: Vec<&str> = fl[1].split(',').collect();
         for key in &keys {
-            // this is a workaround, cgroup file are using `name=systemd`,
-            // but if file system the name is `systemd`
-            if *key == "name=systemd" {
-                m.insert("systemd".to_string(), fl[2].to_string());
-            } else {
-                m.insert(key.to_string(), fl[2].to_string());
-            }
+            m.insert(key.to_string(), fl[2].to_string());
         }
     }
     Ok(m)

src/agent/rustjail/src/cgroups/notifier.rs

@@ -151,12 +151,12 @@ async fn register_memory_event(
     let eventfd = eventfd(0, EfdFlags::EFD_CLOEXEC)?;
 
     let event_control_path = Path::new(&cg_dir).join("cgroup.event_control");
-    let data;
-    if arg.is_empty() {
-        data = format!("{} {}", eventfd, event_file.as_raw_fd());
+
+    let data = if arg.is_empty() {
+        format!("{} {}", eventfd, event_file.as_raw_fd())
     } else {
-        data = format!("{} {} {}", eventfd, event_file.as_raw_fd(), arg);
-    }
+        format!("{} {} {}", eventfd, event_file.as_raw_fd(), arg)
+    };
 
     fs::write(&event_control_path, data)?;
 

src/agent/rustjail/src/console.rs

@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// Copyright 2021 Sony Group Corporation
+//
+
+use anyhow::{anyhow, Result};
+use nix::errno::Errno;
+use nix::pty;
+use nix::sys::{socket, uio};
+use nix::unistd::{self, dup2};
+use std::os::unix::io::{AsRawFd, RawFd};
+use std::path::Path;
+
+pub fn setup_console_socket(csocket_path: &str) -> Result<Option<RawFd>> {
+    if csocket_path.is_empty() {
+        return Ok(None);
+    }
+
+    let socket_fd = socket::socket(
+        socket::AddressFamily::Unix,
+        socket::SockType::Stream,
+        socket::SockFlag::empty(),
+        None,
+    )?;
+
+    match socket::connect(
+        socket_fd,
+        &socket::SockAddr::Unix(socket::UnixAddr::new(Path::new(csocket_path))?),
+    ) {
+        Ok(()) => Ok(Some(socket_fd)),
+        Err(errno) => Err(anyhow!("failed to open console fd: {}", errno)),
+    }
+}
+
+pub fn setup_master_console(socket_fd: RawFd) -> Result<()> {
+    let pseudo = pty::openpty(None, None)?;
+
+    let pty_name: &[u8] = b"/dev/ptmx";
+    let iov = [uio::IoVec::from_slice(pty_name)];
+    let fds = [pseudo.master];
+    let cmsg = socket::ControlMessage::ScmRights(&fds);
+
+    socket::sendmsg(socket_fd, &iov, &[cmsg], socket::MsgFlags::empty(), None)?;
+
+    unistd::setsid()?;
+    let ret = unsafe { libc::ioctl(pseudo.slave, libc::TIOCSCTTY) };
+    Errno::result(ret).map_err(|e| anyhow!(e).context("ioctl TIOCSCTTY"))?;
+
+    dup2(pseudo.slave, std::io::stdin().as_raw_fd())?;
+    dup2(pseudo.slave, std::io::stdout().as_raw_fd())?;
+    dup2(pseudo.slave, std::io::stderr().as_raw_fd())?;
+
+    unistd::close(socket_fd)?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::os::unix::net::UnixListener;
+    use tempfile::{self, tempdir};
+
+    const CONSOLE_SOCKET: &str = "console-socket";
+
+    #[test]
+    fn test_setup_console_socket() {
+        let dir = tempdir()
+            .map_err(|e| anyhow!(e).context("tempdir failed"))
+            .unwrap();
+        let socket_path = dir.path().join(CONSOLE_SOCKET);
+
+        let _listener = UnixListener::bind(&socket_path).unwrap();
+
+        let ret = setup_console_socket(socket_path.to_str().unwrap());
+
+        assert!(ret.is_ok());
+    }
+}

src/agent/rustjail/src/container.rs

@@ -23,6 +23,8 @@ use crate::cgroups::fs::Manager as FsManager;
 #[cfg(test)]
 use crate::cgroups::mock::Manager as FsManager;
 use crate::cgroups::Manager;
+#[cfg(feature = "standard-oci-runtime")]
+use crate::console;
 use crate::log_child;
 use crate::process::Process;
 #[cfg(feature = "seccomp")]
@@ -40,7 +42,7 @@ use nix::pty;
 use nix::sched::{self, CloneFlags};
 use nix::sys::signal::{self, Signal};
 use nix::sys::stat::{self, Mode};
-use nix::unistd::{self, fork, ForkResult, Gid, Pid, Uid};
+use nix::unistd::{self, fork, ForkResult, Gid, Pid, Uid, User};
 use std::os::unix::fs::MetadataExt;
 use std::os::unix::io::AsRawFd;
 
@@ -62,9 +64,7 @@ use rlimit::{setrlimit, Resource, Rlim};
 use tokio::io::AsyncBufReadExt;
 use tokio::sync::Mutex;
 
-use crate::utils;
-
-const EXEC_FIFO_FILENAME: &str = "exec.fifo";
+pub const EXEC_FIFO_FILENAME: &str = "exec.fifo";
 
 const INIT: &str = "INIT";
 const NO_PIVOT: &str = "NO_PIVOT";
@@ -74,6 +74,7 @@ const CLOG_FD: &str = "CLOG_FD";
 const FIFO_FD: &str = "FIFO_FD";
 const HOME_ENV_KEY: &str = "HOME";
 const PIDNS_FD: &str = "PIDNS_FD";
+const CONSOLE_SOCKET_FD: &str = "CONSOLE_SOCKET_FD";
 
 #[derive(Debug)]
 pub struct ContainerStatus {
@@ -82,7 +83,7 @@ pub struct ContainerStatus {
 }
 
 impl ContainerStatus {
-    fn new() -> Self {
+    pub fn new() -> Self {
         ContainerStatus {
             pre_status: ContainerState::Created,
             cur_status: ContainerState::Created,
@@ -99,6 +100,12 @@ impl ContainerStatus {
     }
 }
 
+impl Default for ContainerStatus {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 pub type Config = CreateOpts;
 type NamespaceType = String;
 
@@ -106,7 +113,7 @@ lazy_static! {
     // This locker ensures the child exit signal will be received by the right receiver.
     pub static ref WAIT_PID_LOCKER: Arc<Mutex<bool>> = Arc::new(Mutex::new(false));
 
-    static ref NAMESPACES: HashMap<&'static str, CloneFlags> = {
+    pub static ref NAMESPACES: HashMap<&'static str, CloneFlags> = {
         let mut m = HashMap::new();
         m.insert("user", CloneFlags::CLONE_NEWUSER);
         m.insert("ipc", CloneFlags::CLONE_NEWIPC);
@@ -119,7 +126,7 @@ lazy_static! {
     };
 
 // type to name hashmap, better to be in NAMESPACES
-    static ref TYPETONAME: HashMap<&'static str, &'static str> = {
+    pub static ref TYPETONAME: HashMap<&'static str, &'static str> = {
         let mut m = HashMap::new();
         m.insert("ipc", "ipc");
         m.insert("user", "user");
@@ -215,7 +222,6 @@ pub trait BaseContainer {
     async fn start(&mut self, p: Process) -> Result<()>;
     async fn run(&mut self, p: Process) -> Result<()>;
     async fn destroy(&mut self) -> Result<()>;
-    fn signal(&self, sig: Signal, all: bool) -> Result<()>;
     fn exec(&mut self) -> Result<()>;
 }
 
@@ -237,6 +243,8 @@ pub struct LinuxContainer {
     pub status: ContainerStatus,
     pub created: SystemTime,
     pub logger: Logger,
+    #[cfg(feature = "standard-oci-runtime")]
+    pub console_socket: PathBuf,
 }
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -360,7 +368,6 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
             )));
         }
     }
-
     log_child!(cfd_log, "child process start run");
     let buf = read_sync(crfd)?;
     let spec_str = std::str::from_utf8(&buf)?;
@@ -380,6 +387,9 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
 
     let cm: FsManager = serde_json::from_str(cm_str)?;
 
+    #[cfg(feature = "standard-oci-runtime")]
+    let csocket_fd = console::setup_console_socket(&std::env::var(CONSOLE_SOCKET_FD)?)?;
+
     let p = if spec.process.is_some() {
         spec.process.as_ref().unwrap()
     } else {
@@ -650,12 +660,17 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         }
     }
 
-    // set the "HOME" env getting from "/etc/passwd", if
-    // there's no uid entry in /etc/passwd, set "/" as the
-    // home env.
     if env::var_os(HOME_ENV_KEY).is_none() {
-        let home_dir = utils::home_dir(guser.uid).unwrap_or_else(|_| String::from("/"));
-        env::set_var(HOME_ENV_KEY, home_dir);
+        // try to set "HOME" env by uid
+        if let Ok(Some(user)) = User::from_uid(Uid::from_raw(guser.uid)) {
+            if let Ok(user_home_dir) = user.dir.into_os_string().into_string() {
+                env::set_var(HOME_ENV_KEY, user_home_dir);
+            }
+        }
+        // set default home dir as "/" if "HOME" env is still empty
+        if env::var_os(HOME_ENV_KEY).is_none() {
+            env::set_var(HOME_ENV_KEY, String::from("/"));
+        }
     }
 
     let exec_file = Path::new(&args[0]);
@@ -671,10 +686,19 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     let _ = unistd::close(crfd);
     let _ = unistd::close(cwfd);
 
-    unistd::setsid().context("create a new session")?;
     if oci_process.terminal {
-        unsafe {
-            libc::ioctl(0, libc::TIOCSCTTY);
+        cfg_if::cfg_if! {
+            if #[cfg(feature = "standard-oci-runtime")] {
+                if let Some(csocket_fd) = csocket_fd {
+                    console::setup_master_console(csocket_fd)?;
+                } else {
+                    return Err(anyhow!("failed to get console master socket fd"));
+                }
+            }
+            else {
+                unistd::setsid().context("create a new session")?;
+                unsafe { libc::ioctl(0, libc::TIOCSCTTY) };
+            }
         }
     }
 
@@ -929,6 +953,14 @@ impl BaseContainer for LinuxContainer {
 
         let exec_path = std::env::current_exe()?;
         let mut child = std::process::Command::new(exec_path);
+
+        #[allow(unused_mut)]
+        let mut console_name = PathBuf::from("");
+        #[cfg(feature = "standard-oci-runtime")]
+        if !self.console_socket.as_os_str().is_empty() {
+            console_name = self.console_socket.clone();
+        }
+
         let mut child = child
             .arg("init")
             .stdin(child_stdin)
@@ -938,7 +970,8 @@ impl BaseContainer for LinuxContainer {
             .env(NO_PIVOT, format!("{}", self.config.no_pivot_root))
             .env(CRFD_FD, format!("{}", crfd))
             .env(CWFD_FD, format!("{}", cwfd))
-            .env(CLOG_FD, format!("{}", cfd_log));
+            .env(CLOG_FD, format!("{}", cfd_log))
+            .env(CONSOLE_SOCKET_FD, console_name);
 
         if p.init {
             child = child.env(FIFO_FD, format!("{}", fifofd));
@@ -1033,7 +1066,19 @@ impl BaseContainer for LinuxContainer {
         let st = self.oci_state()?;
 
         for pid in self.processes.keys() {
-            signal::kill(Pid::from_raw(*pid), Some(Signal::SIGKILL))?;
+            match signal::kill(Pid::from_raw(*pid), Some(Signal::SIGKILL)) {
+                Err(Errno::ESRCH) => {
+                    info!(
+                        self.logger,
+                        "kill encounters ESRCH, pid: {}, container: {}",
+                        pid,
+                        self.id.clone()
+                    );
+                    continue;
+                }
+                Err(err) => return Err(anyhow!(err)),
+                Ok(_) => continue,
+            }
         }
 
         if spec.hooks.is_some() {
@@ -1057,18 +1102,6 @@ impl BaseContainer for LinuxContainer {
         Ok(())
     }
 
-    fn signal(&self, sig: Signal, all: bool) -> Result<()> {
-        if all {
-            for pid in self.processes.keys() {
-                signal::kill(Pid::from_raw(*pid), Some(sig))?;
-            }
-        }
-
-        signal::kill(Pid::from_raw(self.init_process_pid), Some(sig))?;
-
-        Ok(())
-    }
-
     fn exec(&mut self) -> Result<()> {
         let fifo = format!("{}/{}", &self.root, EXEC_FIFO_FILENAME);
         let fd = fcntl::open(fifo.as_str(), OFlag::O_WRONLY, Mode::from_bits_truncate(0))?;
@@ -1432,8 +1465,16 @@ impl LinuxContainer {
                 .unwrap()
                 .as_secs(),
             logger: logger.new(o!("module" => "rustjail", "subsystem" => "container", "cid" => id)),
+            #[cfg(feature = "standard-oci-runtime")]
+            console_socket: Path::new("").to_path_buf(),
         })
     }
+
+    #[cfg(feature = "standard-oci-runtime")]
+    pub fn set_console_socket(&mut self, console_socket: &Path) -> Result<()> {
+        self.console_socket = console_socket.to_path_buf();
+        Ok(())
+    }
 }
 
 fn setgroups(grps: &[libc::gid_t]) -> Result<()> {
@@ -1473,7 +1514,7 @@ use std::process::Stdio;
 use std::time::Duration;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 
-async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
+pub async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
     let logger = logger.new(o!("action" => "execute-hook"));
 
     let binary = PathBuf::from(h.path.as_str());
@@ -1482,15 +1523,15 @@ async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
         return Err(anyhow!(nix::Error::EINVAL));
     }
 
-    let args = h.args.clone();
-    let env: HashMap<String, String> = h
-        .env
-        .iter()
-        .map(|e| {
-            let v: Vec<&str> = e.split('=').collect();
-            (v[0].to_string(), v[1].to_string())
-        })
-        .collect();
+    let mut args = h.args.clone();
+    // the hook.args[0] is the hook binary name which shouldn't be included
+    // in the Command.args
+    if args.len() > 1 {
+        args.remove(0);
+    }
+
+    // all invalid envs will be omitted, only valid envs will be passed to hook.
+    let env: HashMap<&str, &str> = h.env.iter().filter_map(|e| valid_env(e)).collect();
 
     // Avoid the exit signal to be reaped by the global reaper.
     let _wait_locker = WAIT_PID_LOCKER.lock().await;
@@ -1501,8 +1542,7 @@ async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
         .stdin(Stdio::piped())
         .stdout(Stdio::piped())
         .stderr(Stdio::piped())
-        .spawn()
-        .unwrap();
+        .spawn()?;
 
     // default timeout 10s
     let mut timeout: u64 = 10;
@@ -1518,27 +1558,39 @@ async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
     let path = h.path.clone();
 
     let join_handle = tokio::spawn(async move {
-        child
-            .stdin
-            .as_mut()
-            .unwrap()
-            .write_all(state.as_bytes())
-            .await
-            .unwrap();
+        if let Some(mut stdin) = child.stdin.take() {
+            match stdin.write_all(state.as_bytes()).await {
+                Ok(_) => {}
+                Err(e) => {
+                    info!(logger, "write to child stdin failed: {:?}", e);
+                }
+            }
+        }
 
-        // Close stdin so that hook program could receive EOF
-        child.stdin.take();
+        // read something from stdout and stderr for debug
+        if let Some(stdout) = child.stdout.as_mut() {
+            let mut out = String::new();
+            match stdout.read_to_string(&mut out).await {
+                Ok(_) => {
+                    info!(logger, "child stdout: {}", out.as_str());
+                }
+                Err(e) => {
+                    info!(logger, "read from child stdout failed: {:?}", e);
+                }
+            }
+        }
 
-        // read something from stdout for debug
-        let mut out = String::new();
-        child
-            .stdout
-            .as_mut()
-            .unwrap()
-            .read_to_string(&mut out)
-            .await
-            .unwrap();
-        info!(logger, "child stdout: {}", out.as_str());
+        let mut err = String::new();
+        if let Some(stderr) = child.stderr.as_mut() {
+            match stderr.read_to_string(&mut err).await {
+                Ok(_) => {
+                    info!(logger, "child stderr: {}", err.as_str());
+                }
+                Err(e) => {
+                    info!(logger, "read from child stderr failed: {:?}", e);
+                }
+            }
+        }
 
         match child.wait().await {
             Ok(exit) => {
@@ -1547,7 +1599,10 @@ async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
                     .ok_or_else(|| anyhow!("hook exit status has no status code"))?;
 
                 if code != 0 {
-                    error!(logger, "hook {} exit status is {}", &path, code);
+                    error!(
+                        logger,
+                        "hook {} exit status is {}, error message is {}", &path, code, err
+                    );
                     return Err(anyhow!(nix::Error::UnknownErrno));
                 }
 
@@ -1624,13 +1679,47 @@ mod tests {
 
     #[tokio::test]
     async fn test_execute_hook() {
-        let xargs = which("xargs").await;
+        let temp_file = "/tmp/test_execute_hook";
+
+        let touch = which("touch").await;
+
+        defer!(fs::remove_file(temp_file).unwrap(););
+        let invalid_str = vec![97, b'\0', 98];
+        let invalid_string = std::str::from_utf8(&invalid_str).unwrap();
+        let invalid_env = format!("{}=value", invalid_string);
 
         execute_hook(
             &slog_scope::logger(),
             &Hook {
-                path: xargs,
-                args: vec![],
+                path: touch,
+                args: vec!["touch".to_string(), temp_file.to_string()],
+                env: vec![invalid_env],
+                timeout: Some(10),
+            },
+            &OCIState {
+                version: "1.2.3".to_string(),
+                id: "321".to_string(),
+                status: ContainerState::Running,
+                pid: 2,
+                bundle: "".to_string(),
+                annotations: Default::default(),
+            },
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(Path::new(&temp_file).exists(), true);
+    }
+
+    #[tokio::test]
+    async fn test_execute_hook_with_error() {
+        let ls = which("ls").await;
+
+        let res = execute_hook(
+            &slog_scope::logger(),
+            &Hook {
+                path: ls,
+                args: vec!["ls".to_string(), "/tmp/not-exist".to_string()],
                 env: vec![],
                 timeout: None,
             },
@@ -1643,8 +1732,13 @@ mod tests {
                 annotations: Default::default(),
             },
         )
-        .await
-        .unwrap()
+        .await;
+
+        let expected_err = nix::Error::UnknownErrno;
+        assert_eq!(
+            res.unwrap_err().downcast::<nix::Error>().unwrap(),
+            expected_err
+        );
     }
 
     #[tokio::test]
@@ -1655,7 +1749,7 @@ mod tests {
             &slog_scope::logger(),
             &Hook {
                 path: sleep,
-                args: vec!["2".to_string()],
+                args: vec!["sleep".to_string(), "2".to_string()],
                 env: vec![],
                 timeout: Some(1),
             },
@@ -1996,14 +2090,6 @@ mod tests {
         assert!(ret.is_ok(), "Expecting Ok, Got {:?}", ret);
     }
 
-    #[test]
-    fn test_linuxcontainer_signal() {
-        let ret = new_linux_container_and_then(|c: LinuxContainer| {
-            c.signal(nix::sys::signal::SIGCONT, true)
-        });
-        assert!(ret.is_ok(), "Expecting Ok, Got {:?}", ret);
-    }
-
     #[test]
     fn test_linuxcontainer_exec() {
         let ret = new_linux_container_and_then(|mut c: LinuxContainer| c.exec());

src/agent/rustjail/src/lib.rs

@@ -30,6 +30,8 @@ extern crate regex;
 
 pub mod capabilities;
 pub mod cgroups;
+#[cfg(feature = "standard-oci-runtime")]
+pub mod console;
 pub mod container;
 pub mod mount;
 pub mod pipestream;
@@ -39,7 +41,6 @@ pub mod seccomp;
 pub mod specconv;
 pub mod sync;
 pub mod sync_with_async;
-pub mod utils;
 pub mod validator;
 
 use std::collections::HashMap;
@@ -265,7 +266,7 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
             swap: Some(mem.Swap),
             kernel: Some(mem.Kernel),
             kernel_tcp: Some(mem.KernelTCP),
-            swappiness: Some(mem.Swappiness as i64),
+            swappiness: Some(mem.Swappiness),
             disable_oom_killer: Some(mem.DisableOOMKiller),
         })
     } else {
@@ -351,13 +352,12 @@ fn seccomp_grpc_to_oci(sec: &grpc::LinuxSeccomp) -> oci::LinuxSeccomp {
 
         for sys in sec.Syscalls.iter() {
             let mut args = Vec::new();
-            let errno_ret: u32;
 
-            if sys.has_errnoret() {
-                errno_ret = sys.get_errnoret();
+            let errno_ret: u32 = if sys.has_errnoret() {
+                sys.get_errnoret()
             } else {
-                errno_ret = libc::EPERM as u32;
-            }
+                libc::EPERM as u32
+            };
 
             for arg in sys.Args.iter() {
                 args.push(oci::LinuxSeccompArg {
@@ -513,6 +513,7 @@ pub fn grpc_to_oci(grpc: &grpc::Spec) -> oci::Spec {
 
 #[cfg(test)]
 mod tests {
+    use super::*;
     #[macro_export]
     macro_rules! skip_if_not_root {
         () => {
@@ -522,4 +523,595 @@ mod tests {
             }
         };
     }
+
+    // Parameters:
+    //
+    // 1: expected Result
+    // 2: actual Result
+    // 3: string used to identify the test on error
+    #[macro_export]
+    macro_rules! assert_result {
+        ($expected_result:expr, $actual_result:expr, $msg:expr) => {
+            if $expected_result.is_ok() {
+                let expected_value = $expected_result.as_ref().unwrap();
+                let actual_value = $actual_result.unwrap();
+                assert!(*expected_value == actual_value, "{}", $msg);
+            } else {
+                assert!($actual_result.is_err(), "{}", $msg);
+
+                let expected_error = $expected_result.as_ref().unwrap_err();
+                let expected_error_msg = format!("{:?}", expected_error);
+
+                let actual_error_msg = format!("{:?}", $actual_result.unwrap_err());
+
+                assert!(expected_error_msg == actual_error_msg, "{}", $msg);
+            }
+        };
+    }
+
+    #[test]
+    fn test_process_grpc_to_oci() {
+        #[derive(Debug)]
+        struct TestData {
+            grpcproc: grpc::Process,
+            result: oci::Process,
+        }
+
+        let tests = &[
+            TestData {
+                // All fields specified
+                grpcproc: grpc::Process {
+                    Terminal: true,
+                    ConsoleSize: protobuf::SingularPtrField::<grpc::Box>::some(grpc::Box {
+                        Height: 123,
+                        Width: 456,
+                        ..Default::default()
+                    }),
+                    User: protobuf::SingularPtrField::<grpc::User>::some(grpc::User {
+                        UID: 1234,
+                        GID: 5678,
+                        AdditionalGids: Vec::from([910, 1112]),
+                        Username: String::from("username"),
+                        ..Default::default()
+                    }),
+                    Args: protobuf::RepeatedField::from(Vec::from([
+                        String::from("arg1"),
+                        String::from("arg2"),
+                    ])),
+                    Env: protobuf::RepeatedField::from(Vec::from([String::from("env")])),
+                    Cwd: String::from("cwd"),
+                    Capabilities: protobuf::SingularPtrField::some(grpc::LinuxCapabilities {
+                        Bounding: protobuf::RepeatedField::from(Vec::from([String::from("bnd")])),
+                        Effective: protobuf::RepeatedField::from(Vec::from([String::from("eff")])),
+                        Inheritable: protobuf::RepeatedField::from(Vec::from([String::from(
+                            "inher",
+                        )])),
+                        Permitted: protobuf::RepeatedField::from(Vec::from([String::from("perm")])),
+                        Ambient: protobuf::RepeatedField::from(Vec::from([String::from("amb")])),
+                        ..Default::default()
+                    }),
+                    Rlimits: protobuf::RepeatedField::from(Vec::from([
+                        grpc::POSIXRlimit {
+                            Type: String::from("r#type"),
+                            Hard: 123,
+                            Soft: 456,
+                            ..Default::default()
+                        },
+                        grpc::POSIXRlimit {
+                            Type: String::from("r#type2"),
+                            Hard: 789,
+                            Soft: 1011,
+                            ..Default::default()
+                        },
+                    ])),
+                    NoNewPrivileges: true,
+                    ApparmorProfile: String::from("apparmor profile"),
+                    OOMScoreAdj: 123456,
+                    SelinuxLabel: String::from("Selinux Label"),
+                    ..Default::default()
+                },
+                result: oci::Process {
+                    terminal: true,
+                    console_size: Some(oci::Box {
+                        height: 123,
+                        width: 456,
+                    }),
+                    user: oci::User {
+                        uid: 1234,
+                        gid: 5678,
+                        additional_gids: Vec::from([910, 1112]),
+                        username: String::from("username"),
+                    },
+                    args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                    env: Vec::from([String::from("env")]),
+                    cwd: String::from("cwd"),
+                    capabilities: Some(oci::LinuxCapabilities {
+                        bounding: Vec::from([String::from("bnd")]),
+                        effective: Vec::from([String::from("eff")]),
+                        inheritable: Vec::from([String::from("inher")]),
+                        permitted: Vec::from([String::from("perm")]),
+                        ambient: Vec::from([String::from("amb")]),
+                    }),
+                    rlimits: Vec::from([
+                        oci::PosixRlimit {
+                            r#type: String::from("r#type"),
+                            hard: 123,
+                            soft: 456,
+                        },
+                        oci::PosixRlimit {
+                            r#type: String::from("r#type2"),
+                            hard: 789,
+                            soft: 1011,
+                        },
+                    ]),
+                    no_new_privileges: true,
+                    apparmor_profile: String::from("apparmor profile"),
+                    oom_score_adj: Some(123456),
+                    selinux_label: String::from("Selinux Label"),
+                },
+            },
+            TestData {
+                // None ConsoleSize
+                grpcproc: grpc::Process {
+                    ConsoleSize: protobuf::SingularPtrField::<grpc::Box>::none(),
+                    OOMScoreAdj: 0,
+                    ..Default::default()
+                },
+                result: oci::Process {
+                    console_size: None,
+                    oom_score_adj: Some(0),
+                    ..Default::default()
+                },
+            },
+            TestData {
+                // None User
+                grpcproc: grpc::Process {
+                    User: protobuf::SingularPtrField::<grpc::User>::none(),
+                    OOMScoreAdj: 0,
+                    ..Default::default()
+                },
+                result: oci::Process {
+                    user: oci::User {
+                        uid: 0,
+                        gid: 0,
+                        additional_gids: vec![],
+                        username: String::from(""),
+                    },
+                    oom_score_adj: Some(0),
+                    ..Default::default()
+                },
+            },
+            TestData {
+                // None Capabilities
+                grpcproc: grpc::Process {
+                    Capabilities: protobuf::SingularPtrField::none(),
+                    OOMScoreAdj: 0,
+                    ..Default::default()
+                },
+                result: oci::Process {
+                    capabilities: None,
+                    oom_score_adj: Some(0),
+                    ..Default::default()
+                },
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let result = process_grpc_to_oci(&d.grpcproc);
+
+            let msg = format!("{}, result: {:?}", msg, result);
+
+            assert_eq!(d.result, result, "{}", msg);
+        }
+    }
+
+    #[test]
+    fn test_root_grpc_to_oci() {
+        #[derive(Debug)]
+        struct TestData {
+            grpcroot: grpc::Root,
+            result: oci::Root,
+        }
+
+        let tests = &[
+            TestData {
+                // Default fields
+                grpcroot: grpc::Root {
+                    ..Default::default()
+                },
+                result: oci::Root {
+                    ..Default::default()
+                },
+            },
+            TestData {
+                // Specified fields, readonly false
+                grpcroot: grpc::Root {
+                    Path: String::from("path"),
+                    Readonly: false,
+                    ..Default::default()
+                },
+                result: oci::Root {
+                    path: String::from("path"),
+                    readonly: false,
+                    ..Default::default()
+                },
+            },
+            TestData {
+                // Specified fields, readonly true
+                grpcroot: grpc::Root {
+                    Path: String::from("path"),
+                    Readonly: true,
+                    ..Default::default()
+                },
+                result: oci::Root {
+                    path: String::from("path"),
+                    readonly: true,
+                    ..Default::default()
+                },
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let result = root_grpc_to_oci(&d.grpcroot);
+
+            let msg = format!("{}, result: {:?}", msg, result);
+
+            assert_eq!(d.result, result, "{}", msg);
+        }
+    }
+
+    #[test]
+    fn test_hooks_grpc_to_oci() {
+        #[derive(Debug)]
+        struct TestData {
+            grpchooks: grpc::Hooks,
+            result: oci::Hooks,
+        }
+
+        let tests = &[
+            TestData {
+                // Default fields
+                grpchooks: grpc::Hooks {
+                    ..Default::default()
+                },
+                result: oci::Hooks {
+                    ..Default::default()
+                },
+            },
+            TestData {
+                // All specified
+                grpchooks: grpc::Hooks {
+                    Prestart: protobuf::RepeatedField::from(Vec::from([
+                        grpc::Hook {
+                            Path: String::from("prestartpath"),
+                            Args: protobuf::RepeatedField::from(Vec::from([
+                                String::from("arg1"),
+                                String::from("arg2"),
+                            ])),
+                            Env: protobuf::RepeatedField::from(Vec::from([
+                                String::from("env1"),
+                                String::from("env2"),
+                            ])),
+                            Timeout: 10,
+                            ..Default::default()
+                        },
+                        grpc::Hook {
+                            Path: String::from("prestartpath2"),
+                            Args: protobuf::RepeatedField::from(Vec::from([
+                                String::from("arg3"),
+                                String::from("arg4"),
+                            ])),
+                            Env: protobuf::RepeatedField::from(Vec::from([
+                                String::from("env3"),
+                                String::from("env4"),
+                            ])),
+                            Timeout: 25,
+                            ..Default::default()
+                        },
+                    ])),
+                    Poststart: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                        Path: String::from("poststartpath"),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
+                        Timeout: 10,
+                        ..Default::default()
+                    }])),
+                    Poststop: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                        Path: String::from("poststoppath"),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
+                        Timeout: 10,
+                        ..Default::default()
+                    }])),
+                    ..Default::default()
+                },
+                result: oci::Hooks {
+                    prestart: Vec::from([
+                        oci::Hook {
+                            path: String::from("prestartpath"),
+                            args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                            env: Vec::from([String::from("env1"), String::from("env2")]),
+                            timeout: Some(10),
+                        },
+                        oci::Hook {
+                            path: String::from("prestartpath2"),
+                            args: Vec::from([String::from("arg3"), String::from("arg4")]),
+                            env: Vec::from([String::from("env3"), String::from("env4")]),
+                            timeout: Some(25),
+                        },
+                    ]),
+                    poststart: Vec::from([oci::Hook {
+                        path: String::from("poststartpath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
+                    poststop: Vec::from([oci::Hook {
+                        path: String::from("poststoppath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
+                },
+            },
+            TestData {
+                // Prestart empty
+                grpchooks: grpc::Hooks {
+                    Prestart: protobuf::RepeatedField::from(Vec::from([])),
+                    Poststart: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                        Path: String::from("poststartpath"),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
+                        Timeout: 10,
+                        ..Default::default()
+                    }])),
+                    Poststop: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                        Path: String::from("poststoppath"),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
+                        Timeout: 10,
+                        ..Default::default()
+                    }])),
+                    ..Default::default()
+                },
+                result: oci::Hooks {
+                    prestart: Vec::from([]),
+                    poststart: Vec::from([oci::Hook {
+                        path: String::from("poststartpath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
+                    poststop: Vec::from([oci::Hook {
+                        path: String::from("poststoppath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
+                },
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let result = hooks_grpc_to_oci(&d.grpchooks);
+
+            let msg = format!("{}, result: {:?}", msg, result);
+
+            assert_eq!(d.result, result, "{}", msg);
+        }
+    }
+
+    #[test]
+    fn test_mount_grpc_to_oci() {
+        #[derive(Debug)]
+        struct TestData {
+            grpcmount: grpc::Mount,
+            result: oci::Mount,
+        }
+
+        let tests = &[
+            TestData {
+                // Default fields
+                grpcmount: grpc::Mount {
+                    ..Default::default()
+                },
+                result: oci::Mount {
+                    ..Default::default()
+                },
+            },
+            TestData {
+                grpcmount: grpc::Mount {
+                    destination: String::from("destination"),
+                    source: String::from("source"),
+                    field_type: String::from("fieldtype"),
+                    options: protobuf::RepeatedField::from(Vec::from([
+                        String::from("option1"),
+                        String::from("option2"),
+                    ])),
+                    ..Default::default()
+                },
+                result: oci::Mount {
+                    destination: String::from("destination"),
+                    source: String::from("source"),
+                    r#type: String::from("fieldtype"),
+                    options: Vec::from([String::from("option1"), String::from("option2")]),
+                },
+            },
+            TestData {
+                grpcmount: grpc::Mount {
+                    destination: String::from("destination"),
+                    source: String::from("source"),
+                    field_type: String::from("fieldtype"),
+                    options: protobuf::RepeatedField::from(Vec::new()),
+                    ..Default::default()
+                },
+                result: oci::Mount {
+                    destination: String::from("destination"),
+                    source: String::from("source"),
+                    r#type: String::from("fieldtype"),
+                    options: Vec::new(),
+                },
+            },
+            TestData {
+                grpcmount: grpc::Mount {
+                    destination: String::new(),
+                    source: String::from("source"),
+                    field_type: String::from("fieldtype"),
+                    options: protobuf::RepeatedField::from(Vec::from([String::from("option1")])),
+                    ..Default::default()
+                },
+                result: oci::Mount {
+                    destination: String::new(),
+                    source: String::from("source"),
+                    r#type: String::from("fieldtype"),
+                    options: Vec::from([String::from("option1")]),
+                },
+            },
+            TestData {
+                grpcmount: grpc::Mount {
+                    destination: String::from("destination"),
+                    source: String::from("source"),
+                    field_type: String::new(),
+                    options: protobuf::RepeatedField::from(Vec::from([String::from("option1")])),
+                    ..Default::default()
+                },
+                result: oci::Mount {
+                    destination: String::from("destination"),
+                    source: String::from("source"),
+                    r#type: String::new(),
+                    options: Vec::from([String::from("option1")]),
+                },
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let result = mount_grpc_to_oci(&d.grpcmount);
+
+            let msg = format!("{}, result: {:?}", msg, result);
+
+            assert_eq!(d.result, result, "{}", msg);
+        }
+    }
+
+    #[test]
+    fn test_hook_grpc_to_oci<'a>() {
+        #[derive(Debug)]
+        struct TestData<'a> {
+            grpchook: &'a [grpc::Hook],
+            result: Vec<oci::Hook>,
+        }
+
+        let tests = &[
+            TestData {
+                // Default fields
+                grpchook: &[
+                    grpc::Hook {
+                        Timeout: 0,
+                        ..Default::default()
+                    },
+                    grpc::Hook {
+                        Timeout: 0,
+                        ..Default::default()
+                    },
+                ],
+                result: vec![
+                    oci::Hook {
+                        timeout: Some(0),
+                        ..Default::default()
+                    },
+                    oci::Hook {
+                        timeout: Some(0),
+                        ..Default::default()
+                    },
+                ],
+            },
+            TestData {
+                // Specified fields
+                grpchook: &[
+                    grpc::Hook {
+                        Path: String::from("path"),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
+                        Timeout: 10,
+                        ..Default::default()
+                    },
+                    grpc::Hook {
+                        Path: String::from("path2"),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg3"),
+                            String::from("arg4"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env3"),
+                            String::from("env4"),
+                        ])),
+                        Timeout: 20,
+                        ..Default::default()
+                    },
+                ],
+                result: vec![
+                    oci::Hook {
+                        path: String::from("path"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    },
+                    oci::Hook {
+                        path: String::from("path2"),
+                        args: Vec::from([String::from("arg3"), String::from("arg4")]),
+                        env: Vec::from([String::from("env3"), String::from("env4")]),
+                        timeout: Some(20),
+                    },
+                ],
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let result = hook_grpc_to_oci(d.grpchook);
+
+            let msg = format!("{}, result: {:?}", msg, result);
+
+            assert_eq!(d.result, result, "{}", msg);
+        }
+    }
 }

src/agent/rustjail/src/mount.rs

@@ -32,16 +32,21 @@ use crate::log_child;
 
 // Info reveals information about a particular mounted filesystem. This
 // struct is populated from the content in the /proc/<pid>/mountinfo file.
-#[derive(std::fmt::Debug)]
+#[derive(std::fmt::Debug, PartialEq)]
 pub struct Info {
     mount_point: String,
     optional: String,
     fstype: String,
 }
 
-const MOUNTINFOFORMAT: &str = "{d} {d} {d}:{d} {} {} {} {}";
+const MOUNTINFO_FORMAT: &str = "{d} {d} {d}:{d} {} {} {} {}";
+const MOUNTINFO_PATH: &str = "/proc/self/mountinfo";
 const PROC_PATH: &str = "/proc";
 
+const ERR_FAILED_PARSE_MOUNTINFO: &str = "failed to parse mountinfo file";
+const ERR_FAILED_PARSE_MOUNTINFO_FINAL_FIELDS: &str =
+    "failed to parse final fields in mountinfo file";
+
 // since libc didn't defined this const for musl, thus redefined it here.
 #[cfg(all(target_os = "linux", target_env = "gnu", not(target_arch = "s390x")))]
 const PROC_SUPER_MAGIC: libc::c_long = 0x00009fa0;
@@ -518,7 +523,7 @@ pub fn pivot_rootfs<P: ?Sized + NixPath + std::fmt::Debug>(path: &P) -> Result<(
 }
 
 fn rootfs_parent_mount_private(path: &str) -> Result<()> {
-    let mount_infos = parse_mount_table()?;
+    let mount_infos = parse_mount_table(MOUNTINFO_PATH)?;
 
     let mut max_len = 0;
     let mut mount_point = String::from("");
@@ -546,8 +551,8 @@ fn rootfs_parent_mount_private(path: &str) -> Result<()> {
 
 // Parse /proc/self/mountinfo because comparing Dev and ino does not work from
 // bind mounts
-fn parse_mount_table() -> Result<Vec<Info>> {
-    let file = File::open("/proc/self/mountinfo")?;
+fn parse_mount_table(mountinfo_path: &str) -> Result<Vec<Info>> {
+    let file = File::open(mountinfo_path)?;
     let reader = BufReader::new(file);
     let mut infos = Vec::new();
 
@@ -569,7 +574,7 @@ fn parse_mount_table() -> Result<Vec<Info>> {
 
         let (_id, _parent, _major, _minor, _root, mount_point, _opts, optional) = scan_fmt!(
             &line,
-            MOUNTINFOFORMAT,
+            MOUNTINFO_FORMAT,
             i32,
             i32,
             i32,
@@ -578,12 +583,17 @@ fn parse_mount_table() -> Result<Vec<Info>> {
             String,
             String,
             String
-        )?;
+        )
+        .map_err(|_| anyhow!(ERR_FAILED_PARSE_MOUNTINFO))?;
 
         let fields: Vec<&str> = line.split(" - ").collect();
         if fields.len() == 2 {
-            let (fstype, _source, _vfs_opts) =
-                scan_fmt!(fields[1], "{} {} {}", String, String, String)?;
+            let final_fields: Vec<&str> = fields[1].split_whitespace().collect();
+
+            if final_fields.len() != 3 {
+                return Err(anyhow!(ERR_FAILED_PARSE_MOUNTINFO_FINAL_FIELDS));
+            }
+            let fstype = final_fields[0].to_string();
 
             let mut optional_new = String::new();
             if optional != "-" {
@@ -598,7 +608,7 @@ fn parse_mount_table() -> Result<Vec<Info>> {
 
             infos.push(info);
         } else {
-            return Err(anyhow!("failed to parse mount info file".to_string()));
+            return Err(anyhow!(ERR_FAILED_PARSE_MOUNTINFO));
         }
     }
 
@@ -619,7 +629,7 @@ fn chroot<P: ?Sized + NixPath>(_path: &P) -> Result<(), nix::Error> {
 
 pub fn ms_move_root(rootfs: &str) -> Result<bool> {
     unistd::chdir(rootfs)?;
-    let mount_infos = parse_mount_table()?;
+    let mount_infos = parse_mount_table(MOUNTINFO_PATH)?;
 
     let root_path = Path::new(rootfs);
     let abs_root_buf = root_path.absolutize()?;
@@ -1046,10 +1056,12 @@ fn readonly_path(path: &str) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::assert_result;
     use crate::skip_if_not_root;
     use std::fs::create_dir;
     use std::fs::create_dir_all;
     use std::fs::remove_dir_all;
+    use std::io;
     use std::os::unix::fs;
     use std::os::unix::io::AsRawFd;
     use tempfile::tempdir;
@@ -1286,6 +1298,113 @@ mod tests {
         let ret = stat::stat(path);
         assert!(ret.is_ok(), "Should pass. Got: {:?}", ret);
     }
+
+    #[test]
+    fn test_mount_from() {
+        #[derive(Debug)]
+        struct TestData<'a> {
+            source: &'a str,
+            destination: &'a str,
+            r#type: &'a str,
+            flags: MsFlags,
+            error_contains: &'a str,
+
+            // if true, a directory will be created at path in source
+            make_source_directory: bool,
+            // if true, a file will be created at path in source
+            make_source_file: bool,
+        }
+
+        impl Default for TestData<'_> {
+            fn default() -> Self {
+                TestData {
+                    source: "tmp",
+                    destination: "dest",
+                    r#type: "tmpfs",
+                    flags: MsFlags::empty(),
+                    error_contains: "",
+                    make_source_directory: true,
+                    make_source_file: false,
+                }
+            }
+        }
+
+        let tests = &[
+            TestData {
+                ..Default::default()
+            },
+            TestData {
+                flags: MsFlags::MS_BIND,
+                ..Default::default()
+            },
+            TestData {
+                r#type: "bind",
+                ..Default::default()
+            },
+            TestData {
+                r#type: "cgroup2",
+                ..Default::default()
+            },
+            TestData {
+                r#type: "bind",
+                make_source_directory: false,
+                error_contains: &format!("{}", std::io::Error::from_raw_os_error(libc::ENOENT)),
+                ..Default::default()
+            },
+            TestData {
+                r#type: "bind",
+                make_source_directory: false,
+                make_source_file: true,
+                ..Default::default()
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+            let tempdir = tempdir().unwrap();
+
+            let (rfd, wfd) = unistd::pipe2(OFlag::O_CLOEXEC).unwrap();
+            defer!({
+                unistd::close(rfd).unwrap();
+                unistd::close(wfd).unwrap();
+            });
+
+            let source_path = tempdir.path().join(d.source).to_str().unwrap().to_string();
+            if d.make_source_directory {
+                std::fs::create_dir_all(&source_path).unwrap();
+            } else if d.make_source_file {
+                std::fs::write(&source_path, []).unwrap();
+            }
+
+            let mount = Mount {
+                source: source_path,
+                destination: d.destination.to_string(),
+                r#type: d.r#type.to_string(),
+                options: vec![],
+            };
+
+            let result = mount_from(
+                wfd,
+                &mount,
+                tempdir.path().to_str().unwrap(),
+                d.flags,
+                "",
+                "",
+            );
+
+            let msg = format!("{}: result: {:?}", msg, result);
+
+            if d.error_contains.is_empty() {
+                assert!(result.is_ok(), "{}", msg);
+            } else {
+                assert!(result.is_err(), "{}", msg);
+
+                let error_msg = format!("{}", result.unwrap_err());
+                assert!(error_msg.contains(d.error_contains), "{}", msg);
+            }
+        }
+    }
+
     #[test]
     fn test_check_proc_mount() {
         let mount = oci::Mount {
@@ -1401,6 +1520,121 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_parse_mount_table() {
+        #[derive(Debug)]
+        struct TestData<'a> {
+            mountinfo_data: Option<&'a str>,
+            result: Result<Vec<Info>>,
+        }
+
+        let tests = &[
+            TestData {
+                mountinfo_data: Some(
+                    "22 933 0:20 / /sys rw,nodev shared:2 - sysfs sysfs rw,noexec",
+                ),
+                result: Ok(vec![Info {
+                    mount_point: "/sys".to_string(),
+                    optional: "shared:2".to_string(),
+                    fstype: "sysfs".to_string(),
+                }]),
+            },
+            TestData {
+                mountinfo_data: Some(
+                    r#"22 933 0:20 / /sys rw,nodev - sysfs sysfs rw,noexec
+                       81 13 1:2 / /tmp/dir rw shared:2 - tmpfs tmpfs rw"#,
+                ),
+                result: Ok(vec![
+                    Info {
+                        mount_point: "/sys".to_string(),
+                        optional: "".to_string(),
+                        fstype: "sysfs".to_string(),
+                    },
+                    Info {
+                        mount_point: "/tmp/dir".to_string(),
+                        optional: "shared:2".to_string(),
+                        fstype: "tmpfs".to_string(),
+                    },
+                ]),
+            },
+            TestData {
+                mountinfo_data: Some(
+                    "22 933 0:20 /foo\040-\040bar /sys rw,nodev shared:2 - sysfs sysfs rw,noexec",
+                ),
+                result: Ok(vec![Info {
+                    mount_point: "/sys".to_string(),
+                    optional: "shared:2".to_string(),
+                    fstype: "sysfs".to_string(),
+                }]),
+            },
+            TestData {
+                mountinfo_data: Some(""),
+                result: Ok(vec![]),
+            },
+            TestData {
+                mountinfo_data: Some("invalid line data - sysfs sysfs rw"),
+                result: Err(anyhow!(ERR_FAILED_PARSE_MOUNTINFO)),
+            },
+            TestData {
+                mountinfo_data: Some("22 96 0:21 / /sys rw,noexec - sysfs"),
+                result: Err(anyhow!(ERR_FAILED_PARSE_MOUNTINFO_FINAL_FIELDS)),
+            },
+            TestData {
+                mountinfo_data: Some("22 96 0:21 / /sys rw,noexec - sysfs sysfs rw rw"),
+                result: Err(anyhow!(ERR_FAILED_PARSE_MOUNTINFO_FINAL_FIELDS)),
+            },
+            TestData {
+                mountinfo_data: Some("22 96 0:21 / /sys rw,noexec shared:2 - x - x"),
+                result: Err(anyhow!(ERR_FAILED_PARSE_MOUNTINFO)),
+            },
+            TestData {
+                mountinfo_data: Some("-"),
+                result: Err(anyhow!(ERR_FAILED_PARSE_MOUNTINFO)),
+            },
+            TestData {
+                mountinfo_data: Some("--"),
+                result: Err(anyhow!(ERR_FAILED_PARSE_MOUNTINFO)),
+            },
+            TestData {
+                mountinfo_data: Some("- -"),
+                result: Err(anyhow!(ERR_FAILED_PARSE_MOUNTINFO)),
+            },
+            TestData {
+                mountinfo_data: Some(" - "),
+                result: Err(anyhow!(ERR_FAILED_PARSE_MOUNTINFO)),
+            },
+            TestData {
+                mountinfo_data: Some(
+                    r#"22 933 0:20 / /sys rw,nodev - sysfs sysfs rw,noexec
+                       invalid line
+                       81 13 1:2 / /tmp/dir rw shared:2 - tmpfs tmpfs rw"#,
+                ),
+                result: Err(anyhow!(ERR_FAILED_PARSE_MOUNTINFO)),
+            },
+            TestData {
+                mountinfo_data: None,
+                result: Err(anyhow!(io::Error::from_raw_os_error(libc::ENOENT))),
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let tempdir = tempdir().unwrap();
+            let mountinfo_path = tempdir.path().join("mountinfo");
+
+            if let Some(mountinfo_data) = d.mountinfo_data {
+                std::fs::write(&mountinfo_path, mountinfo_data).unwrap();
+            }
+
+            let result = parse_mount_table(mountinfo_path.to_str().unwrap());
+
+            let msg = format!("{}: result: {:?}", msg, result);
+
+            assert_result!(d.result, result, msg);
+        }
+    }
+
     #[test]
     fn test_dev_rel_path() {
         // Valid device paths

src/agent/rustjail/src/process.rs

@@ -5,11 +5,11 @@
 
 use libc::pid_t;
 use std::fs::File;
-use std::os::unix::io::RawFd;
+use std::os::unix::io::{AsRawFd, RawFd};
 use tokio::sync::mpsc::Sender;
 
+use nix::errno::Errno;
 use nix::fcntl::{fcntl, FcntlArg, OFlag};
-use nix::sys::signal::{self, Signal};
 use nix::sys::wait::{self, WaitStatus};
 use nix::unistd::{self, Pid};
 use nix::Result;
@@ -80,7 +80,7 @@ pub struct Process {
 pub trait ProcessOperations {
     fn pid(&self) -> Pid;
     fn wait(&self) -> Result<WaitStatus>;
-    fn signal(&self, sig: Signal) -> Result<()>;
+    fn signal(&self, sig: libc::c_int) -> Result<()>;
 }
 
 impl ProcessOperations for Process {
@@ -92,8 +92,10 @@ impl ProcessOperations for Process {
         wait::waitpid(Some(self.pid()), None)
     }
 
-    fn signal(&self, sig: Signal) -> Result<()> {
-        signal::kill(self.pid(), Some(sig))
+    fn signal(&self, sig: libc::c_int) -> Result<()> {
+        let res = unsafe { libc::kill(self.pid().into(), sig) };
+
+        Errno::result(res).map(drop)
     }
 }
 
@@ -135,19 +137,25 @@ impl Process {
         info!(logger, "before create console socket!");
 
         if !p.tty {
-            info!(logger, "created console socket!");
+            if cfg!(feature = "standard-oci-runtime") {
+                p.stdin = Some(std::io::stdin().as_raw_fd());
+                p.stdout = Some(std::io::stdout().as_raw_fd());
+                p.stderr = Some(std::io::stderr().as_raw_fd());
+            } else {
+                info!(logger, "created console socket!");
 
-            let (stdin, pstdin) = unistd::pipe2(OFlag::O_CLOEXEC)?;
-            p.parent_stdin = Some(pstdin);
-            p.stdin = Some(stdin);
+                let (stdin, pstdin) = unistd::pipe2(OFlag::O_CLOEXEC)?;
+                p.parent_stdin = Some(pstdin);
+                p.stdin = Some(stdin);
 
-            let (pstdout, stdout) = create_extended_pipe(OFlag::O_CLOEXEC, pipe_size)?;
-            p.parent_stdout = Some(pstdout);
-            p.stdout = Some(stdout);
+                let (pstdout, stdout) = create_extended_pipe(OFlag::O_CLOEXEC, pipe_size)?;
+                p.parent_stdout = Some(pstdout);
+                p.stdout = Some(stdout);
 
-            let (pstderr, stderr) = create_extended_pipe(OFlag::O_CLOEXEC, pipe_size)?;
-            p.parent_stderr = Some(pstderr);
-            p.stderr = Some(stderr);
+                let (pstderr, stderr) = create_extended_pipe(OFlag::O_CLOEXEC, pipe_size)?;
+                p.parent_stderr = Some(pstderr);
+                p.stderr = Some(stderr);
+            }
         }
         Ok(p)
     }
@@ -281,6 +289,12 @@ mod tests {
         // signal to every process in the process
         // group of the calling process.
         process.pid = 0;
-        assert!(process.signal(Signal::SIGCONT).is_ok());
+        assert!(process.signal(libc::SIGCONT).is_ok());
+
+        if cfg!(feature = "standard-oci-runtime") {
+            assert_eq!(process.stdin.unwrap(), std::io::stdin().as_raw_fd());
+            assert_eq!(process.stdout.unwrap(), std::io::stdout().as_raw_fd());
+            assert_eq!(process.stderr.unwrap(), std::io::stderr().as_raw_fd());
+        }
     }
 }

src/agent/rustjail/src/specconv.rs

@@ -5,7 +5,7 @@
 
 use oci::Spec;
 
-#[derive(Debug)]
+#[derive(Serialize, Deserialize, Debug, Default, Clone)]
 pub struct CreateOpts {
     pub cgroup_name: String,
     pub use_systemd_cgroup: bool,

src/agent/rustjail/src/utils.rs

@@ -1,119 +0,0 @@
-// Copyright (c) 2021 Ant Group
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-use anyhow::{anyhow, Context, Result};
-use libc::gid_t;
-use libc::uid_t;
-use std::fs::File;
-use std::io::{BufRead, BufReader};
-
-const PASSWD_FILE: &str = "/etc/passwd";
-
-// An entry from /etc/passwd
-#[derive(Debug, PartialEq, PartialOrd)]
-pub struct PasswdEntry {
-    // username
-    pub name: String,
-    // user password
-    pub passwd: String,
-    // user id
-    pub uid: uid_t,
-    // group id
-    pub gid: gid_t,
-    // user Information
-    pub gecos: String,
-    // home directory
-    pub dir: String,
-    // User's Shell
-    pub shell: String,
-}
-
-// get an entry for a given `uid` from `/etc/passwd`
-fn get_entry_by_uid(uid: uid_t, path: &str) -> Result<PasswdEntry> {
-    let file = File::open(path).with_context(|| format!("open file {}", path))?;
-    let mut reader = BufReader::new(file);
-
-    let mut line = String::new();
-    loop {
-        line.clear();
-        match reader.read_line(&mut line) {
-            Ok(0) => return Err(anyhow!(format!("file {} is empty", path))),
-            Ok(_) => (),
-            Err(e) => {
-                return Err(anyhow!(format!(
-                    "failed to read file {} with {:?}",
-                    path, e
-                )))
-            }
-        }
-
-        if line.starts_with('#') {
-            continue;
-        }
-
-        let parts: Vec<&str> = line.split(':').map(|part| part.trim()).collect();
-        if parts.len() != 7 {
-            continue;
-        }
-
-        match parts[2].parse() {
-            Err(_e) => continue,
-            Ok(new_uid) => {
-                if uid != new_uid {
-                    continue;
-                }
-
-                let entry = PasswdEntry {
-                    name: parts[0].to_string(),
-                    passwd: parts[1].to_string(),
-                    uid: new_uid,
-                    gid: parts[3].parse().unwrap_or(0),
-                    gecos: parts[4].to_string(),
-                    dir: parts[5].to_string(),
-                    shell: parts[6].to_string(),
-                };
-
-                return Ok(entry);
-            }
-        }
-    }
-}
-
-pub fn home_dir(uid: uid_t) -> Result<String> {
-    get_entry_by_uid(uid, PASSWD_FILE).map(|entry| entry.dir)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::io::Write;
-    use tempfile::Builder;
-
-    #[test]
-    fn test_get_entry_by_uid() {
-        let tmpdir = Builder::new().tempdir().unwrap();
-        let tmpdir_path = tmpdir.path().to_str().unwrap();
-        let temp_passwd = format!("{}/passwd", tmpdir_path);
-
-        let mut tempf = File::create(temp_passwd.as_str()).unwrap();
-        writeln!(tempf, "root:x:0:0:root:/root0:/bin/bash").unwrap();
-        writeln!(tempf, "root:x:1:0:root:/root1:/bin/bash").unwrap();
-        writeln!(tempf, "#root:x:1:0:root:/rootx:/bin/bash").unwrap();
-        writeln!(tempf, "root:x:2:0:root:/root2:/bin/bash").unwrap();
-        writeln!(tempf, "root:x:3:0:root:/root3").unwrap();
-        writeln!(tempf, "root:x:3:0:root:/root3:/bin/bash").unwrap();
-
-        let entry = get_entry_by_uid(0, temp_passwd.as_str()).unwrap();
-        assert_eq!(entry.dir.as_str(), "/root0");
-
-        let entry = get_entry_by_uid(1, temp_passwd.as_str()).unwrap();
-        assert_eq!(entry.dir.as_str(), "/root1");
-
-        let entry = get_entry_by_uid(2, temp_passwd.as_str()).unwrap();
-        assert_eq!(entry.dir.as_str(), "/root2");
-
-        let entry = get_entry_by_uid(3, temp_passwd.as_str()).unwrap();
-        assert_eq!(entry.dir.as_str(), "/root3");
-    }
-}

src/agent/samples/configuration-all-endpoints.toml

@@ -25,6 +25,7 @@ allowed = [
         "ReadStreamRequest",
         "RemoveContainerRequest",
         "ReseedRandomDevRequest",
+        "ResizeVolumeRequest",
         "ResumeContainerRequest",
         "SetGuestDateTimeRequest",
         "SignalProcessRequest",
@@ -34,6 +35,7 @@ allowed = [
         "UpdateContainerRequest",
         "UpdateInterfaceRequest",
         "UpdateRoutesRequest",
+        "VolumeStatsRequest",
         "WaitProcessRequest",
         "WriteStreamRequest"
 ]

src/agent/src/config.rs

@@ -432,6 +432,8 @@ fn get_container_pipe_size(param: &str) -> Result<i32> {
 
 #[cfg(test)]
 mod tests {
+    use crate::assert_result;
+
     use super::*;
     use anyhow::anyhow;
     use std::fs::File;
@@ -439,32 +441,6 @@ mod tests {
     use std::time;
     use tempfile::tempdir;
 
-    // Parameters:
-    //
-    // 1: expected Result
-    // 2: actual Result
-    // 3: string used to identify the test on error
-    macro_rules! assert_result {
-        ($expected_result:expr, $actual_result:expr, $msg:expr) => {
-            if $expected_result.is_ok() {
-                let expected_level = $expected_result.as_ref().unwrap();
-                let actual_level = $actual_result.unwrap();
-                assert!(*expected_level == actual_level, "{}", $msg);
-            } else {
-                let expected_error = $expected_result.as_ref().unwrap_err();
-                let expected_error_msg = format!("{:?}", expected_error);
-
-                if let Err(actual_error) = $actual_result {
-                    let actual_error_msg = format!("{:?}", actual_error);
-
-                    assert!(expected_error_msg == actual_error_msg, "{}", $msg);
-                } else {
-                    assert!(expected_error_msg == "expected error, got OK", "{}", $msg);
-                }
-            }
-        };
-    }
-
     #[test]
     fn test_new() {
         let config: AgentConfig = Default::default();

src/agent/src/device.rs

@@ -51,6 +51,8 @@ pub const DRIVER_VFIO_GK_TYPE: &str = "vfio-gk";
 // VFIO device to be bound to vfio-pci and made available inside the
 // container as a VFIO device node
 pub const DRIVER_VFIO_TYPE: &str = "vfio";
+pub const DRIVER_OVERLAYFS_TYPE: &str = "overlayfs";
+pub const FS_TYPE_HUGETLB: &str = "hugetlbfs";
 
 #[instrument]
 pub fn online_device(path: &str) -> Result<()> {
@@ -592,38 +594,38 @@ fn update_spec_devices(spec: &mut Spec, mut updates: HashMap<&str, DevUpdate>) -
     Ok(())
 }
 
-// update_spec_pci PCI addresses in the OCI spec to be guest addresses
-// instead of host addresses.  It is given a map of (host address =>
-// guest address)
+// update_env_pci alters PCI addresses in a set of environment
+// variables to be correct for the VM instead of the host.  It is
+// given a map of (host address => guest address)
 #[instrument]
-fn update_spec_pci(spec: &mut Spec, updates: HashMap<pci::Address, pci::Address>) -> Result<()> {
-    // Correct PCI addresses in the environment
-    if let Some(process) = spec.process.as_mut() {
-        for envvar in process.env.iter_mut() {
-            let eqpos = envvar
-                .find('=')
-                .ok_or_else(|| anyhow!("Malformed OCI env entry {:?}", envvar))?;
+pub fn update_env_pci(
+    env: &mut [String],
+    pcimap: &HashMap<pci::Address, pci::Address>,
+) -> Result<()> {
+    for envvar in env {
+        let eqpos = envvar
+            .find('=')
+            .ok_or_else(|| anyhow!("Malformed OCI env entry {:?}", envvar))?;
 
-            let (name, eqval) = envvar.split_at(eqpos);
-            let val = &eqval[1..];
+        let (name, eqval) = envvar.split_at(eqpos);
+        let val = &eqval[1..];
 
-            if !name.starts_with("PCIDEVICE_") {
-                continue;
-            }
-
-            let mut guest_addrs = Vec::<String>::new();
-
-            for host_addr in val.split(',') {
-                let host_addr = pci::Address::from_str(host_addr)
-                    .with_context(|| format!("Can't parse {} environment variable", name))?;
-                let guest_addr = updates
-                    .get(&host_addr)
-                    .ok_or_else(|| anyhow!("Unable to translate host PCI address {}", host_addr))?;
-                guest_addrs.push(format!("{}", guest_addr));
-            }
-
-            envvar.replace_range(eqpos + 1.., guest_addrs.join(",").as_str());
+        if !name.starts_with("PCIDEVICE_") {
+            continue;
         }
+
+        let mut guest_addrs = Vec::<String>::new();
+
+        for host_addr in val.split(',') {
+            let host_addr = pci::Address::from_str(host_addr)
+                .with_context(|| format!("Can't parse {} environment variable", name))?;
+            let guest_addr = pcimap
+                .get(&host_addr)
+                .ok_or_else(|| anyhow!("Unable to translate host PCI address {}", host_addr))?;
+            guest_addrs.push(format!("{}", guest_addr));
+        }
+
+        envvar.replace_range(eqpos + 1.., guest_addrs.join(",").as_str());
     }
 
     Ok(())
@@ -768,7 +770,6 @@ pub async fn add_devices(
     sandbox: &Arc<Mutex<Sandbox>>,
 ) -> Result<()> {
     let mut dev_updates = HashMap::<&str, DevUpdate>::with_capacity(devices.len());
-    let mut pci_updates = HashMap::<pci::Address, pci::Address>::new();
 
     for device in devices.iter() {
         let update = add_device(device, sandbox).await?;
@@ -783,8 +784,9 @@ pub async fn add_devices(
                 ));
             }
 
+            let mut sb = sandbox.lock().await;
             for (host, guest) in update.pci {
-                if let Some(other_guest) = pci_updates.insert(host, guest) {
+                if let Some(other_guest) = sb.pcimap.insert(host, guest) {
                     return Err(anyhow!(
                         "Conflicting guest address for host device {} ({} versus {})",
                         host,
@@ -796,6 +798,9 @@ pub async fn add_devices(
         }
     }
 
+    if let Some(process) = spec.process.as_mut() {
+        update_env_pci(&mut process.env, &sandbox.lock().await.pcimap)?
+    }
     update_spec_devices(spec, dev_updates)
 }
 
@@ -860,7 +865,7 @@ pub fn update_device_cgroup(spec: &mut Spec) -> Result<()> {
 mod tests {
     use super::*;
     use crate::uevent::spawn_test_watcher;
-    use oci::{Linux, Process};
+    use oci::Linux;
     use std::iter::FromIterator;
     use tempfile::tempdir;
 
@@ -1199,7 +1204,7 @@ mod tests {
     }
 
     #[test]
-    fn test_update_spec_pci() {
+    fn test_update_env_pci() {
         let example_map = [
             // Each is a host,guest pair of pci addresses
             ("0000:1a:01.0", "0000:01:01.0"),
@@ -1209,17 +1214,11 @@ mod tests {
             ("0000:01:01.0", "ffff:02:1f.7"),
         ];
 
-        let mut spec = Spec {
-            process: Some(Process {
-                env: vec![
-                    "PCIDEVICE_x=0000:1a:01.0,0000:1b:02.0".to_string(),
-                    "PCIDEVICE_y=0000:01:01.0".to_string(),
-                    "NOTAPCIDEVICE_blah=abcd:ef:01.0".to_string(),
-                ],
-                ..Process::default()
-            }),
-            ..Spec::default()
-        };
+        let mut env = vec![
+            "PCIDEVICE_x=0000:1a:01.0,0000:1b:02.0".to_string(),
+            "PCIDEVICE_y=0000:01:01.0".to_string(),
+            "NOTAPCIDEVICE_blah=abcd:ef:01.0".to_string(),
+        ];
 
         let pci_fixups = example_map
             .iter()
@@ -1231,10 +1230,9 @@ mod tests {
             })
             .collect();
 
-        let res = update_spec_pci(&mut spec, pci_fixups);
+        let res = update_env_pci(&mut env, &pci_fixups);
         assert!(res.is_ok());
 
-        let env = &spec.process.as_ref().unwrap().env;
         assert_eq!(env[0], "PCIDEVICE_x=0000:01:01.0,0000:01:02.0");
         assert_eq!(env[1], "PCIDEVICE_y=ffff:02:1f.7");
         assert_eq!(env[2], "NOTAPCIDEVICE_blah=abcd:ef:01.0");

src/agent/src/main.rs

@@ -125,9 +125,7 @@ fn announce(logger: &Logger, config: &AgentConfig) {
 // output to the vsock port specified, or stdout.
 async fn create_logger_task(rfd: RawFd, vsock_port: u32, shutdown: Receiver<bool>) -> Result<()> {
     let mut reader = PipeStream::from_fd(rfd);
-    let mut writer: Box<dyn AsyncWrite + Unpin + Send>;
-
-    if vsock_port > 0 {
+    let mut writer: Box<dyn AsyncWrite + Unpin + Send> = if vsock_port > 0 {
         let listenfd = socket::socket(
             AddressFamily::Vsock,
             SockType::Stream,
@@ -139,10 +137,10 @@ async fn create_logger_task(rfd: RawFd, vsock_port: u32, shutdown: Receiver<bool
         socket::bind(listenfd, &addr)?;
         socket::listen(listenfd, 1)?;
 
-        writer = Box::new(util::get_vsock_stream(listenfd).await?);
+        Box::new(util::get_vsock_stream(listenfd).await?)
     } else {
-        writer = Box::new(tokio::io::stdout());
-    }
+        Box::new(tokio::io::stdout())
+    };
 
     let _ = util::interruptable_io_copier(&mut reader, &mut writer, shutdown).await;
 
@@ -418,3 +416,59 @@ fn reset_sigpipe() {
 
 use crate::config::AgentConfig;
 use std::os::unix::io::{FromRawFd, RawFd};
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::test_utils::test_utils::TestUserType;
+
+    #[tokio::test]
+    async fn test_create_logger_task() {
+        #[derive(Debug)]
+        struct TestData {
+            vsock_port: u32,
+            test_user: TestUserType,
+            result: Result<()>,
+        }
+
+        let tests = &[
+            TestData {
+                // non-root user cannot use privileged vsock port
+                vsock_port: 1,
+                test_user: TestUserType::NonRootOnly,
+                result: Err(anyhow!(nix::errno::Errno::from_i32(libc::EACCES))),
+            },
+            TestData {
+                // passing vsock_port 0 causes logger task to write to stdout
+                vsock_port: 0,
+                test_user: TestUserType::Any,
+                result: Ok(()),
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            if d.test_user == TestUserType::RootOnly {
+                skip_if_not_root!();
+            } else if d.test_user == TestUserType::NonRootOnly {
+                skip_if_root!();
+            }
+
+            let msg = format!("test[{}]: {:?}", i, d);
+            let (rfd, wfd) = unistd::pipe2(OFlag::O_CLOEXEC).unwrap();
+            defer!({
+                // rfd is closed by the use of PipeStream in the crate_logger_task function,
+                // but we will attempt to close in case of a failure
+                let _ = unistd::close(rfd);
+                unistd::close(wfd).unwrap();
+            });
+
+            let (shutdown_tx, shutdown_rx) = channel(true);
+
+            shutdown_tx.send(true).unwrap();
+            let result = create_logger_task(rfd, d.vsock_port, shutdown_rx).await;
+
+            let msg = format!("{}, result: {:?}", msg, result);
+            assert_result!(d.result, result, msg);
+        }
+    }
+}

src/agent/src/metrics.rs

@@ -344,25 +344,25 @@ fn set_gauge_vec_meminfo(gv: &prometheus::GaugeVec, meminfo: &procfs::Meminfo) {
 #[instrument]
 fn set_gauge_vec_cpu_time(gv: &prometheus::GaugeVec, cpu: &str, cpu_time: &procfs::CpuTime) {
     gv.with_label_values(&[cpu, "user"])
-        .set(cpu_time.user as f64);
+        .set(cpu_time.user_ms() as f64);
     gv.with_label_values(&[cpu, "nice"])
-        .set(cpu_time.nice as f64);
+        .set(cpu_time.nice_ms() as f64);
     gv.with_label_values(&[cpu, "system"])
-        .set(cpu_time.system as f64);
+        .set(cpu_time.system_ms() as f64);
     gv.with_label_values(&[cpu, "idle"])
-        .set(cpu_time.idle as f64);
+        .set(cpu_time.idle_ms() as f64);
     gv.with_label_values(&[cpu, "iowait"])
-        .set(cpu_time.iowait.unwrap_or(0) as f64);
+        .set(cpu_time.iowait_ms().unwrap_or(0) as f64);
     gv.with_label_values(&[cpu, "irq"])
-        .set(cpu_time.irq.unwrap_or(0) as f64);
+        .set(cpu_time.irq_ms().unwrap_or(0) as f64);
     gv.with_label_values(&[cpu, "softirq"])
-        .set(cpu_time.softirq.unwrap_or(0) as f64);
+        .set(cpu_time.softirq_ms().unwrap_or(0) as f64);
     gv.with_label_values(&[cpu, "steal"])
-        .set(cpu_time.steal.unwrap_or(0) as f64);
+        .set(cpu_time.steal_ms().unwrap_or(0) as f64);
     gv.with_label_values(&[cpu, "guest"])
-        .set(cpu_time.guest.unwrap_or(0) as f64);
+        .set(cpu_time.guest_ms().unwrap_or(0) as f64);
     gv.with_label_values(&[cpu, "guest_nice"])
-        .set(cpu_time.guest_nice.unwrap_or(0) as f64);
+        .set(cpu_time.guest_nice_ms().unwrap_or(0) as f64);
 }
 
 #[instrument]

src/agent/src/mount.rs

@@ -5,8 +5,8 @@
 
 use std::collections::HashMap;
 use std::fs;
-use std::fs::File;
-use std::io::{BufRead, BufReader};
+use std::fs::{File, OpenOptions};
+use std::io::{BufRead, BufReader, Write};
 use std::iter;
 use std::os::unix::fs::{MetadataExt, PermissionsExt};
 use std::path::Path;
@@ -16,19 +16,20 @@ use std::sync::Arc;
 use tokio::sync::Mutex;
 
 use nix::mount::MsFlags;
-use nix::unistd::Gid;
+use nix::unistd::{Gid, Uid};
 
 use regex::Regex;
 
 use crate::device::{
     get_scsi_device_name, get_virtio_blk_pci_device_name, online_device, wait_for_pmem_device,
     DRIVER_9P_TYPE, DRIVER_BLK_CCW_TYPE, DRIVER_BLK_TYPE, DRIVER_EPHEMERAL_TYPE, DRIVER_LOCAL_TYPE,
-    DRIVER_MMIO_BLK_TYPE, DRIVER_NVDIMM_TYPE, DRIVER_SCSI_TYPE, DRIVER_VIRTIOFS_TYPE,
-    DRIVER_WATCHABLE_BIND_TYPE,
+    DRIVER_MMIO_BLK_TYPE, DRIVER_NVDIMM_TYPE, DRIVER_OVERLAYFS_TYPE, DRIVER_SCSI_TYPE,
+    DRIVER_VIRTIOFS_TYPE, DRIVER_WATCHABLE_BIND_TYPE, FS_TYPE_HUGETLB,
 };
 use crate::linux_abi::*;
 use crate::pci;
 use crate::protocols::agent::Storage;
+use crate::protocols::types::FSGroupChangePolicy;
 use crate::Sandbox;
 #[cfg(target_arch = "s390x")]
 use crate::{ccw, device::get_virtio_blk_ccw_device_name};
@@ -37,12 +38,17 @@ use slog::Logger;
 use tracing::instrument;
 
 pub const TYPE_ROOTFS: &str = "rootfs";
-
+const SYS_FS_HUGEPAGES_PREFIX: &str = "/sys/kernel/mm/hugepages";
 pub const MOUNT_GUEST_TAG: &str = "kataShared";
 
 // Allocating an FSGroup that owns the pod's volumes
 const FS_GID: &str = "fsgid";
 
+const RW_MASK: u32 = 0o660;
+const RO_MASK: u32 = 0o440;
+const EXEC_MASK: u32 = 0o110;
+const MODE_SETGID: u32 = 0o2000;
+
 #[rustfmt::skip]
 lazy_static! {
     pub static ref FLAGS: HashMap<&'static str, (bool, MsFlags)> = {
@@ -85,11 +91,11 @@ lazy_static! {
 }
 
 #[derive(Debug, PartialEq)]
-pub struct InitMount {
-    fstype: &'static str,
-    src: &'static str,
-    dest: &'static str,
-    options: Vec<&'static str>,
+pub struct InitMount<'a> {
+    fstype: &'a str,
+    src: &'a str,
+    dest: &'a str,
+    options: Vec<&'a str>,
 }
 
 #[rustfmt::skip]
@@ -115,7 +121,7 @@ lazy_static!{
 
 #[rustfmt::skip]
 lazy_static! {
-    pub static ref INIT_ROOTFS_MOUNTS: Vec<InitMount> = vec![
+    pub static ref INIT_ROOTFS_MOUNTS: Vec<InitMount<'static>> = vec![
         InitMount{fstype: "proc", src: "proc", dest: "/proc", options: vec!["nosuid", "nodev", "noexec"]},
         InitMount{fstype: "sysfs", src: "sysfs", dest: "/sys", options: vec!["nosuid", "nodev", "noexec"]},
         InitMount{fstype: "devtmpfs", src: "dev", dest: "/dev", options: vec!["nosuid"]},
@@ -130,6 +136,7 @@ pub const STORAGE_HANDLER_LIST: &[&str] = &[
     DRIVER_9P_TYPE,
     DRIVER_VIRTIOFS_TYPE,
     DRIVER_EPHEMERAL_TYPE,
+    DRIVER_OVERLAYFS_TYPE,
     DRIVER_MMIO_BLK_TYPE,
     DRIVER_LOCAL_TYPE,
     DRIVER_SCSI_TYPE,
@@ -192,13 +199,12 @@ async fn ephemeral_storage_handler(
     storage: &Storage,
     sandbox: Arc<Mutex<Sandbox>>,
 ) -> Result<String> {
-    let mut sb = sandbox.lock().await;
-    let new_storage = sb.set_sandbox_storage(&storage.mount_point);
-
-    if !new_storage {
-        return Ok("".to_string());
+    // hugetlbfs
+    if storage.fstype == FS_TYPE_HUGETLB {
+        return handle_hugetlbfs_storage(logger, storage).await;
     }
 
+    // normal ephemeral storage
     fs::create_dir_all(Path::new(&storage.mount_point))?;
 
     // By now we only support one option field: "fsGroup" which
@@ -222,7 +228,7 @@ async fn ephemeral_storage_handler(
             let meta = fs::metadata(&storage.mount_point)?;
             let mut permission = meta.permissions();
 
-            let o_mode = meta.mode() | 0o2000;
+            let o_mode = meta.mode() | MODE_SETGID;
             permission.set_mode(o_mode);
             fs::set_permissions(&storage.mount_point, permission)?;
         }
@@ -233,19 +239,21 @@ async fn ephemeral_storage_handler(
     Ok("".to_string())
 }
 
+#[instrument]
+async fn overlayfs_storage_handler(
+    logger: &Logger,
+    storage: &Storage,
+    _sandbox: Arc<Mutex<Sandbox>>,
+) -> Result<String> {
+    common_storage_handler(logger, storage)
+}
+
 #[instrument]
 async fn local_storage_handler(
     _logger: &Logger,
     storage: &Storage,
     sandbox: Arc<Mutex<Sandbox>>,
 ) -> Result<String> {
-    let mut sb = sandbox.lock().await;
-    let new_storage = sb.set_sandbox_storage(&storage.mount_point);
-
-    if !new_storage {
-        return Ok("".to_string());
-    }
-
     fs::create_dir_all(&storage.mount_point).context(format!(
         "failed to create dir all {:?}",
         &storage.mount_point
@@ -270,7 +278,7 @@ async fn local_storage_handler(
 
         if need_set_fsgid {
             // set SetGid mode mask.
-            o_mode |= 0o2000;
+            o_mode |= MODE_SETGID;
         }
         permission.set_mode(o_mode);
 
@@ -289,12 +297,116 @@ async fn virtio9p_storage_handler(
     common_storage_handler(logger, storage)
 }
 
+#[instrument]
+async fn handle_hugetlbfs_storage(logger: &Logger, storage: &Storage) -> Result<String> {
+    info!(logger, "handle hugetlbfs storage");
+    // Allocate hugepages before mount
+    // /sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages
+    // /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
+    // options eg "pagesize=2097152,size=524288000"(2M, 500M)
+    allocate_hugepages(logger, &storage.options.to_vec()).context("allocate hugepages")?;
+
+    common_storage_handler(logger, storage)?;
+
+    // hugetlbfs return empty string as ephemeral_storage_handler do.
+    // this is a sandbox level storage, but not a container-level mount.
+    Ok("".to_string())
+}
+
+// Allocate hugepages by writing to sysfs
+fn allocate_hugepages(logger: &Logger, options: &[String]) -> Result<()> {
+    info!(logger, "mounting hugePages storage options: {:?}", options);
+
+    let (pagesize, size) = get_pagesize_and_size_from_option(options)
+        .context(format!("parse mount options: {:?}", &options))?;
+
+    info!(
+        logger,
+        "allocate hugepages. pageSize: {}, size: {}", pagesize, size
+    );
+
+    // sysfs entry is always of the form hugepages-${pagesize}kB
+    // Ref: https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt
+    let path = Path::new(SYS_FS_HUGEPAGES_PREFIX)
+        .join(format!("hugepages-{}kB", pagesize / 1024))
+        .join("nr_hugepages");
+
+    // write numpages to nr_hugepages file.
+    let numpages = format!("{}", size / pagesize);
+    info!(logger, "write {} pages to {:?}", &numpages, &path);
+
+    let mut file = OpenOptions::new()
+        .write(true)
+        .open(&path)
+        .context(format!("open nr_hugepages directory {:?}", &path))?;
+
+    file.write_all(numpages.as_bytes())
+        .context(format!("write nr_hugepages failed: {:?}", &path))?;
+
+    // Even if the write succeeds, the kernel isn't guaranteed to be
+    // able to allocate all the pages we requested.  Verify that it
+    // did.
+    let verify = fs::read_to_string(&path).context(format!("reading {:?}", &path))?;
+    let allocated = verify
+        .trim_end()
+        .parse::<u64>()
+        .map_err(|_| anyhow!("Unexpected text {:?} in {:?}", &verify, &path))?;
+    if allocated != size / pagesize {
+        return Err(anyhow!(
+            "Only allocated {} of {} hugepages of size {}",
+            allocated,
+            numpages,
+            pagesize
+        ));
+    }
+
+    Ok(())
+}
+
+// Parse filesystem options string to retrieve hugepage details
+// options eg "pagesize=2048,size=107374182"
+fn get_pagesize_and_size_from_option(options: &[String]) -> Result<(u64, u64)> {
+    let mut pagesize_str: Option<&str> = None;
+    let mut size_str: Option<&str> = None;
+
+    for option in options {
+        let vars: Vec<&str> = option.trim().split(',').collect();
+
+        for var in vars {
+            if let Some(stripped) = var.strip_prefix("pagesize=") {
+                pagesize_str = Some(stripped);
+            } else if let Some(stripped) = var.strip_prefix("size=") {
+                size_str = Some(stripped);
+            }
+
+            if pagesize_str.is_some() && size_str.is_some() {
+                break;
+            }
+        }
+    }
+
+    if pagesize_str.is_none() || size_str.is_none() {
+        return Err(anyhow!("no pagesize/size options found"));
+    }
+
+    let pagesize = pagesize_str
+        .unwrap()
+        .parse::<u64>()
+        .context(format!("parse pagesize: {:?}", &pagesize_str))?;
+    let size = size_str
+        .unwrap()
+        .parse::<u64>()
+        .context(format!("parse size: {:?}", &pagesize_str))?;
+
+    Ok((pagesize, size))
+}
+
 // virtiommio_blk_storage_handler handles the storage for mmio blk driver.
 #[instrument]
 async fn virtiommio_blk_storage_handler(
     logger: &Logger,
     storage: &Storage,
-    _sandbox: Arc<Mutex<Sandbox>>,
+    sandbox: Arc<Mutex<Sandbox>>,
 ) -> Result<String> {
     //The source path is VmPath
     common_storage_handler(logger, storage)
@@ -383,7 +495,9 @@ fn common_storage_handler(logger: &Logger, storage: &Storage) -> Result<String>
     // Mount the storage device.
     let mount_point = storage.mount_point.to_string();
 
-    mount_storage(logger, storage).and(Ok(mount_point))
+    mount_storage(logger, storage)?;
+    set_ownership(logger, storage)?;
+    Ok(mount_point)
 }
 
 // nvdimm_storage_handler handles the storage for NVDIMM driver.
@@ -467,6 +581,91 @@ fn mount_storage(logger: &Logger, storage: &Storage) -> Result<()> {
     )
 }
 
+#[instrument]
+pub fn set_ownership(logger: &Logger, storage: &Storage) -> Result<()> {
+    let logger = logger.new(o!("subsystem" => "mount", "fn" => "set_ownership"));
+
+    // If fsGroup is not set, skip performing ownership change
+    if storage.fs_group.is_none() {
+        return Ok(());
+    }
+    let fs_group = storage.get_fs_group();
+
+    let mut read_only = false;
+    let opts_vec: Vec<String> = storage.options.to_vec();
+    if opts_vec.contains(&String::from("ro")) {
+        read_only = true;
+    }
+
+    let mount_path = Path::new(&storage.mount_point);
+    let metadata = mount_path.metadata().map_err(|err| {
+        error!(logger, "failed to obtain metadata for mount path";
+            "mount-path" => mount_path.to_str(),
+            "error" => err.to_string(),
+        );
+        err
+    })?;
+
+    if fs_group.group_change_policy == FSGroupChangePolicy::OnRootMismatch
+        && metadata.gid() == fs_group.group_id
+    {
+        let mut mask = if read_only { RO_MASK } else { RW_MASK };
+        mask |= EXEC_MASK;
+
+        // With fsGroup change policy to OnRootMismatch, if the current
+        // gid of the mount path root directory matches the desired gid
+        // and the current permission of mount path root directory is correct,
+        // then ownership change will be skipped.
+        let current_mode = metadata.permissions().mode();
+        if (mask & current_mode == mask) && (current_mode & MODE_SETGID != 0) {
+            info!(logger, "skipping ownership change for volume";
+                "mount-path" => mount_path.to_str(),
+                "fs-group" => fs_group.group_id.to_string(),
+            );
+            return Ok(());
+        }
+    }
+
+    info!(logger, "performing recursive ownership change";
+        "mount-path" => mount_path.to_str(),
+        "fs-group" => fs_group.group_id.to_string(),
+    );
+    recursive_ownership_change(
+        mount_path,
+        None,
+        Some(Gid::from_raw(fs_group.group_id)),
+        read_only,
+    )
+}
+
+#[instrument]
+pub fn recursive_ownership_change(
+    path: &Path,
+    uid: Option<Uid>,
+    gid: Option<Gid>,
+    read_only: bool,
+) -> Result<()> {
+    let mut mask = if read_only { RO_MASK } else { RW_MASK };
+    if path.is_dir() {
+        for entry in fs::read_dir(&path)? {
+            recursive_ownership_change(entry?.path().as_path(), uid, gid, read_only)?;
+        }
+        mask |= EXEC_MASK;
+        mask |= MODE_SETGID;
+    }
+    nix::unistd::chown(path, uid, gid)?;
+
+    if gid.is_some() {
+        let metadata = path.metadata()?;
+        let mut permission = metadata.permissions();
+        let target_mode = metadata.mode() | mask;
+        permission.set_mode(target_mode);
+        fs::set_permissions(path, permission)?;
+    }
+
+    Ok(())
+}
+
 /// Looks for `mount_point` entry in the /proc/mounts.
 #[instrument]
 pub fn is_mounted(mount_point: &str) -> Result<bool> {
@@ -534,6 +733,14 @@ pub async fn add_storages(
             "subsystem" => "storage",
             "storage-type" => handler_name.to_owned()));
 
+        {
+            let mut sb = sandbox.lock().await;
+            let new_storage = sb.set_sandbox_storage(&storage.mount_point);
+            if !new_storage {
+                continue;
+            }
+        }
+
         let res = match handler_name.as_str() {
             DRIVER_BLK_TYPE => virtio_blk_storage_handler(&logger, &storage, sandbox.clone()).await,
             DRIVER_BLK_CCW_TYPE => {
@@ -546,6 +753,9 @@ pub async fn add_storages(
             DRIVER_EPHEMERAL_TYPE => {
                 ephemeral_storage_handler(&logger, &storage, sandbox.clone()).await
             }
+            DRIVER_OVERLAYFS_TYPE => {
+                overlayfs_storage_handler(&logger, &storage, sandbox.clone()).await
+            }
             DRIVER_MMIO_BLK_TYPE => {
                 virtiommio_blk_storage_handler(&logger, &storage, sandbox.clone()).await
             }
@@ -649,8 +859,9 @@ pub fn get_mount_fs_type_from_file(mount_file: &str, mount_point: &str) -> Resul
     }
 
     Err(anyhow!(
-        "failed to find FS type for mount point {}",
-        mount_point
+        "failed to find FS type for mount point {}, mount file content: {:?}",
+        mount_point,
+        fs::read_to_string(mount_file)
     ))
 }
 
@@ -659,7 +870,7 @@ pub fn get_cgroup_mounts(
     logger: &Logger,
     cg_path: &str,
     unified_cgroup_hierarchy: bool,
-) -> Result<Vec<InitMount>> {
+) -> Result<Vec<InitMount<'static>>> {
     // cgroup v2
     // https://github.com/kata-containers/agent/blob/8c9bbadcd448c9a67690fbe11a860aaacc69813c/agent.go#L1249
     if unified_cgroup_hierarchy {
@@ -807,20 +1018,16 @@ fn parse_options(option_list: Vec<String>) -> HashMap<String, String> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::{skip_if_not_root, skip_loop_if_not_root, skip_loop_if_root};
+    use crate::test_utils::test_utils::TestUserType;
+    use crate::{skip_if_not_root, skip_loop_by_user, skip_loop_if_not_root, skip_loop_if_root};
+    use protobuf::RepeatedField;
+    use protocols::agent::FSGroup;
     use std::fs::File;
     use std::fs::OpenOptions;
     use std::io::Write;
     use std::path::PathBuf;
     use tempfile::tempdir;
 
-    #[derive(Debug, PartialEq)]
-    enum TestUserType {
-        RootOnly,
-        NonRootOnly,
-        Any,
-    }
-
     #[test]
     fn test_mount() {
         #[derive(Debug)]
@@ -906,11 +1113,7 @@ mod tests {
         for (i, d) in tests.iter().enumerate() {
             let msg = format!("test[{}]: {:?}", i, d);
 
-            if d.test_user == TestUserType::RootOnly {
-                skip_loop_if_not_root!(msg);
-            } else if d.test_user == TestUserType::NonRootOnly {
-                skip_loop_if_root!(msg);
-            }
+            skip_loop_by_user!(msg, d.test_user);
 
             let src: PathBuf;
             let dest: PathBuf;
@@ -919,7 +1122,7 @@ mod tests {
             let dest_filename: String;
 
             if !d.src.is_empty() {
-                src = dir.path().join(d.src.to_string());
+                src = dir.path().join(d.src);
                 src_filename = src
                     .to_str()
                     .expect("failed to convert src to filename")
@@ -929,7 +1132,7 @@ mod tests {
             }
 
             if !d.dest.is_empty() {
-                dest = dir.path().join(d.dest.to_string());
+                dest = dir.path().join(d.dest);
                 dest_filename = dest
                     .to_str()
                     .expect("failed to convert dest to filename")
@@ -1379,4 +1582,539 @@ mod tests {
 
         assert!(testfile.is_file());
     }
+
+    #[test]
+    fn test_mount_storage() {
+        #[derive(Debug)]
+        struct TestData<'a> {
+            test_user: TestUserType,
+            storage: Storage,
+            error_contains: &'a str,
+
+            make_source_dir: bool,
+            make_mount_dir: bool,
+            deny_mount_permission: bool,
+        }
+
+        impl Default for TestData<'_> {
+            fn default() -> Self {
+                TestData {
+                    test_user: TestUserType::Any,
+                    storage: Storage {
+                        mount_point: "mnt".to_string(),
+                        source: "src".to_string(),
+                        fstype: "tmpfs".to_string(),
+                        ..Default::default()
+                    },
+                    make_source_dir: true,
+                    make_mount_dir: false,
+                    deny_mount_permission: false,
+                    error_contains: "",
+                }
+            }
+        }
+
+        let tests = &[
+            TestData {
+                test_user: TestUserType::NonRootOnly,
+                error_contains: "EPERM: Operation not permitted",
+                ..Default::default()
+            },
+            TestData {
+                test_user: TestUserType::RootOnly,
+                ..Default::default()
+            },
+            TestData {
+                storage: Storage {
+                    mount_point: "mnt".to_string(),
+                    source: "src".to_string(),
+                    fstype: "bind".to_string(),
+                    ..Default::default()
+                },
+                make_source_dir: false,
+                make_mount_dir: true,
+                error_contains: "Could not create mountpoint",
+                ..Default::default()
+            },
+            TestData {
+                test_user: TestUserType::NonRootOnly,
+                deny_mount_permission: true,
+                error_contains: "Could not create mountpoint",
+                ..Default::default()
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            skip_loop_by_user!(msg, d.test_user);
+
+            let drain = slog::Discard;
+            let logger = slog::Logger::root(drain, o!());
+
+            let tempdir = tempdir().unwrap();
+
+            let source = tempdir.path().join(&d.storage.source);
+            let mount_point = tempdir.path().join(&d.storage.mount_point);
+
+            let storage = Storage {
+                source: source.to_str().unwrap().to_string(),
+                mount_point: mount_point.to_str().unwrap().to_string(),
+                ..d.storage.clone()
+            };
+
+            if d.make_source_dir {
+                fs::create_dir_all(&storage.source).unwrap();
+            }
+            if d.make_mount_dir {
+                fs::create_dir_all(&storage.mount_point).unwrap();
+            }
+
+            if d.deny_mount_permission {
+                fs::set_permissions(
+                    mount_point.parent().unwrap(),
+                    fs::Permissions::from_mode(0o000),
+                )
+                .unwrap();
+            }
+
+            let result = mount_storage(&logger, &storage);
+
+            // restore permissions so tempdir can be cleaned up
+            if d.deny_mount_permission {
+                fs::set_permissions(
+                    mount_point.parent().unwrap(),
+                    fs::Permissions::from_mode(0o755),
+                )
+                .unwrap();
+            }
+
+            if result.is_ok() {
+                nix::mount::umount(&mount_point).unwrap();
+            }
+
+            let msg = format!("{}: result: {:?}", msg, result);
+            if d.error_contains.is_empty() {
+                assert!(result.is_ok(), "{}", msg);
+            } else {
+                assert!(result.is_err(), "{}", msg);
+                let error_msg = format!("{}", result.unwrap_err());
+                assert!(error_msg.contains(d.error_contains), "{}", msg);
+            }
+        }
+    }
+
+    #[test]
+    fn test_mount_to_rootfs() {
+        #[derive(Debug)]
+        struct TestData<'a> {
+            test_user: TestUserType,
+            src: &'a str,
+            options: Vec<&'a str>,
+            error_contains: &'a str,
+            deny_mount_dir_permission: bool,
+            // if true src will be prepended with a temporary directory
+            mask_src: bool,
+        }
+
+        impl Default for TestData<'_> {
+            fn default() -> Self {
+                TestData {
+                    test_user: TestUserType::Any,
+                    src: "src",
+                    options: vec![],
+                    error_contains: "",
+                    deny_mount_dir_permission: false,
+                    mask_src: true,
+                }
+            }
+        }
+
+        let tests = &[
+            TestData {
+                test_user: TestUserType::NonRootOnly,
+                error_contains: "EPERM: Operation not permitted",
+                ..Default::default()
+            },
+            TestData {
+                test_user: TestUserType::NonRootOnly,
+                src: "dev",
+                mask_src: false,
+                ..Default::default()
+            },
+            TestData {
+                test_user: TestUserType::RootOnly,
+                ..Default::default()
+            },
+            TestData {
+                test_user: TestUserType::NonRootOnly,
+                deny_mount_dir_permission: true,
+                error_contains: "could not create directory",
+                ..Default::default()
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+            skip_loop_by_user!(msg, d.test_user);
+
+            let drain = slog::Discard;
+            let logger = slog::Logger::root(drain, o!());
+            let tempdir = tempdir().unwrap();
+
+            let src = if d.mask_src {
+                tempdir.path().join(&d.src)
+            } else {
+                Path::new(d.src).to_path_buf()
+            };
+            let dest = tempdir.path().join("mnt");
+            let init_mount = InitMount {
+                fstype: "tmpfs",
+                src: src.to_str().unwrap(),
+                dest: dest.to_str().unwrap(),
+                options: d.options.clone(),
+            };
+
+            if d.deny_mount_dir_permission {
+                fs::set_permissions(dest.parent().unwrap(), fs::Permissions::from_mode(0o000))
+                    .unwrap();
+            }
+
+            let result = mount_to_rootfs(&logger, &init_mount);
+
+            // restore permissions so tempdir can be cleaned up
+            if d.deny_mount_dir_permission {
+                fs::set_permissions(dest.parent().unwrap(), fs::Permissions::from_mode(0o755))
+                    .unwrap();
+            }
+
+            if result.is_ok() && d.mask_src {
+                nix::mount::umount(&dest).unwrap();
+            }
+
+            let msg = format!("{}: result: {:?}", msg, result);
+            if d.error_contains.is_empty() {
+                assert!(result.is_ok(), "{}", msg);
+            } else {
+                assert!(result.is_err(), "{}", msg);
+                let error_msg = format!("{}", result.unwrap_err());
+                assert!(error_msg.contains(d.error_contains), "{}", msg);
+            }
+        }
+    }
+
+    #[test]
+    fn test_get_pagesize_and_size_from_option() {
+        let expected_pagesize = 2048;
+        let expected_size = 107374182;
+        let expected = (expected_pagesize, expected_size);
+
+        let data = vec![
+            // (input, expected, is_ok)
+            ("size-1=107374182,pagesize-1=2048", expected, false),
+            ("size-1=107374182,pagesize=2048", expected, false),
+            ("size=107374182,pagesize-1=2048", expected, false),
+            ("size=107374182,pagesize=abc", expected, false),
+            ("size=abc,pagesize=2048", expected, false),
+            ("size=,pagesize=2048", expected, false),
+            ("size=107374182,pagesize=", expected, false),
+            ("size=107374182,pagesize=2048", expected, true),
+            ("pagesize=2048,size=107374182", expected, true),
+            ("foo=bar,pagesize=2048,size=107374182", expected, true),
+            (
+                "foo=bar,pagesize=2048,foo1=bar1,size=107374182",
+                expected,
+                true,
+            ),
+            (
+                "pagesize=2048,foo1=bar1,foo=bar,size=107374182",
+                expected,
+                true,
+            ),
+            (
+                "foo=bar,pagesize=2048,foo1=bar1,size=107374182,foo2=bar2",
+                expected,
+                true,
+            ),
+            (
+                "foo=bar,size=107374182,foo1=bar1,pagesize=2048",
+                expected,
+                true,
+            ),
+        ];
+
+        for case in data {
+            let input = case.0;
+            let r = get_pagesize_and_size_from_option(&[input.to_string()]);
+
+            let is_ok = case.2;
+            if is_ok {
+                let expected = case.1;
+                let (pagesize, size) = r.unwrap();
+                assert_eq!(expected.0, pagesize);
+                assert_eq!(expected.1, size);
+            } else {
+                assert!(r.is_err());
+            }
+        }
+    }
+
+    #[test]
+    fn test_parse_mount_flags_and_options() {
+        #[derive(Debug)]
+        struct TestData<'a> {
+            options_vec: Vec<&'a str>,
+            result: (MsFlags, &'a str),
+        }
+
+        let tests = &[
+            TestData {
+                options_vec: vec![],
+                result: (MsFlags::empty(), ""),
+            },
+            TestData {
+                options_vec: vec!["ro"],
+                result: (MsFlags::MS_RDONLY, ""),
+            },
+            TestData {
+                options_vec: vec!["rw"],
+                result: (MsFlags::empty(), ""),
+            },
+            TestData {
+                options_vec: vec!["ro", "rw"],
+                result: (MsFlags::empty(), ""),
+            },
+            TestData {
+                options_vec: vec!["ro", "nodev"],
+                result: (MsFlags::MS_RDONLY | MsFlags::MS_NODEV, ""),
+            },
+            TestData {
+                options_vec: vec!["option1", "nodev", "option2"],
+                result: (MsFlags::MS_NODEV, "option1,option2"),
+            },
+            TestData {
+                options_vec: vec!["rbind", "", "ro"],
+                result: (MsFlags::MS_BIND | MsFlags::MS_REC | MsFlags::MS_RDONLY, ""),
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let result = parse_mount_flags_and_options(d.options_vec.clone());
+
+            let msg = format!("{}: result: {:?}", msg, result);
+
+            let expected_result = (d.result.0, d.result.1.to_owned());
+            assert_eq!(expected_result, result, "{}", msg);
+        }
+    }
+
+    #[test]
+    fn test_set_ownership() {
+        skip_if_not_root!();
+
+        let logger = slog::Logger::root(slog::Discard, o!());
+
+        #[derive(Debug)]
+        struct TestData<'a> {
+            mount_path: &'a str,
+            fs_group: Option<FSGroup>,
+            read_only: bool,
+            expected_group_id: u32,
+            expected_permission: u32,
+        }
+
+        let tests = &[
+            TestData {
+                mount_path: "foo",
+                fs_group: None,
+                read_only: false,
+                expected_group_id: 0,
+                expected_permission: 0,
+            },
+            TestData {
+                mount_path: "rw_mount",
+                fs_group: Some(FSGroup {
+                    group_id: 3000,
+                    group_change_policy: FSGroupChangePolicy::Always,
+                    unknown_fields: Default::default(),
+                    cached_size: Default::default(),
+                }),
+                read_only: false,
+                expected_group_id: 3000,
+                expected_permission: RW_MASK | EXEC_MASK | MODE_SETGID,
+            },
+            TestData {
+                mount_path: "ro_mount",
+                fs_group: Some(FSGroup {
+                    group_id: 3000,
+                    group_change_policy: FSGroupChangePolicy::OnRootMismatch,
+                    unknown_fields: Default::default(),
+                    cached_size: Default::default(),
+                }),
+                read_only: true,
+                expected_group_id: 3000,
+                expected_permission: RO_MASK | EXEC_MASK | MODE_SETGID,
+            },
+        ];
+
+        let tempdir = tempdir().expect("failed to create tmpdir");
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let mount_dir = tempdir.path().join(d.mount_path);
+            fs::create_dir(&mount_dir)
+                .unwrap_or_else(|_| panic!("{}: failed to create root directory", msg));
+
+            let directory_mode = mount_dir.as_path().metadata().unwrap().permissions().mode();
+            let mut storage_data = Storage::new();
+            if d.read_only {
+                storage_data.set_options(RepeatedField::from_slice(&[
+                    "foo".to_string(),
+                    "ro".to_string(),
+                ]));
+            }
+            if let Some(fs_group) = d.fs_group.clone() {
+                storage_data.set_fs_group(fs_group);
+            }
+            storage_data.mount_point = mount_dir.clone().into_os_string().into_string().unwrap();
+
+            let result = set_ownership(&logger, &storage_data);
+            assert!(result.is_ok());
+
+            assert_eq!(
+                mount_dir.as_path().metadata().unwrap().gid(),
+                d.expected_group_id
+            );
+            assert_eq!(
+                mount_dir.as_path().metadata().unwrap().permissions().mode(),
+                (directory_mode | d.expected_permission)
+            );
+        }
+    }
+
+    #[test]
+    fn test_recursive_ownership_change() {
+        skip_if_not_root!();
+
+        const COUNT: usize = 5;
+
+        #[derive(Debug)]
+        struct TestData<'a> {
+            // Directory where the recursive ownership change should be performed on
+            path: &'a str,
+
+            // User ID for ownership change
+            uid: u32,
+
+            // Group ID for ownership change
+            gid: u32,
+
+            // Set when the permission should be read-only
+            read_only: bool,
+
+            // The expected permission of all directories after ownership change
+            expected_permission_directory: u32,
+
+            // The expected permission of all files after ownership change
+            expected_permission_file: u32,
+        }
+
+        let tests = &[
+            TestData {
+                path: "no_gid_change",
+                uid: 0,
+                gid: 0,
+                read_only: false,
+                expected_permission_directory: 0,
+                expected_permission_file: 0,
+            },
+            TestData {
+                path: "rw_gid_change",
+                uid: 0,
+                gid: 3000,
+                read_only: false,
+                expected_permission_directory: RW_MASK | EXEC_MASK | MODE_SETGID,
+                expected_permission_file: RW_MASK,
+            },
+            TestData {
+                path: "ro_gid_change",
+                uid: 0,
+                gid: 3000,
+                read_only: true,
+                expected_permission_directory: RO_MASK | EXEC_MASK | MODE_SETGID,
+                expected_permission_file: RO_MASK,
+            },
+        ];
+
+        let tempdir = tempdir().expect("failed to create tmpdir");
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let mount_dir = tempdir.path().join(d.path);
+            fs::create_dir(&mount_dir)
+                .unwrap_or_else(|_| panic!("{}: failed to create root directory", msg));
+
+            let directory_mode = mount_dir.as_path().metadata().unwrap().permissions().mode();
+            let mut file_mode: u32 = 0;
+
+            // create testing directories and files
+            for n in 1..COUNT {
+                let nest_dir = mount_dir.join(format!("nested{}", n));
+                fs::create_dir(&nest_dir)
+                    .unwrap_or_else(|_| panic!("{}: failed to create nest directory", msg));
+
+                for f in 1..COUNT {
+                    let filename = nest_dir.join(format!("file{}", f));
+                    File::create(&filename)
+                        .unwrap_or_else(|_| panic!("{}: failed to create file", msg));
+                    file_mode = filename.as_path().metadata().unwrap().permissions().mode();
+                }
+            }
+
+            let uid = if d.uid > 0 {
+                Some(Uid::from_raw(d.uid))
+            } else {
+                None
+            };
+            let gid = if d.gid > 0 {
+                Some(Gid::from_raw(d.gid))
+            } else {
+                None
+            };
+            let result = recursive_ownership_change(&mount_dir, uid, gid, d.read_only);
+
+            assert!(result.is_ok());
+
+            assert_eq!(mount_dir.as_path().metadata().unwrap().gid(), d.gid);
+            assert_eq!(
+                mount_dir.as_path().metadata().unwrap().permissions().mode(),
+                (directory_mode | d.expected_permission_directory)
+            );
+
+            for n in 1..COUNT {
+                let nest_dir = mount_dir.join(format!("nested{}", n));
+                for f in 1..COUNT {
+                    let filename = nest_dir.join(format!("file{}", f));
+                    let file = Path::new(&filename);
+
+                    assert_eq!(file.metadata().unwrap().gid(), d.gid);
+                    assert_eq!(
+                        file.metadata().unwrap().permissions().mode(),
+                        (file_mode | d.expected_permission_file)
+                    );
+                }
+
+                let dir = Path::new(&nest_dir);
+                assert_eq!(dir.metadata().unwrap().gid(), d.gid);
+                assert_eq!(
+                    dir.metadata().unwrap().permissions().mode(),
+                    (directory_mode | d.expected_permission_directory)
+                );
+            }
+        }
+    }
 }

src/agent/src/random.rs

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use anyhow::Result;
+use anyhow::{ensure, Result};
 use nix::errno::Errno;
 use nix::fcntl::{self, OFlag};
 use nix::sys::stat::Mode;
@@ -13,7 +13,7 @@ use tracing::instrument;
 
 pub const RNGDEV: &str = "/dev/random";
 pub const RNDADDTOENTCNT: libc::c_int = 0x40045201;
-pub const RNDRESEEDRNG: libc::c_int = 0x5207;
+pub const RNDRESEEDCRNG: libc::c_int = 0x5207;
 
 // Handle the differing ioctl(2) request types for different targets
 #[cfg(target_env = "musl")]
@@ -24,6 +24,9 @@ type IoctlRequestType = libc::c_ulong;
 #[instrument]
 pub fn reseed_rng(data: &[u8]) -> Result<()> {
     let len = data.len() as libc::c_long;
+
+    ensure!(len > 0, "missing entropy data");
+
     fs::write(RNGDEV, data)?;
 
     let f = {
@@ -41,8 +44,52 @@ pub fn reseed_rng(data: &[u8]) -> Result<()> {
     };
     Errno::result(ret).map(drop)?;
 
-    let ret = unsafe { libc::ioctl(f.as_raw_fd(), RNDRESEEDRNG as IoctlRequestType, 0) };
+    let ret = unsafe { libc::ioctl(f.as_raw_fd(), RNDRESEEDCRNG as IoctlRequestType, 0) };
     Errno::result(ret).map(drop)?;
 
     Ok(())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::skip_if_not_root;
+    use std::fs::File;
+    use std::io::prelude::*;
+
+    #[test]
+    fn test_reseed_rng() {
+        skip_if_not_root!();
+        const POOL_SIZE: usize = 512;
+        let mut f = File::open("/dev/urandom").unwrap();
+        let mut seed = [0; POOL_SIZE];
+        let n = f.read(&mut seed).unwrap();
+        // Ensure the buffer was filled.
+        assert!(n == POOL_SIZE);
+        let ret = reseed_rng(&seed);
+        assert!(ret.is_ok());
+    }
+
+    #[test]
+    fn test_reseed_rng_not_root() {
+        const POOL_SIZE: usize = 512;
+        let mut f = File::open("/dev/urandom").unwrap();
+        let mut seed = [0; POOL_SIZE];
+        let n = f.read(&mut seed).unwrap();
+        // Ensure the buffer was filled.
+        assert!(n == POOL_SIZE);
+        let ret = reseed_rng(&seed);
+        if nix::unistd::Uid::effective().is_root() {
+            assert!(ret.is_ok());
+        } else {
+            assert!(!ret.is_ok());
+        }
+    }
+
+    #[test]
+    fn test_reseed_rng_zero_data() {
+        let seed = [];
+        let ret = reseed_rng(&seed);
+        assert!(!ret.is_ok());
+    }
+}

src/agent/src/sandbox.rs

@@ -8,6 +8,7 @@ use crate::mount::{get_mount_fs_type, remove_mounts, TYPE_ROOTFS};
 use crate::namespace::Namespace;
 use crate::netlink::Handle;
 use crate::network::Network;
+use crate::pci;
 use crate::uevent::{Uevent, UeventMatcher};
 use crate::watcher::BindWatcher;
 use anyhow::{anyhow, Context, Result};
@@ -31,6 +32,8 @@ use tokio::sync::oneshot;
 use tokio::sync::Mutex;
 use tracing::instrument;
 
+pub const ERR_INVALID_CONTAINER_ID: &str = "Invalid container id";
+
 type UeventWatcher = (Box<dyn UeventMatcher>, oneshot::Sender<Uevent>);
 
 #[derive(Debug)]
@@ -56,6 +59,7 @@ pub struct Sandbox {
     pub event_rx: Arc<Mutex<Receiver<String>>>,
     pub event_tx: Option<Sender<String>>,
     pub bind_watcher: BindWatcher,
+    pub pcimap: HashMap<pci::Address, pci::Address>,
 }
 
 impl Sandbox {
@@ -88,6 +92,7 @@ impl Sandbox {
             event_rx,
             event_tx: Some(tx),
             bind_watcher: BindWatcher::new(),
+            pcimap: HashMap::new(),
         })
     }
 
@@ -146,7 +151,12 @@ impl Sandbox {
     pub fn remove_sandbox_storage(&self, path: &str) -> Result<()> {
         let mounts = vec![path.to_string()];
         remove_mounts(&mounts)?;
-        fs::remove_dir_all(path).context(format!("failed to remove dir {:?}", path))?;
+        // "remove_dir" will fail if the mount point is backed by a read-only filesystem.
+        // This is the case with the device mapper snapshotter, where we mount the block device directly
+        // at the underlying sandbox path which was provided from the base RO kataShared path from the host.
+        if let Err(err) = fs::remove_dir(path) {
+            warn!(self.logger, "failed to remove dir {}, {:?}", path, err);
+        }
         Ok(())
     }
 
@@ -229,7 +239,7 @@ impl Sandbox {
     pub fn find_container_process(&mut self, cid: &str, eid: &str) -> Result<&mut Process> {
         let ctr = self
             .get_container(cid)
-            .ok_or_else(|| anyhow!("Invalid container id"))?;
+            .ok_or_else(|| anyhow!(ERR_INVALID_CONTAINER_ID))?;
 
         if eid.is_empty() {
             return ctr
@@ -460,7 +470,7 @@ fn online_memory(logger: &Logger) -> Result<()> {
 
 #[cfg(test)]
 mod tests {
-    use super::Sandbox;
+    use super::*;
     use crate::{mount::baremount, skip_if_not_root};
     use anyhow::{anyhow, Error};
     use nix::mount::MsFlags;
@@ -470,6 +480,7 @@ mod tests {
     use rustjail::specconv::CreateOpts;
     use slog::Logger;
     use std::fs::{self, File};
+    use std::io::prelude::*;
     use std::os::unix::fs::PermissionsExt;
     use std::path::Path;
     use tempfile::{tempdir, Builder, TempDir};
@@ -548,7 +559,7 @@ mod tests {
 
         assert!(
             s.remove_sandbox_storage(srcdir_path).is_err(),
-            "Expect Err as the directory i not a mountpoint"
+            "Expect Err as the directory is not a mountpoint"
         );
 
         assert!(s.remove_sandbox_storage("").is_err());
@@ -559,19 +570,8 @@ mod tests {
             .remove_sandbox_storage(invalid_dir.to_str().unwrap())
             .is_err());
 
-        // Now, create a double mount as this guarantees the directory cannot
-        // be deleted after the first umount.
-        for _i in 0..2 {
-            assert!(bind_mount(srcdir_path, destdir_path, &logger).is_ok());
-        }
+        assert!(bind_mount(srcdir_path, destdir_path, &logger).is_ok());
 
-        assert!(
-            s.remove_sandbox_storage(destdir_path).is_err(),
-            "Expect fail as deletion cannot happen due to the second mount."
-        );
-
-        // This time it should work as the previous two calls have undone the double
-        // mount.
         assert!(s.remove_sandbox_storage(destdir_path).is_ok());
     }
 
@@ -583,7 +583,6 @@ mod tests {
         let logger = slog::Logger::root(slog::Discard, o!());
         let mut s = Sandbox::new(&logger).unwrap();
 
-        // FIX: This test fails, not sure why yet.
         assert!(
             s.unset_and_remove_sandbox_storage("/tmp/testEphePath")
                 .is_err(),
@@ -849,4 +848,259 @@ mod tests {
         let p = s.find_container_process("not-exist-cid", "");
         assert!(p.is_err(), "Expecting Error, Got {:?}", p);
     }
+
+    #[tokio::test]
+    async fn test_find_process() {
+        let logger = slog::Logger::root(slog::Discard, o!());
+
+        let test_pids = [std::i32::MIN, -1, 0, 1, std::i32::MAX];
+
+        for test_pid in test_pids {
+            let mut s = Sandbox::new(&logger).unwrap();
+            let (mut linux_container, _root) = create_linuxcontainer();
+
+            let mut test_process = Process::new(
+                &logger,
+                &oci::Process::default(),
+                "this_is_a_test_process",
+                true,
+                1,
+            )
+            .unwrap();
+            // processes interally only have pids when manually set
+            test_process.pid = test_pid;
+
+            linux_container.processes.insert(test_pid, test_process);
+
+            s.add_container(linux_container);
+
+            let find_result = s.find_process(test_pid);
+
+            // test first if it finds anything
+            assert!(find_result.is_some(), "Should be able to find a process");
+
+            let found_process = find_result.unwrap();
+
+            // then test if it founds the correct process
+            assert_eq!(
+                found_process.pid, test_pid,
+                "Should be able to find correct process"
+            );
+        }
+
+        // to test for nonexistent pids, any pid that isn't the one set
+        // above should work, as linuxcontainer starts with no processes
+        let mut s = Sandbox::new(&logger).unwrap();
+
+        let nonexistent_test_pid = 1234;
+
+        let find_result = s.find_process(nonexistent_test_pid);
+
+        assert!(
+            find_result.is_none(),
+            "Shouldn't find a process for non existent pid"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_online_resources() {
+        #[derive(Debug, Default)]
+        struct TestFile {
+            name: String,
+            content: String,
+        }
+
+        #[derive(Debug, Default)]
+        struct TestDirectory<'a> {
+            name: String,
+            files: &'a [TestFile],
+        }
+
+        #[derive(Debug)]
+        struct TestData<'a> {
+            directory_autogen_name: String,
+            number_autogen_directories: u32,
+
+            extra_directories: &'a [TestDirectory<'a>],
+            pattern: String,
+            to_enable: i32,
+
+            result: Result<i32>,
+        }
+
+        impl Default for TestData<'_> {
+            fn default() -> Self {
+                TestData {
+                    directory_autogen_name: Default::default(),
+                    number_autogen_directories: Default::default(),
+                    extra_directories: Default::default(),
+                    pattern: Default::default(),
+                    to_enable: Default::default(),
+                    result: Ok(Default::default()),
+                }
+            }
+        }
+
+        let tests = &[
+            // 4 well formed directories, request enabled 4,
+            // correct result 4 enabled, should pass
+            TestData {
+                directory_autogen_name: String::from("cpu"),
+                number_autogen_directories: 4,
+                pattern: String::from(r"cpu[0-9]+"),
+                to_enable: 4,
+                result: Ok(4),
+                ..Default::default()
+            },
+            // 0 well formed directories, request enabled 4,
+            // correct result 0 enabled, should pass
+            TestData {
+                number_autogen_directories: 0,
+                to_enable: 4,
+                result: Ok(0),
+                ..Default::default()
+            },
+            // 10 well formed directories, request enabled 4,
+            // correct result 4 enabled, should pass
+            TestData {
+                directory_autogen_name: String::from("cpu"),
+                number_autogen_directories: 10,
+                pattern: String::from(r"cpu[0-9]+"),
+                to_enable: 4,
+                result: Ok(4),
+                ..Default::default()
+            },
+            // 0 well formed directories, request enabled 0,
+            // correct result 0 enabled, should pass
+            TestData {
+                number_autogen_directories: 0,
+                pattern: String::from(r"cpu[0-9]+"),
+                to_enable: 0,
+                result: Ok(0),
+                ..Default::default()
+            },
+            // 4 well formed directories, 1 malformed (no online file),
+            // request enable 5, correct result 4
+            TestData {
+                directory_autogen_name: String::from("cpu"),
+                number_autogen_directories: 4,
+                pattern: String::from(r"cpu[0-9]+"),
+                extra_directories: &[TestDirectory {
+                    name: String::from("cpu4"),
+                    files: &[],
+                }],
+                to_enable: 5,
+                result: Ok(4),
+            },
+            // 3 malformed directories (no online files),
+            // request enable 3, correct result 0
+            TestData {
+                pattern: String::from(r"cpu[0-9]+"),
+                extra_directories: &[
+                    TestDirectory {
+                        name: String::from("cpu0"),
+                        files: &[],
+                    },
+                    TestDirectory {
+                        name: String::from("cpu1"),
+                        files: &[],
+                    },
+                    TestDirectory {
+                        name: String::from("cpu2"),
+                        files: &[],
+                    },
+                ],
+                to_enable: 3,
+                result: Ok(0),
+                ..Default::default()
+            },
+            // 1 malformed directories (online file with content "1"),
+            // request enable 1, correct result 0
+            TestData {
+                pattern: String::from(r"cpu[0-9]+"),
+                extra_directories: &[TestDirectory {
+                    name: String::from("cpu0"),
+                    files: &[TestFile {
+                        name: SYSFS_ONLINE_FILE.to_string(),
+                        content: String::from("1"),
+                    }],
+                }],
+                to_enable: 1,
+                result: Ok(0),
+                ..Default::default()
+            },
+            // 2 well formed directories, 1 malformed (online file with content "1"),
+            // request enable 3, correct result 2
+            TestData {
+                directory_autogen_name: String::from("cpu"),
+                number_autogen_directories: 2,
+                pattern: String::from(r"cpu[0-9]+"),
+                extra_directories: &[TestDirectory {
+                    name: String::from("cpu2"),
+                    files: &[TestFile {
+                        name: SYSFS_ONLINE_FILE.to_string(),
+                        content: String::from("1"),
+                    }],
+                }],
+                to_enable: 3,
+                result: Ok(2),
+            },
+        ];
+
+        let logger = slog::Logger::root(slog::Discard, o!());
+        let tmpdir = Builder::new().tempdir().unwrap();
+        let tmpdir_path = tmpdir.path().to_str().unwrap();
+
+        for (i, d) in tests.iter().enumerate() {
+            let current_test_dir_path = format!("{}/test_{}", tmpdir_path, i);
+            fs::create_dir(&current_test_dir_path).unwrap();
+
+            // create numbered directories and fill using root name
+            for j in 0..d.number_autogen_directories {
+                let subdir_path = format!(
+                    "{}/{}{}",
+                    current_test_dir_path, d.directory_autogen_name, j
+                );
+                let subfile_path = format!("{}/{}", subdir_path, SYSFS_ONLINE_FILE);
+                fs::create_dir(&subdir_path).unwrap();
+                let mut subfile = File::create(subfile_path).unwrap();
+                subfile.write_all(b"0").unwrap();
+            }
+            // create extra directories and fill to specification
+            for j in d.extra_directories {
+                let subdir_path = format!("{}/{}", current_test_dir_path, j.name);
+                fs::create_dir(&subdir_path).unwrap();
+                for file in j.files {
+                    let subfile_path = format!("{}/{}", subdir_path, file.name);
+                    let mut subfile = File::create(&subfile_path).unwrap();
+                    subfile.write_all(file.content.as_bytes()).unwrap();
+                }
+            }
+
+            // run created directory structure against online_resources
+            let result = online_resources(&logger, &current_test_dir_path, &d.pattern, d.to_enable);
+
+            let mut msg = format!(
+                "test[{}]: {:?}, expected {}, actual {}",
+                i,
+                d,
+                d.result.is_ok(),
+                result.is_ok()
+            );
+
+            assert_eq!(result.is_ok(), d.result.is_ok(), "{}", msg);
+
+            if d.result.is_ok() {
+                let test_result_val = *d.result.as_ref().ok().unwrap();
+                let result_val = result.ok().unwrap();
+
+                msg = format!(
+                    "test[{}]: {:?}, expected {}, actual {}",
+                    i, d, test_result_val, result_val
+                );
+
+                assert_eq!(test_result_val, result_val, "{}", msg);
+            }
+        }
+    }
 }

src/agent/src/signal.rs

@@ -58,17 +58,16 @@ async fn handle_sigchild(logger: Logger, sandbox: Arc<Mutex<Sandbox>>) -> Result
             }
 
             let mut p = process.unwrap();
-            let ret: i32;
 
-            match wait_status {
-                WaitStatus::Exited(_, c) => ret = c,
-                WaitStatus::Signaled(_, sig, _) => ret = sig as i32,
+            let ret: i32 = match wait_status {
+                WaitStatus::Exited(_, c) => c,
+                WaitStatus::Signaled(_, sig, _) => sig as i32,
                 _ => {
                     info!(logger, "got wrong status for process";
                                   "child-status" => format!("{:?}", wait_status));
                     continue;
                 }
-            }
+            };
 
             p.exit_code = ret;
             let _ = p.exit_tx.take();

src/agent/src/test_utils.rs

@@ -5,7 +5,14 @@
 #![allow(clippy::module_inception)]
 
 #[cfg(test)]
-mod test_utils {
+pub mod test_utils {
+    #[derive(Debug, PartialEq)]
+    pub enum TestUserType {
+        RootOnly,
+        NonRootOnly,
+        Any,
+    }
+
     #[macro_export]
     macro_rules! skip_if_root {
         () => {
@@ -53,4 +60,40 @@ mod test_utils {
             }
         };
     }
+
+    // Parameters:
+    //
+    // 1: expected Result
+    // 2: actual Result
+    // 3: string used to identify the test on error
+    #[macro_export]
+    macro_rules! assert_result {
+        ($expected_result:expr, $actual_result:expr, $msg:expr) => {
+            if $expected_result.is_ok() {
+                let expected_value = $expected_result.as_ref().unwrap();
+                let actual_value = $actual_result.unwrap();
+                assert!(*expected_value == actual_value, "{}", $msg);
+            } else {
+                assert!($actual_result.is_err(), "{}", $msg);
+
+                let expected_error = $expected_result.as_ref().unwrap_err();
+                let expected_error_msg = format!("{:?}", expected_error);
+
+                let actual_error_msg = format!("{:?}", $actual_result.unwrap_err());
+
+                assert!(expected_error_msg == actual_error_msg, "{}", $msg);
+            }
+        };
+    }
+
+    #[macro_export]
+    macro_rules! skip_loop_by_user {
+        ($msg:expr, $user:expr) => {
+            if $user == TestUserType::RootOnly {
+                skip_loop_if_not_root!($msg);
+            } else if $user == TestUserType::NonRootOnly {
+                skip_loop_if_root!($msg);
+            }
+        };
+    }
 }

src/agent/src/util.rs

@@ -237,8 +237,6 @@ mod tests {
                 JoinError,
             >;
 
-            let result: std::result::Result<u64, std::io::Error>;
-
             select! {
                 res = handle => spawn_result = res,
                 _ = &mut timeout => panic!("timed out"),
@@ -246,7 +244,7 @@ mod tests {
 
             assert!(spawn_result.is_ok());
 
-            result = spawn_result.unwrap();
+            let result: std::result::Result<u64, std::io::Error> = spawn_result.unwrap();
 
             assert!(result.is_ok());
 
@@ -278,8 +276,6 @@ mod tests {
 
         let spawn_result: std::result::Result<std::result::Result<u64, std::io::Error>, JoinError>;
 
-        let result: std::result::Result<u64, std::io::Error>;
-
         select! {
             res = handle => spawn_result = res,
             _ = &mut timeout => panic!("timed out"),
@@ -287,7 +283,7 @@ mod tests {
 
         assert!(spawn_result.is_ok());
 
-        result = spawn_result.unwrap();
+        let result: std::result::Result<u64, std::io::Error> = spawn_result.unwrap();
 
         assert!(result.is_ok());
 
@@ -320,8 +316,6 @@ mod tests {
 
         let spawn_result: std::result::Result<std::result::Result<u64, std::io::Error>, JoinError>;
 
-        let result: std::result::Result<u64, std::io::Error>;
-
         select! {
             res = handle => spawn_result = res,
             _ = &mut timeout => panic!("timed out"),
@@ -329,7 +323,7 @@ mod tests {
 
         assert!(spawn_result.is_ok());
 
-        result = spawn_result.unwrap();
+        let result: std::result::Result<u64, std::io::Error> = spawn_result.unwrap();
 
         assert!(result.is_ok());
 

src/agent/src/watcher.rs

@@ -6,6 +6,7 @@
 #![allow(unknown_lints)]
 
 use std::collections::HashMap;
+use std::os::unix::fs::MetadataExt;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::SystemTime;
@@ -13,6 +14,7 @@ use std::time::SystemTime;
 use anyhow::{ensure, Context, Result};
 use async_recursion::async_recursion;
 use nix::mount::{umount, MsFlags};
+use nix::unistd::{Gid, Uid};
 use slog::{debug, error, info, warn, Logger};
 use thiserror::Error;
 use tokio::fs;
@@ -80,7 +82,8 @@ impl Drop for Storage {
 }
 
 async fn copy(from: impl AsRef<Path>, to: impl AsRef<Path>) -> Result<()> {
-    if fs::symlink_metadata(&from).await?.file_type().is_symlink() {
+    let metadata = fs::symlink_metadata(&from).await?;
+    if metadata.file_type().is_symlink() {
         // if source is a symlink, create new symlink with same link source. If
         // the symlink exists, remove and create new one:
         if fs::symlink_metadata(&to).await.is_ok() {
@@ -88,8 +91,15 @@ async fn copy(from: impl AsRef<Path>, to: impl AsRef<Path>) -> Result<()> {
         }
         fs::symlink(fs::read_link(&from).await?, &to).await?;
     } else {
-        fs::copy(from, to).await?;
+        fs::copy(&from, &to).await?;
     }
+    // preserve the source uid and gid to the destination.
+    nix::unistd::chown(
+        to.as_ref(),
+        Some(Uid::from_raw(metadata.uid())),
+        Some(Gid::from_raw(metadata.gid())),
+    )?;
+
     Ok(())
 }
 
@@ -106,14 +116,29 @@ impl Storage {
 
     async fn update_target(&self, logger: &Logger, source_path: impl AsRef<Path>) -> Result<()> {
         let source_file_path = source_path.as_ref();
+        let metadata = source_file_path.symlink_metadata()?;
 
         // if we are creating a directory: just create it, nothing more to do
-        if source_file_path.symlink_metadata()?.file_type().is_dir() {
+        if metadata.file_type().is_dir() {
             let dest_file_path = self.make_target_path(&source_file_path)?;
 
             fs::create_dir_all(&dest_file_path)
                 .await
                 .with_context(|| format!("Unable to mkdir all for {}", dest_file_path.display()))?;
+            // set the directory permissions to match the source directory permissions
+            fs::set_permissions(&dest_file_path, metadata.permissions())
+                .await
+                .with_context(|| {
+                    format!("Unable to set permissions for {}", dest_file_path.display())
+                })?;
+            // preserve the source directory uid and gid to the destination.
+            nix::unistd::chown(
+                &dest_file_path,
+                Some(Uid::from_raw(metadata.uid())),
+                Some(Gid::from_raw(metadata.gid())),
+            )
+            .with_context(|| format!("Unable to set ownership for {}", dest_file_path.display()))?;
+
             return Ok(());
         }
 
@@ -504,6 +529,7 @@ mod tests {
     use super::*;
     use crate::mount::is_mounted;
     use crate::skip_if_not_root;
+    use nix::unistd::{Gid, Uid};
     use std::fs;
     use std::thread;
 
@@ -895,20 +921,28 @@ mod tests {
 
     #[tokio::test]
     async fn test_copy() {
+        skip_if_not_root!();
+
         // prepare tmp src/destination
         let source_dir = tempfile::tempdir().unwrap();
         let dest_dir = tempfile::tempdir().unwrap();
+        let uid = Uid::from_raw(10);
+        let gid = Gid::from_raw(200);
 
         // verify copy of a regular file
         let src_file = source_dir.path().join("file.txt");
         let dst_file = dest_dir.path().join("file.txt");
         fs::write(&src_file, "foo").unwrap();
+        nix::unistd::chown(&src_file, Some(uid), Some(gid)).unwrap();
+
         copy(&src_file, &dst_file).await.unwrap();
         // verify destination:
-        assert!(!fs::symlink_metadata(dst_file)
+        assert!(!fs::symlink_metadata(&dst_file)
             .unwrap()
             .file_type()
             .is_symlink());
+        assert_eq!(fs::metadata(&dst_file).unwrap().uid(), uid.as_raw());
+        assert_eq!(fs::metadata(&dst_file).unwrap().gid(), gid.as_raw());
 
         // verify copy of a symlink
         let src_symlink_file = source_dir.path().join("symlink_file.txt");
@@ -916,7 +950,7 @@ mod tests {
         tokio::fs::symlink(&src_file, &src_symlink_file)
             .await
             .unwrap();
-        copy(src_symlink_file, &dst_symlink_file).await.unwrap();
+        copy(&src_symlink_file, &dst_symlink_file).await.unwrap();
         // verify destination:
         assert!(fs::symlink_metadata(&dst_symlink_file)
             .unwrap()
@@ -924,6 +958,8 @@ mod tests {
             .is_symlink());
         assert_eq!(fs::read_link(&dst_symlink_file).unwrap(), src_file);
         assert_eq!(fs::read_to_string(&dst_symlink_file).unwrap(), "foo");
+        assert_ne!(fs::metadata(&dst_symlink_file).unwrap().uid(), uid.as_raw());
+        assert_ne!(fs::metadata(&dst_symlink_file).unwrap().gid(), gid.as_raw());
     }
 
     #[tokio::test]
@@ -1069,6 +1105,8 @@ mod tests {
 
     #[tokio::test]
     async fn watch_directory() {
+        skip_if_not_root!();
+
         // Prepare source directory:
         // ./tmp/1.txt
         // ./tmp/A/B/2.txt
@@ -1079,7 +1117,9 @@ mod tests {
 
         // A/C is an empty directory
         let empty_dir = "A/C";
-        fs::create_dir_all(source_dir.path().join(empty_dir)).unwrap();
+        let path = source_dir.path().join(empty_dir);
+        fs::create_dir_all(&path).unwrap();
+        nix::unistd::chown(&path, Some(Uid::from_raw(10)), Some(Gid::from_raw(200))).unwrap();
 
         // delay 20 ms between writes to files in order to ensure filesystem timestamps are unique
         thread::sleep(Duration::from_millis(20));
@@ -1123,7 +1163,9 @@ mod tests {
 
         // create another empty directory A/C/D
         let empty_dir = "A/C/D";
-        fs::create_dir_all(source_dir.path().join(empty_dir)).unwrap();
+        let path = source_dir.path().join(empty_dir);
+        fs::create_dir_all(&path).unwrap();
+        nix::unistd::chown(&path, Some(Uid::from_raw(10)), Some(Gid::from_raw(200))).unwrap();
         assert_eq!(entry.scan(&logger).await.unwrap(), 1);
         assert!(dest_dir.path().join(empty_dir).exists());
     }