Merge pull request #2257 from fidencio/wip/runtime-option-fix

shim-v2: Be compatible with the old runtime options

.github/workflows/kata-deploy-test.yaml

@@ -1,9 +1,12 @@
-on: issue_comment
+on:
+  issue_comment:
+    types: [created, edited]
+
 name: test-kata-deploy
+
 jobs:
   check_comments:
     if: ${{ github.event.issue.pull_request }}
-    types: [created, edited]
     runs-on: ubuntu-latest
     steps:
       - name: Check for Command
@@ -11,7 +14,7 @@ jobs:
         uses: kata-containers/slash-command-action@v1
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-          command: "test-kata-deploy"
+          command: "test_kata_deploy"
           reaction: "true"
           reaction-type: "eyes"
           allow-edits: "false"
@@ -19,6 +22,7 @@ jobs:
       - name: verify command arg is kata-deploy
         run: |
            echo "The command was '${{ steps.command.outputs.command-name }}' with arguments '${{ steps.command.outputs.command-arguments }}'"
+
   create-and-test-container:
     needs: check_comments
     runs-on: ubuntu-latest
@@ -29,22 +33,26 @@ jobs:
             ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
             echo "reference for PR: " ${ref}
             echo "##[set-output name=pr-ref;]${ref}"
-      - uses: actions/checkout@v2-beta
+
+      - name: check out
+        uses: actions/checkout@v2
         with:
-          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
+           ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
+
       - name: build-container-image
         id: build-container-image
         run: |
             PR_SHA=$(git log --format=format:%H -n1)
-            VERSION=$(curl https://raw.githubusercontent.com/kata-containers/kata-containers/main/VERSION)
+            VERSION="2.0.0"
             ARTIFACT_URL="https://github.com/kata-containers/kata-containers/releases/download/${VERSION}/kata-static-${VERSION}-x86_64.tar.xz"
-            wget "${ARTIFACT_URL}" -O ./kata-deploy/kata-static.tar.xz
-            docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:${PR_SHA} ./kata-deploy
+            wget "${ARTIFACT_URL}" -O tools/packaging/kata-deploy/kata-static.tar.xz
+            docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:${PR_SHA} ./tools/packaging/kata-deploy
             docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
             docker push katadocker/kata-deploy-ci:$PR_SHA
             echo "##[set-output name=pr-sha;]${PR_SHA}"
+
       - name: test-kata-deploy-ci-in-aks
-        uses: ./kata-deploy/action
+        uses: ./tools/packaging/kata-deploy/action
         with:
           packaging-sha: ${{ steps.build-container-image.outputs.pr-sha }}
         env:

.github/workflows/require-pr-porting-labels.yaml

@@ -12,6 +12,9 @@ on:
       - reopened
       - labeled
       - unlabeled
+   pull_request:
+     branches:
+      - main
 
 jobs:
   check-pr-porting-labels:

.github/workflows/snap-release.yaml

@@ -9,6 +9,8 @@ jobs:
     steps:
       - name: Check out Git repository
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
 
       - name: Install Snapcraft
         uses: samuelmeuli/action-snapcraft@v1
@@ -33,5 +35,5 @@ jobs:
           snap_file="kata-containers_${snap_version}_amd64.snap"
           # Upload the snap if it exists
           if [ -f ${snap_file} ]; then
-            snapcraft upload --release=candidate ${snap_file}
+            snapcraft upload --release=stable ${snap_file}
           fi

.github/workflows/snap.yaml

@@ -6,6 +6,8 @@ jobs:
     steps:
       - name: Check out
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
 
       - name: Install Snapcraft
         uses: samuelmeuli/action-snapcraft@v1

.github/workflows/static-checks.yaml

@@ -1,10 +1,18 @@
-on: ["pull_request"]
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - labeled
+      - unlabeled
+
 name: Static checks
 jobs:
   test:
+    if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
     strategy:
       matrix:
-        go-version: [1.13.x, 1.14.x, 1.15.x]
+        go-version: [1.15.x, 1.16.x]
         os: [ubuntu-20.04]
     runs-on: ${{ matrix.os }}
     env:
@@ -13,7 +21,7 @@ jobs:
       TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }}
       TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }}
       RUST_BACKTRACE: "1"
-      target_branch: ${TRAVIS_BRANCH}
+      target_branch: ${{ github.base_ref }}
     steps:
     - name: Install Go
       uses: actions/setup-go@v2
@@ -51,6 +59,10 @@ jobs:
         PATH=$PATH:"$HOME/.cargo/bin"
         rustup target add x86_64-unknown-linux-musl
         rustup component add rustfmt clippy
+    # Check whether the vendored code is up-to-date & working as the first thing
+    - name: Check vendored code
+      run: |
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && make vendor
     # Must build before static checks as we depend on some generated code in runtime and agent
     - name: Build
       run: |

Makefile

@@ -15,7 +15,7 @@ TOOLS =
 
 TOOLS += agent-ctl
 
-STANDARD_TARGETS = build check clean install test
+STANDARD_TARGETS = build check clean install test vendor
 
 include utils.mk
 

VERSION

@@ -1 +1 @@
-2.1-alpha1
+2.2.0-alpha1

ci/install_yq.sh

@@ -18,7 +18,9 @@ function install_yq() {
 	GOPATH=${GOPATH:-${HOME}/go}
 	local yq_path="${GOPATH}/bin/yq"
 	local yq_pkg="github.com/mikefarah/yq"
-	[ -x  "${GOPATH}/bin/yq" ] && return
+	local yq_version=3.4.1
+
+	[ -x  "${GOPATH}/bin/yq" ] && [ "`${GOPATH}/bin/yq --version`"X == "yq version ${yq_version}"X ] && return
 
 	read -r -a sysInfo <<< "$(uname -sm)"
 
@@ -56,8 +58,6 @@ function install_yq() {
 		die "Please install curl"
 	fi
 
-	local yq_version=3.4.1
-
 	## NOTE: ${var,,} => gives lowercase value of var
 	local yq_url="https://${yq_pkg}/releases/download/${yq_version}/yq_${goos,,}_${goarch}"
 	curl -o "${yq_path}" -LSsf "${yq_url}"

ci/lib.sh

@@ -5,18 +5,27 @@
 
 export tests_repo="${tests_repo:-github.com/kata-containers/tests}"
 export tests_repo_dir="$GOPATH/src/$tests_repo"
-export branch="${branch:-main}"
+export branch="${target_branch:-main}"
 
+# Clones the tests repository and checkout to the branch pointed out by
+# the global $branch variable.
+# If the clone exists and `CI` is exported then it does nothing. Otherwise
+# it will clone the repository or `git pull` the latest code.
+#
 clone_tests_repo()
 {
-	if [ -d "$tests_repo_dir" -a -n "$CI" ]
-	then
-		return
+	if [ -d "$tests_repo_dir" ]; then
+		[ -n "$CI" ] && return
+		pushd "${tests_repo_dir}"
+		git checkout "${branch}"
+		git pull
+		popd
+	else
+		git clone -q "https://${tests_repo}" "$tests_repo_dir"
+		pushd "${tests_repo_dir}"
+		git checkout "${branch}"
+		popd
 	fi
-
-	go get -d -u "$tests_repo" || true
-
-	pushd "${tests_repo_dir}" && git checkout "${branch}" && popd
 }
 
 run_static_checks()

ci/openshift-ci/images/Dockerfile.buildroot

@@ -0,0 +1,9 @@
+# Copyright (c) 2021 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# This is the build root image for Kata Containers on OpenShift CI.
+#
+FROM centos:8
+
+RUN yum -y update && yum -y install git sudo wget

docs/Developer-Guide.md

@@ -1,54 +1,55 @@
-* [Warning](#warning)
-* [Assumptions](#assumptions)
-* [Initial setup](#initial-setup)
-* [Requirements to build individual components](#requirements-to-build-individual-components)
-* [Build and install the Kata Containers runtime](#build-and-install-the-kata-containers-runtime)
-* [Check hardware requirements](#check-hardware-requirements)
-    * [Configure to use initrd or rootfs image](#configure-to-use-initrd-or-rootfs-image)
-    * [Enable full debug](#enable-full-debug)
-        * [debug logs and shimv2](#debug-logs-and-shimv2)
-            * [Enabling full `containerd` debug](#enabling-full-containerd-debug)
-            * [Enabling just `containerd shim` debug](#enabling-just-containerd-shim-debug)
-            * [Enabling `CRI-O` and `shimv2` debug](#enabling-cri-o-and-shimv2-debug)
-        * [journald rate limiting](#journald-rate-limiting)
-            * [`systemd-journald` suppressing messages](#systemd-journald-suppressing-messages)
-            * [Disabling `systemd-journald` rate limiting](#disabling-systemd-journald-rate-limiting)
-* [Create and install rootfs and initrd image](#create-and-install-rootfs-and-initrd-image)
-    * [Build a custom Kata agent - OPTIONAL](#build-a-custom-kata-agent---optional)
-    * [Get the osbuilder](#get-the-osbuilder)
-    * [Create a rootfs image](#create-a-rootfs-image)
-        * [Create a local rootfs](#create-a-local-rootfs)
-        * [Add a custom agent to the image - OPTIONAL](#add-a-custom-agent-to-the-image---optional)
-        * [Build a rootfs image](#build-a-rootfs-image)
-        * [Install the rootfs image](#install-the-rootfs-image)
-    * [Create an initrd image - OPTIONAL](#create-an-initrd-image---optional)
-        * [Create a local rootfs for initrd image](#create-a-local-rootfs-for-initrd-image)
-        * [Build an initrd image](#build-an-initrd-image)
-        * [Install the initrd image](#install-the-initrd-image)
-* [Install guest kernel images](#install-guest-kernel-images)
-* [Install a hypervisor](#install-a-hypervisor)
-    * [Build a custom QEMU](#build-a-custom-qemu)
-        * [Build a custom QEMU for aarch64/arm64 - REQUIRED](#build-a-custom-qemu-for-aarch64arm64---required)
-* [Run Kata Containers with Containerd](#run-kata-containers-with-containerd)
-* [Run Kata Containers with Kubernetes](#run-kata-containers-with-kubernetes)
-* [Troubleshoot Kata Containers](#troubleshoot-kata-containers)
-* [Appendices](#appendices)
-    * [Checking Docker default runtime](#checking-docker-default-runtime)
-    * [Set up a debug console](#set-up-a-debug-console)
-      * [Simple debug console setup](#simple-debug-console-setup)
-          * [Enable agent debug console](#enable-agent-debug-console)
-          * [Connect to debug console](#connect-to-debug-console)
-      * [Traditional debug console setup](#traditional-debug-console-setup)
-          * [Create a custom image containing a shell](#create-a-custom-image-containing-a-shell)
-          * [Build the debug image](#build-the-debug-image)
-          * [Configure runtime for custom debug image](#configure-runtime-for-custom-debug-image)
-          * [Connect to the virtual machine using the debug console](#connect-to-the-virtual-machine-using-the-debug-console)
-              * [Enabling debug console for QEMU](#enabling-debug-console-for-qemu)
-              * [Enabling debug console for cloud-hypervisor / firecracker](#enabling-debug-console-for-cloud-hypervisor--firecracker)
-          * [Create a container](#create-a-container)
-          * [Connect to the virtual machine using the debug console](#connect-to-the-virtual-machine-using-the-debug-console)
-    * [Obtain details of the image](#obtain-details-of-the-image)
-    * [Capturing kernel boot logs](#capturing-kernel-boot-logs)
+- [Warning](#warning)
+- [Assumptions](#assumptions)
+- [Initial setup](#initial-setup)
+- [Requirements to build individual components](#requirements-to-build-individual-components)
+- [Build and install the Kata Containers runtime](#build-and-install-the-kata-containers-runtime)
+- [Check hardware requirements](#check-hardware-requirements)
+  - [Configure to use initrd or rootfs image](#configure-to-use-initrd-or-rootfs-image)
+  - [Enable full debug](#enable-full-debug)
+    - [debug logs and shimv2](#debug-logs-and-shimv2)
+      - [Enabling full `containerd` debug](#enabling-full-containerd-debug)
+      - [Enabling just `containerd shim` debug](#enabling-just-containerd-shim-debug)
+      - [Enabling `CRI-O` and `shimv2` debug](#enabling-cri-o-and-shimv2-debug)
+    - [journald rate limiting](#journald-rate-limiting)
+      - [`systemd-journald` suppressing messages](#systemd-journald-suppressing-messages)
+      - [Disabling `systemd-journald` rate limiting](#disabling-systemd-journald-rate-limiting)
+- [Create and install rootfs and initrd image](#create-and-install-rootfs-and-initrd-image)
+  - [Build a custom Kata agent - OPTIONAL](#build-a-custom-kata-agent---optional)
+  - [Get the osbuilder](#get-the-osbuilder)
+  - [Create a rootfs image](#create-a-rootfs-image)
+    - [Create a local rootfs](#create-a-local-rootfs)
+    - [Add a custom agent to the image - OPTIONAL](#add-a-custom-agent-to-the-image---optional)
+    - [Build a rootfs image](#build-a-rootfs-image)
+    - [Install the rootfs image](#install-the-rootfs-image)
+  - [Create an initrd image - OPTIONAL](#create-an-initrd-image---optional)
+    - [Create a local rootfs for initrd image](#create-a-local-rootfs-for-initrd-image)
+    - [Build an initrd image](#build-an-initrd-image)
+    - [Install the initrd image](#install-the-initrd-image)
+- [Install guest kernel images](#install-guest-kernel-images)
+- [Install a hypervisor](#install-a-hypervisor)
+  - [Build a custom QEMU](#build-a-custom-qemu)
+    - [Build a custom QEMU for aarch64/arm64 - REQUIRED](#build-a-custom-qemu-for-aarch64arm64---required)
+- [Run Kata Containers with Containerd](#run-kata-containers-with-containerd)
+- [Run Kata Containers with Kubernetes](#run-kata-containers-with-kubernetes)
+- [Troubleshoot Kata Containers](#troubleshoot-kata-containers)
+- [Appendices](#appendices)
+  - [Checking Docker default runtime](#checking-docker-default-runtime)
+  - [Set up a debug console](#set-up-a-debug-console)
+    - [Simple debug console setup](#simple-debug-console-setup)
+      - [Enable agent debug console](#enable-agent-debug-console)
+      - [Start `kata-monitor` - ONLY NEEDED FOR 2.0.x](#start-kata-monitor---only-needed-for-20x)
+      - [Connect to debug console](#connect-to-debug-console)
+    - [Traditional debug console setup](#traditional-debug-console-setup)
+      - [Create a custom image containing a shell](#create-a-custom-image-containing-a-shell)
+      - [Build the debug image](#build-the-debug-image)
+      - [Configure runtime for custom debug image](#configure-runtime-for-custom-debug-image)
+      - [Create a container](#create-a-container)
+      - [Connect to the virtual machine using the debug console](#connect-to-the-virtual-machine-using-the-debug-console)
+        - [Enabling debug console for QEMU](#enabling-debug-console-for-qemu)
+        - [Enabling debug console for cloud-hypervisor / firecracker](#enabling-debug-console-for-cloud-hypervisor--firecracker)
+        - [Connecting to the debug console](#connecting-to-the-debug-console)
+  - [Obtain details of the image](#obtain-details-of-the-image)
+  - [Capturing kernel boot logs](#capturing-kernel-boot-logs)
 
 # Warning
 
@@ -304,7 +305,7 @@ You MUST choose one of `alpine`, `centos`, `clearlinux`, `debian`, `euleros`, `f
 > - You should only do this step if you are testing with the latest version of the agent.
 
 ```
-$ sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/bin ../../../src/agent/target/x86_64-unknown-linux-musl/release/kata-agent
+$ sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/usr/bin ../../../src/agent/target/x86_64-unknown-linux-musl/release/kata-agent
 $ sudo install -o root -g root -m 0440 ../../../src/agent/kata-agent.service ${ROOTFS_DIR}/usr/lib/systemd/system/
 $ sudo install -o root -g root -m 0440 ../../../src/agent/kata-containers.target ${ROOTFS_DIR}/usr/lib/systemd/system/
 ```
@@ -353,12 +354,13 @@ You MUST choose one of `alpine`, `centos`, `clearlinux`, `euleros`, and `fedora`
 >
 > - Check the [compatibility matrix](../tools/osbuilder/README.md#platform-distro-compatibility-matrix) before creating rootfs.
 
-Optionally, add your custom agent binary to the rootfs with the following, `LIBC` default is `musl`, if `ARCH` is `ppc64le`, should set the `LIBC=gnu` and `ARCH=powerpc64le`:
+Optionally, add your custom agent binary to the rootfs with the following commands. The default `$LIBC` used
+is `musl`, but on ppc64le and s390x, `gnu` should be used. Also, Rust refers to ppc64le as `powerpc64le`:
 ```
-$ export ARCH=$(shell uname -m)
-$ [ ${ARCH} == "ppc64le" ] && export LIBC=gnu || export LIBC=musl
+$ export ARCH=$(uname -m)
+$ [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
 $ [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
-$ sudo install -o root -g root -m 0550 -T ../../../src/agent/target/$(ARCH)-unknown-linux-$(LIBC)/release/kata-agent ${ROOTFS_DIR}/sbin/init
+$ sudo install -o root -g root -m 0550 -T ../../../src/agent/target/${ARCH}-unknown-linux-${LIBC}/release/kata-agent ${ROOTFS_DIR}/sbin/init
 ```
 
 ### Build an initrd image
@@ -384,31 +386,56 @@ You can build and install the guest kernel image as shown [here](../tools/packag
 
 # Install a hypervisor
 
-When setting up Kata using a [packaged installation method](install/README.md#installing-on-a-linux-system), the `qemu-lite` hypervisor is installed automatically. For other installation methods, you will need to manually install a suitable hypervisor.
+When setting up Kata using a [packaged installation method](install/README.md#installing-on-a-linux-system), the
+`QEMU` VMM is installed automatically. Cloud-Hypervisor and Firecracker VMMs are available from the [release tarballs](https://github.com/kata-containers/kata-containers/releases), as well as through [`kata-deploy`](../tools/packaging/kata-deploy/README.md).
+You may choose to manually build your VMM/hypervisor.
 
 ## Build a custom QEMU
 
-Your QEMU directory need to be prepared with source code. Alternatively, you can use the [Kata containers QEMU](https://github.com/kata-containers/qemu/tree/master) and checkout the recommended branch:
+Kata Containers makes use of upstream QEMU branch. The exact version
+and repository utilized can be found by looking at the [versions file](../versions.yaml).
 
+Find the correct version of QEMU from the versions file:
 ```
-$ go get -d github.com/kata-containers/qemu
-$ qemu_branch=$(grep qemu-lite- ${GOPATH}/src/github.com/kata-containers/kata-containers/versions.yaml | cut -d '"' -f2)
-$ cd ${GOPATH}/src/github.com/kata-containers/qemu
-$ git checkout -b $qemu_branch remotes/origin/$qemu_branch
-$ your_qemu_directory=${GOPATH}/src/github.com/kata-containers/qemu
+$ source ${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging/scripts/lib.sh
+$ qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version")
+$ echo ${qemu_version}
+```
+Get source from the matching branch of QEMU:
+```
+$ go get -d github.com/qemu/qemu
+$ cd ${GOPATH}/src/github.com/qemu/qemu
+$ git checkout ${qemu_version}
+$ your_qemu_directory=${GOPATH}/src/github.com/qemu/qemu
 ```
 
-To build a version of QEMU using the same options as the default `qemu-lite` version , you could use the `configure-hypervisor.sh` script:
-
+There are scripts to manage the build and packaging of QEMU. For the examples below, set your
+environment as:
+```
+$ go get -d github.com/kata-containers/kata-containers
+$ packaging_dir="${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging"
+```
+
+Kata often utilizes patches for not-yet-upstream and/or backported fixes for components,
+including QEMU. These can be found in the [packaging/QEMU directory](../tools/packaging/qemu/patches),
+and it's *recommended* that you apply them. For example, suppose that you are going to build QEMU
+version 5.2.0, do:
 ```
-$ go get -d github.com/kata-containers/kata-containers/tools/packaging
 $ cd $your_qemu_directory
-$ ${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging/scripts/configure-hypervisor.sh kata-qemu > kata.cfg
+$ $packaging_dir/scripts/apply_patches.sh $packaging_dir/qemu/patches/5.2.x/
+```
+
+To build utilizing the same options as Kata, you should make use of the `configure-hypervisor.sh` script. For example:
+```
+$ cd $your_qemu_directory
+$ $packaging_dir/scripts/configure-hypervisor.sh kata-qemu > kata.cfg
 $ eval ./configure "$(cat kata.cfg)"
 $ make -j $(nproc)
 $ sudo -E make install
 ```
 
+See the [static-build script for QEMU](../tools/packaging/static-build/qemu/build-static-qemu.sh) for a reference on how to get, setup, configure and build QEMU for Kata.
+
 ### Build a custom QEMU for aarch64/arm64 - REQUIRED
 > **Note:**
 >
@@ -476,6 +503,16 @@ debug_console_enabled = true
 
 This will pass `agent.debug_console agent.debug_console_vport=1026` to agent as kernel parameters, and sandboxes created using this parameters will start a shell in guest if new connection is accept from VSOCK.
 
+#### Start `kata-monitor` - ONLY NEEDED FOR 2.0.x
+
+For Kata Containers `2.0.x` releases, the `kata-runtime exec` command depends on the`kata-monitor` running, in order to get the sandbox's `vsock` address to connect to. Thus, first start the `kata-monitor` process.
+
+```
+$ sudo kata-monitor
+```
+
+`kata-monitor` will serve at `localhost:8090` by default.
+
 #### Connect to debug console
 
 Command `kata-runtime exec` is used to connect to the debug console.
@@ -613,11 +650,14 @@ sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.debug_cons
 > **Note** Ports 1024 and 1025 are reserved for communication with the agent
 > and gathering of agent logs respectively. 
 
-Next, connect to the debug console. The VSOCKS paths vary slightly between
-cloud-hypervisor and firecracker.
+##### Connecting to the debug console
+
+Next, connect to the debug console. The VSOCKS paths vary slightly between each
+VMM solution.
+
 In case of cloud-hypervisor, connect to the `vsock` as shown:
 ```
-$ sudo su -c 'cd /var/run/vc/vm/{sandbox_id}/root/ && socat stdin unix-connect:clh.sock'
+$ sudo su -c 'cd /var/run/vc/vm/${sandbox_id}/root/ && socat stdin unix-connect:clh.sock'
 CONNECT 1026
 ```
 
@@ -625,12 +665,18 @@ CONNECT 1026
 
 For firecracker, connect to the `hvsock` as shown:
 ```
-$ sudo su -c 'cd /var/run/vc/firecracker/{sandbox_id}/root/ && socat stdin unix-connect:kata.hvsock'
+$ sudo su -c 'cd /var/run/vc/firecracker/${sandbox_id}/root/ && socat stdin unix-connect:kata.hvsock'
 CONNECT 1026
 ```
 
 **Note**: You need to press the `RETURN` key to see the shell prompt.
 
+
+For QEMU, connect to the `vsock` as shown:
+```
+$ sudo su -c 'cd /var/run/vc/vm/${sandbox_id} && socat "stdin,raw,echo=0,escape=0x11" "unix-connect:console.sock"'
+```
+
 To disconnect from the virtual machine, type `CONTROL+q` (hold down the
 `CONTROL` key and press `q`).
 

docs/Documentation-Requirements.md

@@ -25,7 +25,7 @@ All documents must:
 - Have a `.md` file extension.
 - Include a TOC (table of contents) at the top of the document with links to
   all heading sections. We recommend using the
-  [`kata-check-markdown`](https://github.com/kata-containers/tests/tree/master/cmd/check-markdown)
+  [`kata-check-markdown`](https://github.com/kata-containers/tests/tree/main/cmd/check-markdown)
   tool to generate the TOC.
 - Be linked to from another document in the same repository.
 
@@ -118,7 +118,7 @@ This section lists requirements for displaying commands and command output.
 The requirements must be adhered to since documentation containing code blocks
 is validated by the CI system, which executes the command blocks with the help
 of the
-[doc-to-script](https://github.com/kata-containers/tests/tree/master/.ci/kata-doc-to-script.sh)
+[doc-to-script](https://github.com/kata-containers/tests/tree/main/.ci/kata-doc-to-script.sh)
 utility.
 
 - If a document includes commands the user should run, they **MUST** be shown
@@ -202,7 +202,7 @@ and compare them with standard tools (e.g. `diff(1)`).
 
 Since this project uses a number of terms not found in conventional
 dictionaries, we have a
-[spell checking tool](https://github.com/kata-containers/tests/tree/master/cmd/check-spelling)
+[spell checking tool](https://github.com/kata-containers/tests/tree/main/cmd/check-spelling)
 that checks both dictionary words and the additional terms we use.
 
 Run the spell checking tool on your document before raising a PR to ensure it

docs/Limitations.md

@@ -19,6 +19,8 @@
         * [Support for joining an existing VM network](#support-for-joining-an-existing-vm-network)
         * [docker --net=host](#docker---nethost)
         * [docker run --link](#docker-run---link)
+    * [Storage limitations](#storage-limitations)
+        * [Kubernetes `volumeMounts.subPaths`](#kubernetes-volumemountssubpaths)
     * [Host resource sharing](#host-resource-sharing)
         * [docker run --privileged](#docker-run---privileged)
 * [Miscellaneous](#miscellaneous)
@@ -26,7 +28,7 @@
 * [Appendices](#appendices)
     * [The constraints challenge](#the-constraints-challenge)
 
----
+***
 
 # Overview
 
@@ -92,7 +94,9 @@ This section lists items that might be possible to fix.
 ### checkpoint and restore
 
 The runtime does not provide `checkpoint` and `restore` commands. There
-are discussions about using VM save and restore to give [`criu`](https://github.com/checkpoint-restore/criu)-like functionality, which might provide a solution.
+are discussions about using VM save and restore to give us a
+`[criu](https://github.com/checkpoint-restore/criu)`-like functionality,
+which might provide a solution.
 
 Note that the OCI standard does not specify `checkpoint` and `restore`
 commands.
@@ -216,6 +220,17 @@ Equivalent functionality can be achieved with the newer docker networking comman
 See more documentation at
 [docs.docker.com](https://docs.docker.com/engine/userguide/networking/default_network/dockerlinks/).
 
+## Storage limitations
+
+### Kubernetes `volumeMounts.subPaths`
+
+Kubernetes `volumeMount.subPath` is not supported by Kata Containers at the
+moment.
+
+See [this issue](https://github.com/kata-containers/runtime/issues/2812) for more details.
+[Another issue](https://github.com/kata-containers/kata-containers/issues/1728) focuses on the case of `emptyDir`.
+
+
 ## Host resource sharing
 
 ### docker run --privileged
@@ -224,7 +239,7 @@ Privileged support in Kata is essentially different from `runc` containers.
 Kata does support `docker run --privileged` command, but in this case full access
 to the guest VM is provided in addition to some host access.
 
-The container runs with elevated capabilities within the guest and is granted 
+The container runs with elevated capabilities within the guest and is granted
 access to guest devices instead of the host devices.
 This is also true with using `securityContext privileged=true` with Kubernetes.
 

docs/Release-Process.md

@@ -18,8 +18,7 @@
 ## Requirements
 
 - [hub](https://github.com/github/hub)
-
-- OBS account with permissions on [`/home:katacontainers`](https://build.opensuse.org/project/subprojects/home:katacontainers)
+  * Using an [application token](https://github.com/settings/tokens) is required for hub.
 
 - GitHub permissions to push tags and create releases in Kata repositories.
 
@@ -30,16 +29,12 @@
 
 ## Release Process
 
+
 ### Bump all Kata repositories
 
-  - We have set up a Jenkins job to bump the version in the `VERSION` file in all Kata repositories. Go to the [Jenkins bump-job page](http://jenkins.katacontainers.io/job/release/build) to trigger a new job.
-  - Start a new job with variables for the job passed as:
-     - `BRANCH=<the-branch-you-want-to-bump>`
-     - `NEW_VERSION=<the-new-kata-version>`
-
-     For example, in the case where you want to make a patch release `1.10.2`, the variable `NEW_VERSION` should be `1.10.2` and `BRANCH` should point to  `stable-1.10`. In case of an alpha or release candidate release, `BRANCH` should point to `master` branch.
-
-  Alternatively, you can also bump the repositories using a script in the Kata packaging repo
+  Bump the repositories using a script in the Kata packaging repo, where:
+  - `BRANCH=<the-branch-you-want-to-bump>`
+  - `NEW_VERSION=<the-new-kata-version>`
   ```
   $ cd ${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging/release
   $ export NEW_VERSION=<the-new-kata-version>
@@ -47,6 +42,23 @@
   $ ./update-repository-version.sh -p "$NEW_VERSION" "$BRANCH"
   ```
 
+### Point tests repository to stable branch
+
+  If you create a new stable branch, i.e. if your release changes a major or minor version number (not a patch release), then
+  you should modify the `tests` repository to point to that newly created stable branch and not the `main` branch.
+  The objective is that changes in the CI on the main branch will not impact the stable branch.
+
+  In the test directory, change references the main branch in:
+  * `README.md`
+  * `versions.yaml`
+  * `cmd/github-labels/labels.yaml.in`
+  * `cmd/pmemctl/pmemctl.sh`
+  * `.ci/lib.sh`
+  * `.ci/static-checks.sh`
+
+  See the commits in [the corresponding PR for stable-2.1](https://github.com/kata-containers/tests/pull/3504) for an example of the changes.
+
+
 ### Merge all bump version Pull requests
 
   - The above step will create a GitHub pull request in the Kata projects. Trigger the CI using `/test` command on each bump Pull request.
@@ -56,7 +68,7 @@
 ### Tag all Kata repositories
 
   Once all the pull requests to bump versions in all Kata repositories are merged,
-  tag all the repositories as shown below.  
+  tag all the repositories as shown below.
   ```
   $ cd ${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging/release
   $ git checkout  <kata-branch-to-release>
@@ -66,7 +78,7 @@
 
 ### Check Git-hub Actions
 
-  We make use of [GitHub actions](https://github.com/features/actions) in this [file](https://github.com/kata-containers/kata-containers/blob/master/.github/workflows/main.yaml) in the `kata-containers/kata-containers` repository to build and upload release artifacts. This action is auto triggered with the above step when a new tag is pushed to the `kata-containers/kata-conatiners` repository.
+  We make use of [GitHub actions](https://github.com/features/actions) in this [file](https://github.com/kata-containers/kata-containers/blob/main/.github/workflows/main.yaml) in the `kata-containers/kata-containers` repository to build and upload release artifacts. This action is auto triggered with the above step when a new tag is pushed to the `kata-containers/kata-containers` repository.
 
   Check the [actions status page](https://github.com/kata-containers/kata-containers/actions) to verify all steps in the actions workflow have completed successfully. On success, a static tarball containing Kata release artifacts will be uploaded to the [Release page](https://github.com/kata-containers/kata-containers/releases).
 

docs/Stable-Branch-Strategy.md

@@ -32,16 +32,16 @@ provides additional information regarding release `99.123.77` in the previous ex
   changing the existing behavior*.
 
 - When `MAJOR` increases, the new release adds **new features, bug fixes, or
-  both** and which *changes the behavior from the previous release* (incompatible with previous releases).
+  both** and which **changes the behavior from the previous release** (incompatible with previous releases).
 
   A major release will also likely require a change of the container manager version used, 
-for example Docker\*. Please refer to the release notes for further details.
+for example Containerd or CRI-O. Please refer to the release notes for further details.
 
 ## Release Strategy
 
 Any new features added since the last release will be available in the next minor
 release. These will include bug fixes as well. To facilitate a stable user environment, 
-Kata provides stable branch-based releases and a master branch release.
+Kata provides stable branch-based releases and a main branch release.
 
 ## Stable branch patch criteria
 
@@ -49,9 +49,10 @@ No new features should be introduced to stable branches.  This is intended to li
 providing only bug and security fixes.
 
 ## Branch Management
-Kata Containers will maintain two stable release branches in addition to the master branch.
-Once a new MAJOR or MINOR release is created from master, a new stable branch is created for
-the prior MAJOR or MINOR release and the older stable branch is no longer maintained. End of
+Kata Containers will maintain **one** stable release branch, in addition to the main branch, for
+each active major release.
+Once a new MAJOR or MINOR release is created from main, a new stable branch is created for
+the prior MAJOR or MINOR release and the previous stable branch is no longer maintained. End of
 maintenance for a branch is announced on the Kata Containers mailing list.  Users can determine
 the version currently installed by running `kata-runtime kata-env`. It is recommended to use the
 latest stable branch available.
@@ -61,59 +62,59 @@ A couple of examples follow to help clarify this process.
 ### New bug fix introduced
 
 A bug fix is submitted against the runtime which does not introduce new inter-component dependencies.
-This fix is applied to both the master and stable branches, and there is no need to create a new
+This fix is applied to both the main and stable branches, and there is no need to create a new
 stable branch.
 
 | Branch | Original version | New version |
 |--|--|--|
-| `master` | `1.3.0-rc0` | `1.3.0-rc1` |
-| `stable-1.2` | `1.2.0` | `1.2.1` |
-| `stable-1.1` | `1.1.2` | `1.1.3` |
+| `main` | `2.3.0-rc0` | `2.3.0-rc1` |
+| `stable-2.2` | `2.2.0` | `2.2.1` |
+| `stable-2.1` | (unmaintained) | (unmaintained) |
 
 
 ### New release made feature or change adding new inter-component dependency
 
 A new feature is introduced, which adds a new inter-component dependency. In this case a new stable
-branch is created (stable-1.3) starting from master and the older stable branch (stable-1.1)
+branch is created (stable-2.3) starting from main and the previous stable branch (stable-2.2)
 is dropped from maintenance.
 
 
 | Branch | Original version | New version |
 |--|--|--|
-| `master` | `1.3.0-rc1` | `1.3.0` |
-| `stable-1.3` | N/A| `1.3.0` |
-| `stable-1.2` | `1.2.1` | `1.2.2` |
-| `stable-1.1` | `1.1.3` | (unmaintained) |
+| `main` | `2.3.0-rc1` | `2.3.0` |
+| `stable-2.3` | N/A| `2.3.0` |
+| `stable-2.2` | `2.2.1` | (unmaintained) |
+| `stable-2.1` | (unmaintained) | (unmaintained) |
 
-Note, the stable-1.1 branch will still exist with tag 1.1.3, but under current plans it is
-not maintained further. The next tag applied to master will be 1.4.0-alpha0. We would then
+Note, the stable-2.2 branch will still exist with tag 2.2.1, but under current plans it is
+not maintained further. The next tag applied to main will be 2.4.0-alpha0. We would then
 create a couple of alpha releases gathering features targeted for that particular release (in
-this case 1.4.0), followed by a release candidate. The release candidate marks a feature freeze.
+this case 2.4.0), followed by a release candidate. The release candidate marks a feature freeze.
 A new stable branch is created for the release candidate. Only bug fixes and any security issues
-are added to the branch going forward until release 1.4.0 is made.
+are added to the branch going forward until release 2.4.0 is made.
    
 ## Backporting Process 
 
-Development that occurs against the master branch and applicable code commits should also be submitted
+Development that occurs against the main branch and applicable code commits should also be submitted
 against the stable branches. Some guidelines for this process follow::
   1. Only bug and security fixes which do not introduce inter-component dependencies are
  candidates for stable branches. These PRs should be marked with "bug" in GitHub.
-  2. Once a PR is created against master which meets requirement of (1), a comparable one
+  2. Once a PR is created against main which meets requirement of (1), a comparable one
  should also be submitted against the stable branches. It is the responsibility of the submitter
  to apply their pull request against stable, and it is the responsibility of the
  reviewers to help identify stable-candidate pull requests.
  
 ## Continuous Integration Testing
 
-The test repository is forked to create stable branches from master. Full CI
-runs on each stable and master PR using its respective tests repository branch.
+The test repository is forked to create stable branches from main. Full CI
+runs on each stable and main PR using its respective tests repository branch.
 
 ### An alternative method for CI testing:
 
-Ideally, the continuous integration infrastructure will run the same test suite on both master
+Ideally, the continuous integration infrastructure will run the same test suite on both main
 and the stable branches.  When tests are modified or new feature tests are introduced, explicit
 logic should exist within the testing CI to make sure only applicable tests are executed against
-stable and master. While this is not in place currently, it should be considered in the long term.
+stable and main. While this is not in place currently, it should be considered in the long term.
 
 ## Release Management
 
@@ -121,7 +122,7 @@ stable and master. While this is not in place currently, it should be considered
 
 Releases are made every three weeks, which include a GitHub release as
 well as binary packages. These patch releases are made for both stable branches, and a "release candidate"
-for the next `MAJOR` or `MINOR` is created from master. If there are no changes across all the repositories, no
+for the next `MAJOR` or `MINOR` is created from main. If there are no changes across all the repositories, no
 release is created and an announcement is made on the developer mailing list to highlight this.
 If a release is being made, each repository is tagged for this release, regardless
 of whether changes are introduced. The release schedule can be seen on the
@@ -142,10 +143,10 @@ maturity, we have increased the cadence from six weeks to twelve weeks. The rele
 ### Compatibility
 Kata guarantees compatibility between components that are within one minor release of each other. 
  
-This is critical for dependencies which cross between host (runtime, shim, proxy) and
+This is critical for dependencies which cross between host (shimv2 runtime) and
 the guest (hypervisor, rootfs and agent).  For example, consider a cluster with a long-running
-deployment, workload-never-dies, all on Kata version 1.1.3 components. If the operator updates
-the Kata components to the next new minor release (i.e. 1.2.0), we need to guarantee that the 1.2.0
-runtime still communicates with 1.1.3 agent within workload-never-dies.
+deployment, workload-never-dies, all on Kata version 2.1.3 components. If the operator updates
+the Kata components to the next new minor release (i.e. 2.2.0), we need to guarantee that the 2.2.0
+shimv2 runtime still communicates with 2.1.3 agent within workload-never-dies.
 
 Handling live-update is out of the scope of this document. See this [`kata-runtime` issue](https://github.com/kata-containers/runtime/issues/492) for details.

docs/design/README.md

@@ -8,4 +8,5 @@ Kata Containers design documents:
 - [VSocks](VSocks.md)
 - [VCPU handling](vcpu-handling.md)
 - [Host cgroups](host-cgroups.md)
+- [`Inotify` support](inotify.md)
 - [Metrics(Kata 2.0)](kata-2-0-metrics.md)

docs/design/inotify.md

@@ -0,0 +1,30 @@
+# Kata Containers support for `inotify`
+
+## Background on `inotify` usage
+
+A common pattern in Kubernetes is to watch for changes to files/directories passed in as `ConfigMaps`
+or `Secrets`. Sidecar's normally use `inotify` to watch for changes and then signal the primary container to reload
+the updated configuration. Kata Containers typically will pass these host files into the guest using `virtiofs`, which
+does not support `inotify` today. While we work to enable this use case in `virtiofs`, we introduced a workaround in Kata Containers.
+This document describes how Kata Containers implements this workaround.
+
+### Detecting a `watchable` mount
+
+Kubernetes creates `secrets` and `ConfigMap` mounts at very specific locations on the host filesystem. For container mounts,
+the `Kata Containers` runtime will check the source of the mount to identify these special cases. For these use cases, only a single file
+or very few would typically need to be watched. To avoid excessive overheads in making a mount watchable,
+we enforce a limit of eight files per mount. If a `secret` or `ConfigMap` mount contains more than 8 files, it will not be
+considered watchable. We similarly enforce a limit of 1 MB per mount to be considered watchable. Non-watchable mounts will
+continue to propagate changes from the mount on the host to the container workload, but these updates will not trigger an
+`inotify` event.
+
+If at any point a mount grows beyond the eight file or 1MB limit, it will no longer be `watchable.`
+
+### Presenting a `watchable` mount to the workload
+
+For mounts that are considered `watchable`, inside the guest, the `kata-agent` will poll the mount presented from
+the host through `virtiofs` and copy any changed files to a `tmpfs` mount that is presented to the container. In this way,
+for `watchable` mounts, Kata will do the polling on behalf of the workload and existing workloads needn't change their usage
+of `inotify`.
+
+![drawing](arch-images/inotify-workaround.png)

docs/how-to/README.md

@@ -37,3 +37,4 @@
 - [How to use Kata Containers with `virtio-mem`](how-to-use-virtio-mem-with-kata.md)
 - [How to set sandbox Kata Containers configurations with pod annotations](how-to-set-sandbox-config-kata.md)
 - [How to monitor Kata Containers in K8s](how-to-set-prometheus-in-k8s.md)
+- [How to use hotplug memory on arm64 in Kata Containers](how-to-hotplug-memory-arm64.md)

docs/how-to/how-to-hotplug-memory-arm64.md

@@ -0,0 +1,32 @@
+# How to use memory hotplug feature in Kata Containers on arm64
+
+- [Introduction](#introduction)
+- [Install UEFI ROM](#install-uefi-rom)
+- [Run for test](#run-for-test)
+
+## Introduction
+
+Memory hotplug is a key feature for containers to allocate memory dynamically in deployment.
+As Kata Container bases on VM, this feature needs support both from VMM and guest kernel. Luckily, it has been fully supported for the current default version of QEMU and guest kernel used by Kata on arm64. For other VMMs, e.g, Cloud Hypervisor, the enablement work is on the road. Apart from VMM and guest kernel, memory hotplug also depends on ACPI which depends on firmware either. On x86, you can boot a VM using QEMU with ACPI enabled directly, because it boots up with firmware implicitly. For arm64, however, you need specify firmware explicitly. That is to say, if you are ready to run a normal Kata Container on arm64, what you need extra to do is to install the UEFI ROM before use the memory hotplug feature.
+
+## Install UEFI ROM
+
+We have offered a helper script for you to install the UEFI ROM. If you have installed Kata normally on your host, you just need to run the script as fellows:
+
+```bash
+$ pushd $GOPATH/src/github.com/kata-containers/tests
+$ sudo .ci/aarch64/install_rom_aarch64.sh
+$ popd
+```
+
+## Run for test
+
+Let's test if the memory hotplug is ready for Kata after install the UEFI ROM. Make sure containerd is ready to run Kata before test.
+
+```bash
+$ sudo ctr image pull docker.io/library/ubuntu:latest
+$ sudo ctr run --runtime io.containerd.run.kata.v2 -t --rm docker.io/library/ubuntu:latest hello sh -c "free -h"
+$ sudo ctr run --runtime io.containerd.run.kata.v2 -t --memory-limit 536870912 --rm docker.io/library/ubuntu:latest hello sh -c "free -h"
+```
+
+Compare the results between the two tests. If the latter is 0.5G larger than the former, you have done what you want, and congratulation!

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -26,6 +26,7 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.runtime.disable_new_netns` | `boolean` | determines if a new netns is created for the hypervisor process |
 | `io.katacontainers.config.runtime.internetworking_model` | string| determines how the VM should be connected to the container network interface. Valid values are `macvtap`, `tcfilter` and `none` |
 | `io.katacontainers.config.runtime.sandbox_cgroup_only`| `boolean` | determines if Kata processes are managed only in sandbox cgroup |
+| `io.katacontainers.config.runtime.enable_pprof` | `boolean` | enables Golang `pprof` for `containerd-shim-kata-v2` process |
 
 ## Agent Options
 | Key | Value Type | Comments |
@@ -60,7 +61,7 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.enable_swap` | `boolean` | enable swap of VM memory |
 | `io.katacontainers.config.hypervisor.enable_vhost_user_store` | `boolean` | enable vhost-user storage device (QEMU) |
 | `io.katacontainers.config.hypervisor.enable_virtio_mem` | `boolean` | enable virtio-mem (QEMU) |
-| `io.katacontainers.config.hypervisor.entropy_source` | string| the path to a host source of entropy (`/dev/random`, `/dev/urandom` or real hardware RNG device) |
+| `io.katacontainers.config.hypervisor.entropy_source` (R) | string| the path to a host source of entropy (`/dev/random`, `/dev/urandom` or real hardware RNG device) |
 | `io.katacontainers.config.hypervisor.file_mem_backend` (R) | string | file based memory backend root directory |
 | `io.katacontainers.config.hypervisor.firmware_hash` | string | container firmware SHA-512 hash value |
 | `io.katacontainers.config.hypervisor.firmware` | string | the guest firmware that will run the container VM |
@@ -78,7 +79,7 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.kernel` | string | the kernel used to boot the container VM |
 | `io.katacontainers.config.hypervisor.machine_accelerators` | string | machine specific accelerators for the hypervisor |
 | `io.katacontainers.config.hypervisor.machine_type` | string | the type of machine being emulated by the hypervisor |
-| `io.katacontainers.config.hypervisor.memory_offset` | uint32| the memory space used for `nvdimm` device by the hypervisor |
+| `io.katacontainers.config.hypervisor.memory_offset` | uint64| the memory space used for `nvdimm` device by the hypervisor |
 | `io.katacontainers.config.hypervisor.memory_slots` | uint32| the memory slots assigned to the VM by the hypervisor |
 | `io.katacontainers.config.hypervisor.msize_9p` | uint32 | the `msize` for 9p shares |
 | `io.katacontainers.config.hypervisor.path` | string | the hypervisor that will run the container VM |
@@ -95,6 +96,8 @@ There are several kinds of Kata configurations and they are listed below.
 
 In case of CRI-O, all annotations specified in the pod spec are passed down to Kata.
 
+# containerd Configuration
+
 For containerd, annotations specified in the pod spec are passed down to Kata
 starting with version `1.3.0` of containerd. Additionally, extra configuration is
 needed for containerd, by providing a `pod_annotations` field in the containerd config
@@ -107,11 +110,9 @@ for passing annotations to Kata from containerd:
 $ cat /etc/containerd/config
 ....
 
-[plugins.cri.containerd.runtimes.kata]
-           runtime_type = "io.containerd.runc.v1"
+         [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
+           runtime_type = "io.containerd.kata.v2"
            pod_annotations = ["io.katacontainers.*"]
-           [plugins.cri.containerd.runtimes.kata.options]
-             BinaryName = "/usr/bin/kata-runtime"
 ....
 
 ```
@@ -197,6 +198,7 @@ the configuration entry:
 | Key | Config file entry | Comments |
 |-------| ----- | ----- |
 | `ctlpath`  | `valid_ctlpaths` | Valid paths for `acrnctl` binary |
+| `entropy_source` | `valid_entropy_sources` | Valid entropy sources, e.g. `/dev/random` |
 | `file_mem_backend`  | `valid_file_mem_backends` | Valid locations for the file-based memory backend root directory |
 | `jailer_path`  | `valid_jailer_paths`| Valid paths for the jailer constraining the container VM (Firecracker) |
 | `path`  | `valid_hypervisor_paths` | Valid hypervisors to run the container VM |

docs/how-to/how-to-use-k8s-with-cri-containerd-and-kata.md

@@ -7,9 +7,10 @@
     * [Configure Kubelet to use containerd](#configure-kubelet-to-use-containerd)
     * [Configure HTTP proxy - OPTIONAL](#configure-http-proxy---optional)
 * [Start Kubernetes](#start-kubernetes)
-* [Install a Pod Network](#install-a-pod-network)
+* [Configure Pod Network](#configure-pod-network)
 * [Allow pods to run in the master node](#allow-pods-to-run-in-the-master-node)
-* [Create an untrusted pod using Kata Containers](#create-an-untrusted-pod-using-kata-containers)
+* [Create runtime class for Kata Containers](#create-runtime-class-for-kata-containers)
+* [Run pod in Kata Containers](#run-pod-in-kata-containers)
 * [Delete created pod](#delete-created-pod)
 
 This document describes how to set up a single-machine Kubernetes (k8s) cluster.
@@ -18,9 +19,6 @@ The Kubernetes cluster will use the
 [CRI containerd plugin](https://github.com/containerd/cri) and
 [Kata Containers](https://katacontainers.io) to launch untrusted workloads.
 
-For Kata Containers 1.5.0-rc2 and above, we will use `containerd-shim-kata-v2` (short as `shimv2` in this documentation)
-to launch Kata Containers. For the previous version of Kata Containers, the Pods are launched with `kata-runtime`.
-
 ## Requirements
 
 - Kubernetes, Kubelet, `kubeadm`
@@ -125,43 +123,33 @@ $ sudo systemctl daemon-reload
   $ sudo -E kubectl get pods
   ```
 
-## Install a Pod Network
+## Configure Pod Network
 
 A pod network plugin is needed to allow pods to communicate with each other.
+You can find more about CNI plugins from the [Creating a cluster with `kubeadm`](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#instructions) guide.
 
-- Install the `flannel` plugin by following the
-  [Using `kubeadm` to Create a Cluster](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#instructions)
-  guide, starting from the **Installing a pod network** section.
-
-- Create a pod network using flannel
-
-  > **Note:** There is no known way to determine programmatically the best version (commit) to use.
-  > See https://github.com/coreos/flannel/issues/995.
+By default the CNI plugin binaries is installed under `/opt/cni/bin` (in package `kubernetes-cni`), you only need to create a configuration file for CNI plugin.
 
   ```bash
-  $ sudo -E kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
-  ```
+  $ sudo -E mkdir -p /etc/cni/net.d
 
-- Wait for the pod network to become available
-
-  ```bash
-  # number of seconds to wait for pod network to become available
-  $ timeout_dns=420
-
-  $ while [ "$timeout_dns" -gt 0 ]; do
-      if sudo -E kubectl get pods --all-namespaces | grep dns | grep Running; then
-          break
-      fi
-
-      sleep 1s
-      ((timeout_dns--))
-   done
-  ```
-
-- Check the pod network is running
-
-  ```bash
-  $ sudo -E kubectl get pods --all-namespaces | grep dns | grep Running && echo "OK" || ( echo "FAIL" && false )
+  $ sudo -E cat > /etc/cni/net.d/10-mynet.conf <<EOF
+  {
+    "cniVersion": "0.2.0",
+    "name": "mynet",
+    "type": "bridge",
+    "bridge": "cni0",
+    "isGateway": true,
+    "ipMasq": true,
+    "ipam": {
+      "type": "host-local",
+      "subnet": "172.19.0.0/24",
+      "routes": [
+        { "dst": "0.0.0.0/0" }
+      ]
+    }
+  }
+  EOF
   ```
 
 ## Allow pods to run in the master node
@@ -172,24 +160,38 @@ By default, the cluster will not schedule pods in the master node. To enable mas
 $ sudo -E kubectl taint nodes --all node-role.kubernetes.io/master-
 ```
 
-## Create an untrusted pod using Kata Containers
+## Create runtime class for Kata Containers
 
 By default, all pods are created with the default runtime configured in CRI containerd plugin.
+From Kubernetes v1.12, users can use [`RuntimeClass`](https://kubernetes.io/docs/concepts/containers/runtime-class/#runtime-class) to specify a different runtime for Pods.
 
-If a pod has the `io.kubernetes.cri.untrusted-workload` annotation set to `"true"`, the CRI plugin runs the pod with the
+```bash
+$ cat > runtime.yaml <<EOF
+apiVersion: node.k8s.io/v1beta1
+kind: RuntimeClass
+metadata:
+  name: kata
+handler: kata
+EOF
+
+$ sudo -E kubectl apply -f runtime.yaml
+```
+
+## Run pod in Kata Containers
+
+If a pod has the `runtimeClassName` set to `kata`, the CRI plugin runs the pod with the
 [Kata Containers runtime](../../src/runtime/README.md).
 
-- Create an untrusted pod configuration
+- Create an pod configuration that using Kata Containers runtime
 
   ```bash
-  $ cat << EOT | tee nginx-untrusted.yaml
+  $ cat << EOT | tee nginx-kata.yaml
   apiVersion: v1
   kind: Pod
   metadata:
-    name: nginx-untrusted
-    annotations:
-      io.kubernetes.cri.untrusted-workload: "true"
+    name: nginx-kata
   spec:
+    runtimeClassName: kata
     containers:
     - name: nginx
       image: nginx
@@ -197,9 +199,9 @@ If a pod has the `io.kubernetes.cri.untrusted-workload` annotation set to `"true
   EOT
   ```
 
-- Create an untrusted pod
+- Create the pod
   ```bash
-  $ sudo -E kubectl apply -f nginx-untrusted.yaml
+  $ sudo -E kubectl apply -f nginx-kata.yaml
   ```
 
 - Check pod is running
@@ -216,5 +218,5 @@ If a pod has the `io.kubernetes.cri.untrusted-workload` annotation set to `"true
 ## Delete created pod
 
 ```bash
-$ sudo -E kubectl delete -f nginx-untrusted.yaml
+$ sudo -E kubectl delete -f nginx-kata.yaml
 ```

docs/how-to/how-to-use-sysctls-with-kata.md

@@ -16,11 +16,10 @@ To get a complete list of kernel parameters, run:
 $ sudo sysctl -a
 ```
 
-Both Docker and Kubernetes provide mechanisms for setting namespaced sysctls. 
-Namespaced sysctls can be set per pod in the case of Kubernetes or per container
-in case of Docker.
+Kubernetes provide mechanisms for setting namespaced sysctls. 
+Namespaced sysctls can be set per pod in the case of Kubernetes.
 The following sysctls are known to be namespaced and can be set with 
-Docker and Kubernetes:
+Kubernetes:
 
 - `kernel.shm*`
 - `kernel.msg*`
@@ -30,31 +29,10 @@ Docker and Kubernetes:
 
 ### Namespaced Sysctls:
 
-Kata Containers supports setting namespaced sysctls with Docker and Kubernetes.
+Kata Containers supports setting namespaced sysctls with Kubernetes.
 All namespaced sysctls can be set in the same way as regular Linux based
 containers, the difference being, in the case of Kata they are set inside the guest.
 
-#### Setting Namespaced Sysctls with Docker:
-
-```
-$ sudo docker run --runtime=kata-runtime -it alpine cat /proc/sys/fs/mqueue/queues_max
-256
-$ sudo docker run --runtime=kata-runtime --sysctl fs.mqueue.queues_max=512 -it alpine cat /proc/sys/fs/mqueue/queues_max
-512
-```
-
-... and:
-
-```
-$ sudo docker run --runtime=kata-runtime -it alpine cat /proc/sys/kernel/shmmax
-18446744073692774399
-$ sudo docker run --runtime=kata-runtime --sysctl kernel.shmmax=1024 -it alpine cat /proc/sys/kernel/shmmax
-1024
-```
-
-For additional documentation on setting sysctls with Docker please refer to [Docker-sysctl-doc](https://docs.docker.com/engine/reference/commandline/run/#configure-namespaced-kernel-parameters-sysctls-at-runtime).
-
-
 #### Setting Namespaced Sysctls with Kubernetes:
 
 Kubernetes considers certain sysctls as safe and others as unsafe. For detailed
@@ -100,7 +78,7 @@ spec:
 
 ### Non-Namespaced Sysctls:
 
-Docker and Kubernetes disallow sysctls without a namespace.
+Kubernetes disallow sysctls without a namespace.
 The recommendation is to set them directly on the host or use a privileged
 container in the case of Kubernetes.
 

docs/how-to/how-to-use-virtio-fs-with-kata.md

@@ -9,4 +9,4 @@ Container deployments utilize explicit or implicit file sharing between host fil
 
 As of the 2.0 release of Kata Containers, [virtio-fs](https://virtio-fs.gitlab.io/) is the default filesystem sharing mechanism.
 
-virtio-fs support works out of the box for `cloud-hypervisor` and `qemu`, when Kata Containers is deployed using `kata-deploy`. Learn more about `kata-deploy` and how to use `kata-deploy` in Kubernetes [here](https://github.com/kata-containers/packaging/tree/master/kata-deploy#kubernetes-quick-start).
+virtio-fs support works out of the box for `cloud-hypervisor` and `qemu`, when Kata Containers is deployed using `kata-deploy`. Learn more about `kata-deploy` and how to use `kata-deploy` in Kubernetes [here](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy#kubernetes-quick-start).

docs/how-to/how-to-use-virtio-mem-with-kata.md

@@ -13,26 +13,23 @@ Kata Containers with `virtio-mem` supports memory resize.
 
 ## Requisites
 
-Kata Containers with `virtio-mem` requires Linux and the QEMU that support `virtio-mem`.
-The Linux kernel and QEMU upstream version still not support `virtio-mem`.  @davidhildenbrand is working on them.
-Please use following unofficial version of the Linux kernel and QEMU that support `virtio-mem` with Kata Containers.
+Kata Containers just supports `virtio-mem` with QEMU.
+Install and setup Kata Containers as shown [here](../install/README.md).
 
-The Linux kernel is at https://github.com/davidhildenbrand/linux/tree/virtio-mem-rfc-v4.
-The Linux kernel config that can work with Kata Containers is at https://gist.github.com/teawater/016194ee84748c768745a163d08b0fb9.
-
-The QEMU is at https://github.com/teawater/qemu/tree/kata-virtio-mem. (The original source is at https://github.com/davidhildenbrand/qemu/tree/virtio-mem.  Its base version of QEMU cannot work with Kata Containers.  So merge the commit of `virtio-mem` to upstream QEMU.)
-
-Set Linux and the QEMU that support `virtio-mem` with following line in the Kata Containers QEMU configuration `configuration-qemu.toml`:
-```toml
-[hypervisor.qemu]
-path = "qemu-dir"
-kernel = "vmlinux-dir"
+### With x86_64
+The `virtio-mem` config of the x86_64 Kata Linux kernel is open.
+Enable `virtio-mem` as follows:
+```
+$ sudo sed -i -e 's/^#enable_virtio_mem.*$/enable_virtio_mem = true/g' /etc/kata-containers/configuration.toml
 ```
 
-Enable `virtio-mem` with following line in the Kata Containers configuration:
-```toml
-enable_virtio_mem = true
+### With other architectures
+The `virtio-mem` config of the others Kata Linux kernel is not open.
+You can open `virtio-mem` config as follows:
 ```
+CONFIG_VIRTIO_MEM=y
+```
+Then you can build and install the guest kernel image as shown [here](../../tools/packaging/kernel/README.md#build-kata-containers-kernel).
 
 ## Run a Kata Container utilizing `virtio-mem`
 
@@ -41,13 +38,35 @@ Use following command to enable memory overcommitment of a Linux kernel.  Becaus
 $ echo 1 | sudo tee /proc/sys/vm/overcommit_memory
 ```
 
-Use following command start a Kata Container.
+Use following command to start a Kata Container.
 ```
-$ docker run --rm -it --runtime=kata --name test busybox
+$ pod_yaml=pod.yaml
+$ container_yaml=${REPORT_DIR}/container.yaml
+$ image="quay.io/prometheus/busybox:latest"
+$ cat << EOF > "${pod_yaml}"
+metadata:
+  name: busybox-sandbox1
+EOF
+$ cat << EOF > "${container_yaml}"
+metadata:
+  name: busybox-killed-vmm
+image:
+  image: "$image"
+command:
+- top
+EOF
+$ sudo crictl pull $image
+$ podid=$(sudo crictl runp $pod_yaml)
+$ cid=$(sudo crictl create $podid $container_yaml $pod_yaml)
+$ sudo crictl start $cid
 ```
 
-Use following command set the memory size of test to default_memory + 512m.
+Use the following command to set the container memory limit to 2g and the memory size of the VM to its default_memory + 2g.
 ```
-$ docker update -m 512m --memory-swap -1 test
+$ sudo crictl update --memory $((2*1024*1024*1024)) $cid
 ```
 
+Use the following command to set the container memory limit to 1g and the memory size of the VM to its default_memory + 1g.
+```
+$ sudo crictl update --memory $((1*1024*1024*1024)) $cid
+```

docs/how-to/run-kata-with-k8s.md

@@ -171,10 +171,10 @@ $ sudo systemctl daemon-reload
 $ sudo systemctl restart kubelet
 
 # If using CRI-O
-$ sudo kubeadm init --skip-preflight-checks --cri-socket /var/run/crio/crio.sock --pod-network-cidr=10.244.0.0/16
+$ sudo kubeadm init --ignore-preflight-errors=all --cri-socket /var/run/crio/crio.sock --pod-network-cidr=10.244.0.0/16
 
 # If using CRI-containerd
-$ sudo kubeadm init --skip-preflight-checks --cri-socket /run/containerd/containerd.sock --pod-network-cidr=10.244.0.0/16
+$ sudo kubeadm init --ignore-preflight-errors=all --cri-socket /run/containerd/containerd.sock --pod-network-cidr=10.244.0.0/16
 
 $ export KUBECONFIG=/etc/kubernetes/admin.conf
 ```

docs/install/README.md

@@ -50,9 +50,7 @@ Kata packages are provided by official distribution repositories for:
 | Distribution (link to installation guide)                | Minimum versions                                                               |
 |----------------------------------------------------------|--------------------------------------------------------------------------------|
 | [CentOS](centos-installation-guide.md)                   | 8                                                                              |
-| [Fedora](fedora-installation-guide.md)                   | 32, Rawhide                                                                    |
-| [openSUSE](opensuse-installation-guide.md)               | [Leap 15.1](opensuse-leap-15.1-installation-guide.md)<br>Leap 15.2, Tumbleweed |
-| [SUSE Linux Enterprise (SLE)](sle-installation-guide.md) | SLE 15 SP1, 15 SP2                                                             |
+| [Fedora](fedora-installation-guide.md)                   | 34                                                                             |
 
 > **Note::**
 >

docs/install/centos-installation-guide.md

@@ -3,15 +3,9 @@
 1. Install the Kata Containers components with the following commands:
 
    ```bash
+   $ sudo -E dnf install -y centos-release-advanced-virtualization
+   $ sudo -E dnf module disable -y virt:rhel
    $ source /etc/os-release
-   $ cat <<EOF | sudo -E tee /etc/yum.repos.d/advanced-virt.repo
-     [advanced-virt]
-     name=Advanced Virtualization
-     baseurl=http://mirror.centos.org/\$contentdir/\$releasever/virt/\$basearch/advanced-virtualization
-     enabled=1
-     gpgcheck=1
-     skip_if_unavailable=1
-     EOF
    $ cat <<EOF | sudo -E tee /etc/yum.repos.d/kata-containers.repo
      [kata-containers]
      name=Kata Containers
@@ -20,8 +14,7 @@
      gpgcheck=1
      skip_if_unavailable=1
      EOF
-   $ sudo -E dnf module disable -y virt:rhel
-   $ sudo -E dnf install -y kata-runtime
+   $ sudo -E dnf install -y kata-containers
    ```
 
 2. Decide which container manager to use and select the corresponding link that follows:

docs/install/fedora-installation-guide.md

@@ -3,7 +3,7 @@
 1. Install the Kata Containers components with the following commands:
 
    ```bash
-   $ sudo -E dnf -y install kata-runtime
+   $ sudo -E dnf -y install kata-containers
    ```
 
 2. Decide which container manager to use and select the corresponding link that follows:

docs/install/minikube-installation-guide.md

@@ -18,7 +18,7 @@
 cluster locally. It creates a single node Kubernetes stack in a local VM.
 
 [Kata Containers](https://github.com/kata-containers) can be installed into a Minikube cluster using
-[`kata-deploy`](https://github.com/kata-containers/packaging/tree/master/kata-deploy).
+[`kata-deploy`](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy).
 
 This document details the pre-requisites, installation steps, and how to check
 the installation has been successful.
@@ -135,7 +135,7 @@ $ kubectl apply -f kata-deploy/base/kata-deploy.yaml
 This installs the Kata Containers components into `/opt/kata` inside the Minikube node. It can take
 a few minutes for the operation to complete. You can check the installation has worked by checking
 the status of the `kata-deploy` pod, which will be executing
-[this script](https://github.com/kata-containers/packaging/blob/master/kata-deploy/scripts/kata-deploy.sh),
+[this script](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy/scripts/kata-deploy.sh),
 and will be executing a `sleep infinity` once it has successfully completed its work.
 You can accomplish this by running the following:
 

docs/install/opensuse-installation-guide.md

@@ -1,10 +0,0 @@
-# Install Kata Containers on openSUSE
-
-1. Install the Kata Containers components with the following commands:
-
-   ```bash
-   $ sudo -E zypper -n install katacontainers
-   ```
-
-2. Decide which container manager to use and select the corresponding link that follows:
-   - [Kubernetes](../Developer-Guide.md#run-kata-containers-with-kubernetes)

docs/install/opensuse-leap-15.1-installation-guide.md

@@ -1,11 +0,0 @@
-# Install Kata Containers on openSUSE Leap 15.1
-
-1. Install the Kata Containers components with the following commands:
-
-   ```bash
-   $ sudo -E zypper addrepo --refresh "https://download.opensuse.org/repositories/devel:/kubic/openSUSE_Leap_15.1/devel:kubic.repo"
-   $ sudo -E zypper -n --gpg-auto-import-keys install katacontainers
-   ```
-
-2. Decide which container manager to use and select the corresponding link that follows:
-   - [Kubernetes](../Developer-Guide.md#run-kata-containers-with-kubernetes)

docs/install/sle-installation-guide.md

@@ -1,13 +0,0 @@
-# Install Kata Containers on SLE
-
-1. Install the Kata Containers components with the following commands:
-
-   ```bash
-   $ source /etc/os-release
-   $ DISTRO_VERSION=$(sed "s/-/_/g" <<< "$VERSION")
-   $ sudo -E zypper addrepo --refresh "https://download.opensuse.org/repositories/devel:/kubic/SLE_${DISTRO_VERSION}_Backports/devel:kubic.repo"
-   $ sudo -E zypper -n --gpg-auto-import-keys install katacontainers
-   ```
-
-2. Decide which container manager to use and select the corresponding link that follows:
-   - [Kubernetes](../Developer-Guide.md#run-kata-containers-with-kubernetes)

docs/install/snap-installation-guide.md

@@ -2,9 +2,6 @@
 
 * [Install Kata Containers](#install-kata-containers)
 * [Configure Kata Containers](#configure-kata-containers)
-* [Integration with non-compatible shim v2 Container Engines](#integration-with-non-compatible-shim-v2-container-engines)
-    * [Integration with Docker](#integration-with-docker)
-    * [Integration with Podman](#integration-with-podman)
 * [Integration with shim v2 Container Engines](#integration-with-shim-v2-container-engines)
 * [Remove Kata Containers snap package](#remove-kata-containers-snap-package)
 
@@ -14,23 +11,10 @@
 Kata Containers can be installed in any Linux distribution that supports
 [snapd](https://docs.snapcraft.io/installing-snapd).
 
-> NOTE: From Kata Containers 2.x, only the [Containerd Runtime V2 (Shim API)](https://github.com/containerd/containerd/tree/master/runtime/v2)
-> is supported, note that some container engines (`docker`, `podman`, etc) may not
-> be able to run Kata Containers 2.x.
-
-Kata Containers 1.x is released through the *stable* channel while Kata Containers
-2.x is available in the *candidate* channel.
-
-Run the following command to install **Kata Containers 1.x**:
+Run the following command to install **Kata Containers**:
 
 ```sh
-$ sudo snap install kata-containers --classic
-```
-
-Run the following command to install **Kata Containers 2.x**:
-
-```sh
-$ sudo snap install kata-containers --candidate --classic
+$ sudo snap install kata-containers --stable --classic
 ```
 
 ## Configure Kata Containers
@@ -46,55 +30,6 @@ $ sudo cp /snap/kata-containers/current/usr/share/defaults/kata-containers/confi
 $ $EDITOR /etc/kata-containers/configuration.toml
 ```
 
-## Integration with non-compatible shim v2 Container Engines
-
-At the time of writing this document, `docker` and `podman` **do not support Kata
-Containers 2.x, therefore Kata Containers 1.x must be used instead.**
-
-The path to the runtime provided by the Kata Containers 1.x snap package is
-`/snap/bin/kata-containers.runtime`, it should be used to run Kata Containers 1.x.
-
-### Integration with Docker
-
-`/etc/docker/daemon.json` is the configuration file for `docker`, use the
-following configuration to add a new runtime (`kata`) to `docker`.
-
-```json
-{
-  "runtimes": {
-    "kata": {
-      "path": "/snap/bin/kata-containers.runtime"
-    }
-  }
-}
-```
-
-Once the above configuration has been applied, use the
-following commands to restart `docker` and run Kata Containers 1.x.
-
-```sh
-$ sudo systemctl restart docker
-$ docker run -ti --runtime kata busybox sh
-```
-
-### Integration with Podman
-
-`/usr/share/containers/containers.conf` is the configuration file for `podman`,
-add the following configuration in the `[engine.runtimes]` section.
-
-```toml
-kata = [
-   "/snap/bin/kata-containers.runtime"
-]
-```
-
-Once the above configuration has been applied, use the following command to run
-Kata Containers 1.x with `podman`
-
-```sh
-$ sudo podman run -ti --runtime kata docker.io/library/busybox sh
-```
-
 ## Integration with shim v2 Container Engines
 
 The Container engine daemon (`cri-o`, `containerd`, etc) needs to be able to find the

docs/install/ubuntu-installation-guide.md

@@ -1,15 +0,0 @@
-# Install Kata Containers on Ubuntu
-
-1. Install the Kata Containers components with the following commands:
-
-   ```bash
-   $ ARCH=$(arch)
-   $ BRANCH="${BRANCH:-master}"
-   $ sudo sh -c "echo 'deb http://download.opensuse.org/repositories/home:/katacontainers:/releases:/${ARCH}:/${BRANCH}/xUbuntu_$(lsb_release -rs)/ /' > /etc/apt/sources.list.d/kata-containers.list"
-   $ curl -sL  http://download.opensuse.org/repositories/home:/katacontainers:/releases:/${ARCH}:/${BRANCH}/xUbuntu_$(lsb_release -rs)/Release.key | sudo apt-key add -
-   $ sudo -E apt-get update
-   $ sudo -E apt-get -y install kata-runtime kata-proxy kata-shim
-   ```
-
-2. Decide which container manager to use and select the corresponding link that follows:
-   - [Kubernetes](../Developer-Guide.md#run-kata-containers-with-kubernetes)

docs/use-cases/Intel-GPU-passthrough-and-Kata.md

@@ -65,8 +65,8 @@ configuration in the Kata `configuration.toml` file as shown below.
 $ sudo sed -i -e 's/^# *\(hotplug_vfio_on_root_bus\).*=.*$/\1 = true/g' /usr/share/defaults/kata-containers/configuration.toml
 ```
 
-Make sure you are using the `pc` machine type by verifying `machine_type = "pc"` is
-set in the `configuration.toml`.
+Make sure you are using the `q35` machine type by verifying `machine_type = "q35"` is
+set in the `configuration.toml`. Make sure `pcie_root_port` is set to a positive value.
 
 ## Build Kata Containers kernel with GPU support
 

docs/use-cases/Nvidia-GPU-passthrough-and-Kata.md

@@ -75,13 +75,6 @@ To use non-large BARs devices (for example, Nvidia Tesla T4), you need Kata vers
 Follow the [Kata Containers setup instructions](../install/README.md)
 to install the latest version of Kata.
 
-The following configuration in the Kata `configuration.toml` file as shown below can work:
-```
-machine_type = "pc"
-
-hotplug_vfio_on_root_bus = true
-```
-
 To use large BARs devices (for example, Nvidia Tesla P100), you need Kata version 1.11.0 or above.
 
 The following configuration in the Kata `configuration.toml` file as shown below can work:

docs/use-cases/using-Intel-QAT-and-kata.md

@@ -74,7 +74,7 @@ Make sure to check [`01.org`](https://01.org/intel-quickassist-technology) for
 the latest driver.
 
 ```bash
-$ export QAT_DRIVER_VER=qat1.7.l.4.12.0-00011.tar.gz
+$ export QAT_DRIVER_VER=qat1.7.l.4.14.0-00031.tar.gz
 $ export QAT_DRIVER_URL=https://downloadmirror.intel.com/30178/eng/${QAT_DRIVER_VER}
 $ export QAT_CONF_LOCATION=~/QAT_conf
 $ export QAT_DOCKERFILE=https://raw.githubusercontent.com/intel/intel-device-plugins-for-kubernetes/master/demo/openssl-qat-engine/Dockerfile
@@ -402,7 +402,7 @@ different hypervisor, different install method for Kata, or a different
 Intel® QAT chipset then the command will need to be modified. 
 
 > **Note: The following was tested with 
-[containerd v1.3.9](https://github.com/containerd/containerd/releases/tag/v1.3.9).**
+[containerd v1.4.6](https://github.com/containerd/containerd/releases/tag/v1.4.6).**
 
 ```bash
 $ config_file="/opt/kata/share/defaults/kata-containers/configuration-qemu.toml"

docs/use-cases/zun_kata.md

@@ -10,9 +10,6 @@ Currently, the instructions are based on the following links:
 
 - https://docs.openstack.org/zun/latest/admin/clear-containers.html
 
-- ../install/ubuntu-installation-guide.md
-
-
 ## Install Git to use with DevStack
 
 ```sh
@@ -54,7 +51,7 @@ $ zun delete test
 
 ## Install Kata Containers
 
-Follow [these instructions](../install/ubuntu-installation-guide.md)
+Follow [these instructions](../install/README.md)
 to install the Kata Containers components.
 
 ## Update Docker with new Kata Containers runtime

snap/snapcraft.yaml

@@ -80,6 +80,8 @@ parts:
       - uidmap
       - gnupg2
     override-build: |
+      [ "$(uname -m)" = "ppc64le" ] || [ "$(uname -m)" = "s390x" ] && sudo apt-get --no-install-recommends install -y protobuf-compiler
+
       yq=${SNAPCRAFT_STAGE}/yq
 
       # set GOPATH
@@ -88,6 +90,7 @@ parts:
 
       export GOROOT=${SNAPCRAFT_STAGE}
       export PATH="${GOROOT}/bin:${PATH}"
+      export GO111MODULE="auto"
 
       http_proxy=${http_proxy:-""}
       https_proxy=${https_proxy:-""}
@@ -117,9 +120,13 @@ parts:
       export USE_DOCKER=1
       export DEBUG=1
       case "$(uname -m)" in
-        aarch64|ppc64le|s390x)
+        aarch64)
           sudo -E PATH=$PATH make initrd DISTRO=alpine
         ;;
+        ppc64le|s390x)
+          # Cannot use alpine on ppc64le/s390x because it would require a musl agent
+          sudo -E PATH=$PATH make initrd DISTRO=ubuntu
+        ;;
         x86_64)
           # In some build systems it's impossible to build a rootfs image, try with the initrd image
           sudo -E PATH=$PATH make image DISTRO=clearlinux || sudo -E PATH=$PATH make initrd DISTRO=alpine
@@ -141,6 +148,7 @@ parts:
       export GOPATH=${SNAPCRAFT_STAGE}/gopath
       export GOROOT=${SNAPCRAFT_STAGE}
       export PATH="${GOROOT}/bin:${PATH}"
+      export GO111MODULE="auto"
       kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
 
       cd ${kata_dir}/src/runtime
@@ -180,7 +188,10 @@ parts:
       - bison
       - flex
     override-build: |
+      [ "$(uname -m)" = "s390x" ] && sudo apt-get --no-install-recommends install -y libssl-dev
+
       export GOPATH=${SNAPCRAFT_STAGE}/gopath
+      export GO111MODULE="auto"
       kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
 
       cd ${kata_dir}/tools/packaging/kernel
@@ -202,8 +213,10 @@ parts:
       ln -sf ${vmlinuz_name} ${kata_kernel_dir}/vmlinuz.container
 
       # Install raw kernel
+      vmlinux_path=vmlinux
+      [ "$(uname -m)" = "s390x" ] && vmlinux_path=arch/s390/boot/compressed/vmlinux
       vmlinux_name=vmlinux-${kernel_suffix}
-      cp vmlinux ${kata_kernel_dir}/${vmlinux_name}
+      cp ${vmlinux_path} ${kata_kernel_dir}/${vmlinux_name}
       ln -sf ${vmlinux_name} ${kata_kernel_dir}/vmlinux.container
 
   qemu:
@@ -227,11 +240,13 @@ parts:
       - libblkid-dev
       - libffi-dev
       - libmount-dev
+      - libseccomp-dev
       - libselinux1-dev
       - ninja-build
     override-build: |
       yq=${SNAPCRAFT_STAGE}/yq
       export GOPATH=${SNAPCRAFT_STAGE}/gopath
+      export GO111MODULE="auto"
       kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
 
       versions_file="${kata_dir}/versions.yaml"
@@ -267,7 +282,7 @@ parts:
       ${kata_dir}/tools/packaging/scripts/apply_patches.sh "${patches_version_dir}"
 
       # Only x86_64 supports libpmem
-      [ "$(uname -m)" = "x86_64" ] && sudo apt-get --no-install-recommends install -y apt-utils ca-certificates libpmem-dev libseccomp-dev
+      [ "$(uname -m)" = "x86_64" ] && sudo apt-get --no-install-recommends install -y apt-utils ca-certificates libpmem-dev
 
       configure_hypervisor=${kata_dir}/tools/packaging/scripts/configure-hypervisor.sh
       chmod +x ${configure_hypervisor}

src/agent/.gitignore

@@ -1 +1,2 @@
 tarpaulin-report.html
+vendor/

src/agent/Cargo.toml

@@ -13,20 +13,25 @@ lazy_static = "1.3.0"
 ttrpc = { version = "0.5.0", features = ["async", "protobuf-codec"], default-features = false }
 protobuf = "=2.14.0"
 libc = "0.2.58"
-nix = "0.17.0"
-prctl = "1.0.0"
+nix = "0.21.0"
+capctl = "0.2.0"
 serde_json = "1.0.39"
 scan_fmt = "0.2.3"
 scopeguard = "1.0.0"
 regex = "1"
 
+# Async helpers
 async-trait = "0.1.42"
-tokio = { version = "1.2.0", features = ["rt", "rt-multi-thread", "sync", "macros", "io-util", "time", "signal", "io-std", "process", "fs"] }
+async-recursion = "0.3.2"
 futures = "0.3.12"
-netlink-sys = { version = "0.6.0", features = ["tokio_socket",]}
-tokio-vsock = "0.3.0"
-rtnetlink = "0.7.0"
-netlink-packet-utils = "0.4.0"
+
+# Async runtime
+tokio = { version = "1.2.0", features = ["full"] }
+tokio-vsock = "0.3.1"
+
+netlink-sys = { version = "0.7.0", features = ["tokio_socket",]}
+rtnetlink = "0.8.0"
+netlink-packet-utils = "0.4.1"
 ipnetwork = "0.17.0"
 
 # slog:
@@ -40,12 +45,20 @@ slog-scope = "4.1.2"
 slog-stdlog = "4.0.0"
 log = "0.4.11"
 
-# for testing
-tempfile = "3.1.0"
 prometheus = { version = "0.9.0", features = ["process"] }
 procfs = "0.7.9"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.2.2" }
+cgroups = { package = "cgroups-rs", version = "0.2.5" }
+
+# Tracing
+tracing = "0.1.26"
+tracing-subscriber = "0.2.18"
+tracing-opentelemetry = "0.13.0"
+opentelemetry = "0.14.0"
+vsock-exporter = { path = "vsock-exporter" }
+
+[dev-dependencies]
+tempfile = "3.1.0"
 
 [workspace]
 members = [

src/agent/Makefile

@@ -27,40 +27,7 @@ COMMIT_MSG = $(if $(COMMIT),$(COMMIT),unknown)
 # Exported to allow cargo to see it
 export VERSION_COMMIT := $(if $(COMMIT),$(VERSION)-$(COMMIT),$(VERSION))
 
-##VAR BUILD_TYPE=release|debug type of rust build
-BUILD_TYPE = release
-
-##VAR ARCH=arch target to build (format: uname -m)
-ARCH = $(shell uname -m)
-##VAR LIBC=musl|gnu
-LIBC ?= musl
-ifneq ($(LIBC),musl)
-    ifeq ($(LIBC),gnu)
-        override LIBC = gnu
-    else
-        $(error "ERROR: A non supported LIBC value was passed. Supported values are musl and gnu")
-    endif
-endif
-
-ifeq ($(ARCH), ppc64le)
-    override ARCH = powerpc64le
-    override LIBC = gnu
-    $(warning "WARNING: powerpc64le-unknown-linux-musl target is unavailable")
-endif
-
-ifeq ($(ARCH), s390x)
-    override LIBC = gnu
-    $(warning "WARNING: s390x-unknown-linux-musl target is unavailable")
-endif
-
-
-EXTRA_RUSTFLAGS :=
-ifeq ($(ARCH), aarch64)
-    override EXTRA_RUSTFLAGS = -C link-arg=-lgcc
-    $(warning "WARNING: aarch64-musl needs extra symbols from libgcc")
-endif
-
-TRIPLE = $(ARCH)-unknown-linux-$(LIBC)
+include ../../utils.mk
 
 TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET)
 
@@ -154,6 +121,10 @@ clean:
 	@rm -f $(GENERATED_FILES)
 	@rm -f tarpaulin-report.html
 
+vendor:
+	@cargo vendor
+
+
 #TARGET test: run cargo tests
 test:
 	@cargo test --all --target $(TRIPLE)
@@ -223,7 +194,8 @@ codecov-html: check_tarpaulin
 	help \
 	show-header \
 	show-summary \
-	optimize
+	optimize \
+	vendor
 
 ##TARGET generate-protocols: generate/update grpc agent protocols
 generate-protocols:

src/agent/VERSION

@@ -1 +0,0 @@
-2.0.0

src/agent/VERSION

@@ -0,0 +1 @@
+../../VERSION

src/agent/kata-agent.service.in

@@ -15,7 +15,7 @@ Wants=kata-containers.target
 StandardOutput=tty
 Type=simple
 ExecStart=@BINDIR@/@AGENT_NAME@
-LimitNOFILE=infinity
+LimitNOFILE=1048576
 # ExecStop is required for static agent tracing; in all other scenarios
 # the runtime handles shutting down the VM.
 ExecStop=/bin/sync ; /usr/bin/systemctl --force poweroff

src/agent/oci/src/lib.rs

@@ -8,7 +8,7 @@ extern crate serde;
 extern crate serde_derive;
 extern crate serde_json;
 
-use libc::mode_t;
+use libc::{self, mode_t};
 use std::collections::HashMap;
 
 mod serialize;
@@ -27,6 +27,10 @@ where
     *d == T::default()
 }
 
+fn default_seccomp_errno() -> u32 {
+    libc::EPERM as u32
+}
+
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
 pub struct Spec {
     #[serde(
@@ -54,7 +58,7 @@ pub struct Spec {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub windows: Option<Windows<String>>,
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub vm: Option<VM>,
+    pub vm: Option<Vm>,
 }
 
 impl Spec {
@@ -67,7 +71,7 @@ impl Spec {
     }
 }
 
-pub type LinuxRlimit = POSIXRlimit;
+pub type LinuxRlimit = PosixRlimit;
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
 pub struct Process {
@@ -89,7 +93,7 @@ pub struct Process {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub capabilities: Option<LinuxCapabilities>,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
-    pub rlimits: Vec<POSIXRlimit>,
+    pub rlimits: Vec<PosixRlimit>,
     #[serde(default, rename = "noNewPrivileges")]
     pub no_new_privileges: bool,
     #[serde(
@@ -195,9 +199,9 @@ pub struct Hooks {
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
 pub struct Linux {
     #[serde(default, rename = "uidMappings", skip_serializing_if = "Vec::is_empty")]
-    pub uid_mappings: Vec<LinuxIDMapping>,
+    pub uid_mappings: Vec<LinuxIdMapping>,
     #[serde(default, rename = "gidMappings", skip_serializing_if = "Vec::is_empty")]
-    pub gid_mappings: Vec<LinuxIDMapping>,
+    pub gid_mappings: Vec<LinuxIdMapping>,
     #[serde(default, skip_serializing_if = "HashMap::is_empty")]
     pub sysctl: HashMap<String, String>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -257,7 +261,7 @@ pub const UTSNAMESPACE: &str = "uts";
 pub const CGROUPNAMESPACE: &str = "cgroup";
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
-pub struct LinuxIDMapping {
+pub struct LinuxIdMapping {
     #[serde(default, rename = "containerID")]
     pub container_id: u32,
     #[serde(default, rename = "hostID")]
@@ -267,7 +271,7 @@ pub struct LinuxIDMapping {
 }
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
-pub struct POSIXRlimit {
+pub struct PosixRlimit {
     #[serde(default)]
     pub r#type: String,
     #[serde(default)]
@@ -293,7 +297,7 @@ pub struct LinuxInterfacePriority {
 }
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
-pub struct LinuxBlockIODevice {
+pub struct LinuxBlockIoDevice {
     #[serde(default)]
     pub major: i64,
     #[serde(default)]
@@ -303,7 +307,7 @@ pub struct LinuxBlockIODevice {
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
 pub struct LinuxWeightDevice {
     #[serde(flatten)]
-    pub blk: LinuxBlockIODevice,
+    pub blk: LinuxBlockIoDevice,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub weight: Option<u16>,
     #[serde(
@@ -317,13 +321,13 @@ pub struct LinuxWeightDevice {
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
 pub struct LinuxThrottleDevice {
     #[serde(flatten)]
-    pub blk: LinuxBlockIODevice,
+    pub blk: LinuxBlockIoDevice,
     #[serde(default)]
     pub rate: u64,
 }
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
-pub struct LinuxBlockIO {
+pub struct LinuxBlockIo {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub weight: Option<u16>,
     #[serde(
@@ -387,7 +391,7 @@ pub struct LinuxMemory {
 }
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
-pub struct LinuxCPU {
+pub struct LinuxCpu {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub shares: Option<u64>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -449,11 +453,11 @@ pub struct LinuxResources {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub memory: Option<LinuxMemory>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub cpu: Option<LinuxCPU>,
+    pub cpu: Option<LinuxCpu>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub pids: Option<LinuxPids>,
     #[serde(skip_serializing_if = "Option::is_none", rename = "blockIO")]
-    pub block_io: Option<LinuxBlockIO>,
+    pub block_io: Option<LinuxBlockIo>,
     #[serde(
         default,
         skip_serializing_if = "Vec::is_empty",
@@ -513,7 +517,7 @@ pub struct Solaris {
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub anet: Vec<SolarisAnet>,
     #[serde(default, skip_serializing_if = "Option::is_none", rename = "cappedCPU")]
-    pub capped_cpu: Option<SolarisCappedCPU>,
+    pub capped_cpu: Option<SolarisCappedCpu>,
     #[serde(
         default,
         skip_serializing_if = "Option::is_none",
@@ -523,7 +527,7 @@ pub struct Solaris {
 }
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
-pub struct SolarisCappedCPU {
+pub struct SolarisCappedCpu {
     #[serde(default, skip_serializing_if = "String::is_empty")]
     pub ncpus: String,
 }
@@ -601,7 +605,7 @@ pub struct WindowsResources {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub memory: Option<WindowsMemoryResources>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub cpu: Option<WindowsCPUResources>,
+    pub cpu: Option<WindowsCpuResources>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub storage: Option<WindowsStorageResources>,
 }
@@ -613,7 +617,7 @@ pub struct WindowsMemoryResources {
 }
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
-pub struct WindowsCPUResources {
+pub struct WindowsCpuResources {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub count: Option<u64>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -671,14 +675,14 @@ pub struct WindowsHyperV {
 }
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
-pub struct VM {
-    pub hypervisor: VMHypervisor,
-    pub kernel: VMKernel,
-    pub image: VMImage,
+pub struct Vm {
+    pub hypervisor: VmHypervisor,
+    pub kernel: VmKernel,
+    pub image: VmImage,
 }
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
-pub struct VMHypervisor {
+pub struct VmHypervisor {
     #[serde(default)]
     pub path: String,
     #[serde(default, skip_serializing_if = "String::is_empty")]
@@ -686,7 +690,7 @@ pub struct VMHypervisor {
 }
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
-pub struct VMKernel {
+pub struct VmKernel {
     #[serde(default)]
     pub path: String,
     #[serde(default, skip_serializing_if = "String::is_empty")]
@@ -696,7 +700,7 @@ pub struct VMKernel {
 }
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
-pub struct VMImage {
+pub struct VmImage {
     #[serde(default)]
     pub path: String,
     #[serde(default)]
@@ -710,6 +714,8 @@ pub struct LinuxSeccomp {
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub architectures: Vec<Arch>,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub flags: Vec<LinuxSeccompFlag>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub syscalls: Vec<LinuxSyscall>,
 }
 
@@ -733,14 +739,20 @@ pub const ARCHS390: &str = "SCMP_ARCH_S390";
 pub const ARCHS390X: &str = "SCMP_ARCH_S390X";
 pub const ARCHPARISC: &str = "SCMP_ARCH_PARISC";
 pub const ARCHPARISC64: &str = "SCMP_ARCH_PARISC64";
+pub const ARCHRISCV64: &str = "SCMP_ARCH_RISCV64";
+
+pub type LinuxSeccompFlag = String;
 
 pub type LinuxSeccompAction = String;
 
 pub const ACTKILL: &str = "SCMP_ACT_KILL";
+pub const ACTKILLPROCESS: &str = "SCMP_ACT_KILL_PROCESS";
+pub const ACTKILLTHREAD: &str = "SCMP_ACT_KILL_THREAD";
 pub const ACTTRAP: &str = "SCMP_ACT_TRAP";
 pub const ACTERRNO: &str = "SCMP_ACT_ERRNO";
 pub const ACTTRACE: &str = "SCMP_ACT_TRACE";
 pub const ACTALLOW: &str = "SCMP_ACT_ALLOW";
+pub const ACTLOG: &str = "SCMP_ACT_LOG";
 
 pub type LinuxSeccompOperator = String;
 
@@ -770,6 +782,8 @@ pub struct LinuxSyscall {
     pub names: Vec<String>,
     #[serde(default, skip_serializing_if = "String::is_empty")]
     pub action: LinuxSeccompAction,
+    #[serde(default = "default_seccomp_errno", rename = "errnoRet")]
+    pub errno_ret: u32,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub args: Vec<LinuxSeccompArg>,
 }
@@ -787,11 +801,11 @@ pub struct LinuxIntelRdt {
 #[derive(Debug, Serialize, Deserialize, Copy, Clone, PartialEq)]
 #[serde(rename_all = "lowercase")]
 pub enum ContainerState {
-    CREATING,
-    CREATED,
-    RUNNING,
-    STOPPED,
-    PAUSED,
+    Creating,
+    Created,
+    Running,
+    Stopped,
+    Paused,
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
@@ -832,7 +846,7 @@ mod tests {
         let expected = State {
             version: "0.2.0".to_string(),
             id: "oci-container1".to_string(),
-            status: ContainerState::RUNNING,
+            status: ContainerState::Running,
             pid: 4422,
             bundle: "/containers/redis".to_string(),
             annotations: [("myKey".to_string(), "myValue".to_string())]
@@ -1257,12 +1271,12 @@ mod tests {
                     ambient: vec!["CAP_NET_BIND_SERVICE".to_string()],
                 }),
                 rlimits: vec![
-                    crate::POSIXRlimit {
+                    crate::PosixRlimit {
                         r#type: "RLIMIT_CORE".to_string(),
                         hard: 1024,
                         soft: 1024,
                     },
-                    crate::POSIXRlimit {
+                    crate::PosixRlimit {
                         r#type: "RLIMIT_NOFILE".to_string(),
                         hard: 1024,
                         soft: 1024,
@@ -1394,12 +1408,12 @@ mod tests {
             .cloned()
             .collect(),
             linux: Some(crate::Linux {
-                uid_mappings: vec![crate::LinuxIDMapping {
+                uid_mappings: vec![crate::LinuxIdMapping {
                     container_id: 0,
                     host_id: 1000,
                     size: 32000,
                 }],
-                gid_mappings: vec![crate::LinuxIDMapping {
+                gid_mappings: vec![crate::LinuxIdMapping {
                     container_id: 0,
                     host_id: 1000,
                     size: 32000,
@@ -1444,7 +1458,7 @@ mod tests {
                         swappiness: Some(0),
                         disable_oom_killer: Some(false),
                     }),
-                    cpu: Some(crate::LinuxCPU {
+                    cpu: Some(crate::LinuxCpu {
                         shares: Some(1024),
                         quota: Some(1000000),
                         period: Some(500000),
@@ -1454,17 +1468,17 @@ mod tests {
                         mems: "0-7".to_string(),
                     }),
                     pids: Some(crate::LinuxPids { limit: 32771 }),
-                    block_io: Some(crate::LinuxBlockIO {
+                    block_io: Some(crate::LinuxBlockIo {
                         weight: Some(10),
                         leaf_weight: Some(10),
                         weight_device: vec![
                             crate::LinuxWeightDevice {
-                                blk: crate::LinuxBlockIODevice { major: 8, minor: 0 },
+                                blk: crate::LinuxBlockIoDevice { major: 8, minor: 0 },
                                 weight: Some(500),
                                 leaf_weight: Some(300),
                             },
                             crate::LinuxWeightDevice {
-                                blk: crate::LinuxBlockIODevice {
+                                blk: crate::LinuxBlockIoDevice {
                                     major: 8,
                                     minor: 16,
                                 },
@@ -1473,13 +1487,13 @@ mod tests {
                             },
                         ],
                         throttle_read_bps_device: vec![crate::LinuxThrottleDevice {
-                            blk: crate::LinuxBlockIODevice { major: 8, minor: 0 },
+                            blk: crate::LinuxBlockIoDevice { major: 8, minor: 0 },
                             rate: 600,
                         }],
                         throttle_write_bps_device: vec![],
                         throttle_read_iops_device: vec![],
                         throttle_write_iops_device: vec![crate::LinuxThrottleDevice {
-                            blk: crate::LinuxBlockIODevice {
+                            blk: crate::LinuxBlockIoDevice {
                                 major: 8,
                                 minor: 16,
                             },
@@ -1565,9 +1579,11 @@ mod tests {
                 seccomp: Some(crate::LinuxSeccomp {
                     default_action: "SCMP_ACT_ALLOW".to_string(),
                     architectures: vec!["SCMP_ARCH_X86".to_string(), "SCMP_ARCH_X32".to_string()],
+                    flags: vec![],
                     syscalls: vec![crate::LinuxSyscall {
                         names: vec!["getcwd".to_string(), "chmod".to_string()],
                         action: "SCMP_ACT_ERRNO".to_string(),
+                        errno_ret: crate::default_seccomp_errno(),
                         args: vec![],
                     }],
                 }),

src/agent/protocols/hack/update-generated-proto.sh

@@ -65,7 +65,7 @@ $GOPATH/src/github.com/kata-containers/kata-containers/src/agent/protocols/proto
 }
 
 if [ "$(basename $(pwd))" != "agent" ]; then
-	die "Please go to directory of protocols before execute this shell"
+	die "Please go to root directory of agent before execute this shell"
 fi
 
 # Protocol buffer files required to generate golang/rust bindings.

src/agent/protocols/protos/agent.proto

@@ -32,7 +32,6 @@ service AgentService {
 	rpc ExecProcess(ExecProcessRequest) returns (google.protobuf.Empty);
 	rpc SignalProcess(SignalProcessRequest) returns (google.protobuf.Empty);
 	rpc WaitProcess(WaitProcessRequest) returns (WaitProcessResponse); // wait & reap like waitpid(2)
-	rpc ListProcesses(ListProcessesRequest) returns (ListProcessesResponse);
 	rpc UpdateContainer(UpdateContainerRequest) returns (google.protobuf.Empty);
 	rpc StatsContainer(StatsContainerRequest) returns (StatsContainerResponse);
 	rpc PauseContainer(PauseContainerRequest) returns (google.protobuf.Empty);
@@ -126,18 +125,6 @@ message WaitProcessResponse {
 	int32 status = 1;
 }
 
-// ListProcessesRequest contains the options used to list running processes inside the container
-message ListProcessesRequest {
-	string container_id = 1;
-	string format = 2;
-	repeated string args = 3;
-}
-
-// ListProcessesResponse represents the list of running processes inside the container
-message ListProcessesResponse {
-	bytes process_list = 1;
-}
-
 message UpdateContainerRequest {
 	string container_id = 1;
 	LinuxResources resources = 2;

src/agent/protocols/protos/oci.proto

@@ -441,7 +441,8 @@ message LinuxInterfacePriority {
 message LinuxSeccomp {
 	string DefaultAction = 1;
 	repeated string Architectures = 2;
-	repeated LinuxSyscall Syscalls = 3  [(gogoproto.nullable) = false];
+	repeated string Flags = 3;
+	repeated LinuxSyscall Syscalls = 4  [(gogoproto.nullable) = false];
 }
 
 message LinuxSeccompArg {
@@ -454,7 +455,10 @@ message LinuxSeccompArg {
 message LinuxSyscall {
 	repeated string Names = 1;
 	string Action = 2;
-	repeated LinuxSeccompArg Args = 3  [(gogoproto.nullable) = false];
+	oneof ErrnoRet {
+		uint32 errnoret = 3;
+	}
+	repeated LinuxSeccompArg Args = 4  [(gogoproto.nullable) = false];
 }
 
 message LinuxIntelRdt {

src/agent/rustjail/Cargo.toml

@@ -11,9 +11,9 @@ serde_derive = "1.0.91"
 oci = { path = "../oci" }
 protocols = { path ="../protocols" }
 caps = "0.5.0"
-nix = "0.17.0"
+nix = "0.21.0"
 scopeguard = "1.0.0"
-prctl = "1.0.0"
+capctl = "0.2.0"
 lazy_static = "1.3.0"
 libc = "0.2.58"
 protobuf = "=2.14.0"
@@ -23,8 +23,7 @@ scan_fmt = "0.2"
 regex = "1.1"
 path-absolutize = "1.2.0"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.2.1" }
-tempfile = "3.1.0"
+cgroups = { package = "cgroups-rs", version = "0.2.5" }
 rlimit = "0.5.3"
 
 tokio = { version = "1.2.0", features = ["sync", "io-util", "process", "time", "macros"] }
@@ -34,3 +33,4 @@ inotify = "0.9.2"
 
 [dev-dependencies]
 serial_test = "0.5.0"
+tempfile = "3.1.0"

src/agent/rustjail/src/cgroups/fs/mod.rs

@@ -24,7 +24,7 @@ use anyhow::{anyhow, Context, Result};
 use libc::{self, pid_t};
 use nix::errno::Errno;
 use oci::{
-    LinuxBlockIO, LinuxCPU, LinuxDevice, LinuxDeviceCgroup, LinuxHugepageLimit, LinuxMemory,
+    LinuxBlockIo, LinuxCpu, LinuxDevice, LinuxDeviceCgroup, LinuxHugepageLimit, LinuxMemory,
     LinuxNetwork, LinuxPids, LinuxResources,
 };
 
@@ -272,7 +272,7 @@ fn set_hugepages_resources(
 
 fn set_block_io_resources(
     _cg: &cgroups::Cgroup,
-    blkio: &LinuxBlockIO,
+    blkio: &LinuxBlockIo,
     res: &mut cgroups::Resources,
 ) {
     info!(sl!(), "cgroup manager set block io");
@@ -302,7 +302,7 @@ fn set_block_io_resources(
         build_blk_io_device_throttle_resource(&blkio.throttle_write_iops_device);
 }
 
-fn set_cpu_resources(cg: &cgroups::Cgroup, cpu: &LinuxCPU) -> Result<()> {
+fn set_cpu_resources(cg: &cgroups::Cgroup, cpu: &LinuxCpu) -> Result<()> {
     info!(sl!(), "cgroup manager set cpu");
 
     let cpuset_controller: &CpuSetController = cg.controller_of().unwrap();
@@ -349,14 +349,34 @@ fn set_memory_resources(cg: &cgroups::Cgroup, memory: &LinuxMemory, update: bool
         mem_controller.set_kmem_limit(-1)?;
     }
 
-    set_resource!(mem_controller, set_limit, memory, limit);
-    set_resource!(mem_controller, set_soft_limit, memory, reservation);
-    set_resource!(mem_controller, set_kmem_limit, memory, kernel);
-    set_resource!(mem_controller, set_tcp_limit, memory, kernel_tcp);
+    // If the memory update is set to -1 we should also
+    // set swap to -1, it means unlimited memory.
+    let mut swap = memory.swap.unwrap_or(0);
+    if memory.limit == Some(-1) {
+        swap = -1;
+    }
 
-    if let Some(swap) = memory.swap {
-        // set memory swap
-        let swap = if cg.v2() {
+    if memory.limit.is_some() && swap != 0 {
+        let memstat = get_memory_stats(cg)
+            .into_option()
+            .ok_or_else(|| anyhow!("failed to get the cgroup memory stats"))?;
+        let memusage = memstat.get_usage();
+
+        // When update memory limit, the kernel would check the current memory limit
+        // set against the new swap setting, if the current memory limit is large than
+        // the new swap, then set limit first, otherwise the kernel would complain and
+        // refused to set; on the other hand, if the current memory limit is smaller than
+        // the new swap, then we should set the swap first and then set the memor limit.
+        if swap == -1 || memusage.get_limit() < swap as u64 {
+            mem_controller.set_memswap_limit(swap)?;
+            set_resource!(mem_controller, set_limit, memory, limit);
+        } else {
+            set_resource!(mem_controller, set_limit, memory, limit);
+            mem_controller.set_memswap_limit(swap)?;
+        }
+    } else {
+        set_resource!(mem_controller, set_limit, memory, limit);
+        swap = if cg.v2() {
             convert_memory_swap_to_v2_value(swap, memory.limit.unwrap_or(0))?
         } else {
             swap
@@ -366,6 +386,10 @@ fn set_memory_resources(cg: &cgroups::Cgroup, memory: &LinuxMemory, update: bool
         }
     }
 
+    set_resource!(mem_controller, set_soft_limit, memory, reservation);
+    set_resource!(mem_controller, set_kmem_limit, memory, kernel);
+    set_resource!(mem_controller, set_tcp_limit, memory, kernel_tcp);
+
     if let Some(swappiness) = memory.swappiness {
         if (0..=100).contains(&swappiness) {
             mem_controller.set_swappiness(swappiness as u64)?;
@@ -489,63 +513,61 @@ lazy_static! {
     };
 
     pub static ref DEFAULT_ALLOWED_DEVICES: Vec<LinuxDeviceCgroup> = {
-        let mut v = Vec::new();
+        vec![
+            // all mknod to all char devices
+            LinuxDeviceCgroup {
+                allow: true,
+                r#type: "c".to_string(),
+                major: Some(WILDCARD),
+                minor: Some(WILDCARD),
+                access: "m".to_string(),
+            },
 
-        // all mknod to all char devices
-        v.push(LinuxDeviceCgroup {
-            allow: true,
-            r#type: "c".to_string(),
-            major: Some(WILDCARD),
-            minor: Some(WILDCARD),
-            access: "m".to_string(),
-        });
+            // all mknod to all block devices
+            LinuxDeviceCgroup {
+                allow: true,
+                r#type: "b".to_string(),
+                major: Some(WILDCARD),
+                minor: Some(WILDCARD),
+                access: "m".to_string(),
+            },
 
-        // all mknod to all block devices
-        v.push(LinuxDeviceCgroup {
-            allow: true,
-            r#type: "b".to_string(),
-            major: Some(WILDCARD),
-            minor: Some(WILDCARD),
-            access: "m".to_string(),
-        });
+            // all read/write/mknod to char device /dev/console
+            LinuxDeviceCgroup {
+                allow: true,
+                r#type: "c".to_string(),
+                major: Some(5),
+                minor: Some(1),
+                access: "rwm".to_string(),
+            },
 
-        // all read/write/mknod to char device /dev/console
-        v.push(LinuxDeviceCgroup {
-            allow: true,
-            r#type: "c".to_string(),
-            major: Some(5),
-            minor: Some(1),
-            access: "rwm".to_string(),
-        });
+            // all read/write/mknod to char device /dev/pts/<N>
+            LinuxDeviceCgroup {
+                allow: true,
+                r#type: "c".to_string(),
+                major: Some(136),
+                minor: Some(WILDCARD),
+                access: "rwm".to_string(),
+            },
 
-        // all read/write/mknod to char device /dev/pts/<N>
-        v.push(LinuxDeviceCgroup {
-            allow: true,
-            r#type: "c".to_string(),
-            major: Some(136),
-            minor: Some(WILDCARD),
-            access: "rwm".to_string(),
-        });
+            // all read/write/mknod to char device /dev/ptmx
+            LinuxDeviceCgroup {
+                allow: true,
+                r#type: "c".to_string(),
+                major: Some(5),
+                minor: Some(2),
+                access: "rwm".to_string(),
+            },
 
-        // all read/write/mknod to char device /dev/ptmx
-        v.push(LinuxDeviceCgroup {
-            allow: true,
-            r#type: "c".to_string(),
-            major: Some(5),
-            minor: Some(2),
-            access: "rwm".to_string(),
-        });
-
-        // all read/write/mknod to char device /dev/net/tun
-        v.push(LinuxDeviceCgroup {
-            allow: true,
-            r#type: "c".to_string(),
-            major: Some(10),
-            minor: Some(200),
-            access: "rwm".to_string(),
-        });
-
-        v
+            // all read/write/mknod to char device /dev/net/tun
+            LinuxDeviceCgroup {
+                allow: true,
+                r#type: "c".to_string(),
+                major: Some(10),
+                minor: Some(200),
+                access: "rwm".to_string(),
+            },
+        ]
     };
 }
 
@@ -901,12 +923,12 @@ pub fn get_mounts() -> Result<HashMap<String, String>> {
     let paths = get_paths()?;
 
     for l in fs::read_to_string(MOUNTS)?.lines() {
-        let p: Vec<&str> = l.split(" - ").collect();
+        let p: Vec<&str> = l.splitn(2, " - ").collect();
         let pre: Vec<&str> = p[0].split(' ').collect();
         let post: Vec<&str> = p[1].split(' ').collect();
 
         if post.len() != 3 {
-            warn!(sl!(), "mountinfo corrupted!");
+            warn!(sl!(), "can't parse {} line {:?}", MOUNTS, l);
             continue;
         }
 

src/agent/rustjail/src/cgroups/notifier.rs

@@ -8,7 +8,7 @@ use eventfd::{eventfd, EfdFlags};
 use nix::sys::eventfd;
 use std::fs::{self, File};
 use std::os::unix::io::{AsRawFd, FromRawFd};
-use std::path::{Path, PathBuf};
+use std::path::Path;
 
 use crate::pipestream::PipeStream;
 use futures::StreamExt as _;
@@ -35,7 +35,7 @@ pub async fn notify_oom(cid: &str, cg_dir: String) -> Result<Receiver<String>> {
 // Flat keyed file format:
 //   KEY0 VAL0\n
 //   KEY1 VAL1\n
-fn get_value_from_cgroup(path: &PathBuf, key: &str) -> Result<i64> {
+fn get_value_from_cgroup(path: &Path, key: &str) -> Result<i64> {
     let content = fs::read_to_string(path)?;
     info!(
         sl!(),
@@ -117,12 +117,12 @@ async fn register_memory_event_v2(
                     return;
                 }
             }
-        }
 
-        // When a cgroup is destroyed, an event is sent to eventfd.
-        // So if the control path is gone, return instead of notifying.
-        if !Path::new(&event_control_path).exists() {
-            return;
+            // When a cgroup is destroyed, an event is sent to eventfd.
+            // So if the control path is gone, return instead of notifying.
+            if !Path::new(&event_control_path).exists() {
+                return;
+            }
         }
     });
 
@@ -139,19 +139,6 @@ async fn notify_on_oom(cid: &str, dir: String) -> Result<Receiver<String>> {
     register_memory_event(cid, dir, "memory.oom_control", "").await
 }
 
-// level is one of "low", "medium", or "critical"
-async fn notify_memory_pressure(cid: &str, dir: String, level: &str) -> Result<Receiver<String>> {
-    if dir.is_empty() {
-        return Err(anyhow!("memory controller missing"));
-    }
-
-    if level != "low" && level != "medium" && level != "critical" {
-        return Err(anyhow!("invalid pressure level {}", level));
-    }
-
-    register_memory_event(cid, dir, "memory.pressure_level", level).await
-}
-
 async fn register_memory_event(
     cid: &str,
     cg_dir: String,

src/agent/rustjail/src/configs/device.rs

@@ -1,56 +0,0 @@
-// Copyright (c) 2019 Ant Financial
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-use libc::*;
-use serde;
-#[macro_use]
-use serde_derive;
-use serde_json;
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct Device {
-    #[serde(default)]
-    r#type: char,
-    #[serde(default)]
-    path: String,
-    #[serde(default)]
-    major: i64,
-    #[serde(default)]
-    minor: i64,
-    #[serde(default)]
-    permissions: String,
-    #[serde(default)]
-    file_mode: mode_t,
-    #[serde(default)]
-    uid: i32,
-    #[serde(default)]
-    gid: i32,
-    #[serde(default)]
-    allow: bool,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct BlockIODevice {
-    #[serde(default)]
-    major: i64,
-    #[serde(default)]
-    minor: i64,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct WeightDevice {
-    block: BlockIODevice,
-    #[serde(default)]
-    weight: u16,
-    #[serde(default, rename = "leafWeight")]
-    leaf_weight: u16,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct ThrottleDevice {
-    block: BlockIODevice,
-    #[serde(default)]
-    rate: u64,
-}

src/agent/rustjail/src/configs/mod.rs

@@ -1,368 +0,0 @@
-// Copyright (c) 2019 Ant Financial
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-use serde;
-#[macro_use]
-use serde_derive;
-use serde_json;
-
-use protocols::oci::State as OCIState;
-
-use std::collections::HashMap;
-use std::fmt;
-use std::path::PathBuf;
-use std::time::Duration;
-
-use nix::unistd;
-
-use self::device::{Device, ThrottleDevice, WeightDevice};
-use self::namespaces::Namespaces;
-use crate::specconv::CreateOpts;
-
-pub mod device;
-pub mod namespaces;
-pub mod validator;
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct Rlimit {
-    #[serde(default)]
-    r#type: i32,
-    #[serde(default)]
-    hard: i32,
-    #[serde(default)]
-    soft: i32,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct IDMap {
-    #[serde(default)]
-    container_id: i32,
-    #[serde(default)]
-    host_id: i32,
-    #[serde(default)]
-    size: i32,
-}
-
-type Action = i32;
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct Seccomp {
-    #[serde(default)]
-    default_action: Action,
-    #[serde(default)]
-    architectures: Vec<String>,
-    #[serde(default)]
-    syscalls: Vec<Syscall>,
-}
-
-type Operator = i32;
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct Arg {
-    #[serde(default)]
-    index: u32,
-    #[serde(default)]
-    value: u64,
-    #[serde(default)]
-    value_two: u64,
-    #[serde(default)]
-    op: Operator,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct Syscall {
-    #[serde(default, skip_serializing_if = "String::is_empty")]
-    name: String,
-    #[serde(default)]
-    action: Action,
-    #[serde(default, skip_serializing_if = "Vec::is_empty")]
-    args: Vec<Arg>,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct Config<'a> {
-    #[serde(default)]
-    no_pivot_root: bool,
-    #[serde(default)]
-    parent_death_signal: i32,
-    #[serde(default)]
-    rootfs: String,
-    #[serde(default)]
-    readonlyfs: bool,
-    #[serde(default, rename = "rootPropagation")]
-    root_propagation: i32,
-    #[serde(default)]
-    mounts: Vec<Mount>,
-    #[serde(default)]
-    devices: Vec<Device>,
-    #[serde(default)]
-    mount_label: String,
-    #[serde(default)]
-    hostname: String,
-    #[serde(default)]
-    namespaces: Namespaces,
-    #[serde(default)]
-    capabilities: Option<Capabilities>,
-    #[serde(default)]
-    networks: Vec<Network>,
-    #[serde(default)]
-    routes: Vec<Route>,
-    #[serde(default)]
-    cgroups: Option<Cgroup<'a>>,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
-    apparmor_profile: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
-    process_label: String,
-    #[serde(default, skip_serializing_if = "Vec::is_empty")]
-    rlimits: Vec<Rlimit>,
-    #[serde(default)]
-    oom_score_adj: Option<i32>,
-    #[serde(default)]
-    uid_mappings: Vec<IDMap>,
-    #[serde(default)]
-    gid_mappings: Vec<IDMap>,
-    #[serde(default)]
-    mask_paths: Vec<String>,
-    #[serde(default)]
-    readonly_paths: Vec<String>,
-    #[serde(default)]
-    sysctl: HashMap<String, String>,
-    #[serde(default)]
-    seccomp: Option<Seccomp>,
-    #[serde(default)]
-    no_new_privileges: bool,
-    hooks: Option<Hooks>,
-    #[serde(default)]
-    version: String,
-    #[serde(default)]
-    labels: Vec<String>,
-    #[serde(default)]
-    no_new_keyring: bool,
-    #[serde(default)]
-    intel_rdt: Option<IntelRdt>,
-    #[serde(default)]
-    rootless_euid: bool,
-    #[serde(default)]
-    rootless_cgroups: bool,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct Hooks {
-    prestart: Vec<Box<Hook>>,
-    poststart: Vec<Box<Hook>>,
-    poststop: Vec<Box<Hook>>,
-}
-#[derive(Serialize, Deserialize, Debug)]
-pub struct Capabilities {
-    bounding: Vec<String>,
-    effective: Vec<String>,
-    inheritable: Vec<String>,
-    permitted: Vec<String>,
-    ambient: Vec<String>,
-}
-
-pub trait Hook {
-    fn run(&self, state: &OCIState) -> Result<()>;
-}
-
-pub struct FuncHook {
-    // run: fn(&OCIState) -> Result<()>,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct Command {
-    #[serde(default)]
-    path: String,
-    #[serde(default)]
-    args: Vec<String>,
-    #[serde(default)]
-    env: Vec<String>,
-    #[serde(default)]
-    dir: String,
-    #[serde(default)]
-    timeout: Duration,
-}
-
-pub struct CommandHook {
-    command: Command,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct Mount {
-    #[serde(default)]
-    source: String,
-    #[serde(default)]
-    destination: String,
-    #[serde(default)]
-    device: String,
-    #[serde(default)]
-    flags: i32,
-    #[serde(default)]
-    propagation_flags: Vec<i32>,
-    #[serde(default)]
-    data: String,
-    #[serde(default)]
-    relabel: String,
-    #[serde(default)]
-    extensions: i32,
-    #[serde(default)]
-    premount_cmds: Vec<Command>,
-    #[serde(default)]
-    postmount_cmds: Vec<Command>,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct HugepageLimit {
-    #[serde(default)]
-    page_size: String,
-    #[serde(default)]
-    limit: u64,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct IntelRdt {
-    #[serde(default, skip_serializing_if = "String::is_empty")]
-    l3_cache_schema: String,
-    #[serde(
-        default,
-        rename = "memBwSchema",
-        skip_serializing_if = "String::is_empty"
-    )]
-    mem_bw_schema: String,
-}
-
-pub type FreezerState = String;
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct Cgroup<'a> {
-    #[serde(default, skip_serializing_if = "String::is_empty")]
-    name: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
-    parent: String,
-    #[serde(default)]
-    path: String,
-    #[serde(default)]
-    scope_prefix: String,
-    paths: HashMap<String, String>,
-    resource: &'a Resources<'a>,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct Resources<'a> {
-    #[serde(default)]
-    allow_all_devices: bool,
-    #[serde(default, skip_serializing_if = "Vec::is_empty")]
-    allowed_devices: Vec<&'a Device>,
-    #[serde(default, skip_serializing_if = "Vec::is_empty")]
-    denied_devices: Vec<&'a Device>,
-    #[serde(default)]
-    devices: Vec<&'a Device>,
-    #[serde(default)]
-    memory: i64,
-    #[serde(default)]
-    memory_reservation: i64,
-    #[serde(default)]
-    memory_swap: i64,
-    #[serde(default)]
-    kernel_memory: i64,
-    #[serde(default)]
-    kernel_memory_tcp: i64,
-    #[serde(default)]
-    cpu_shares: u64,
-    #[serde(default)]
-    cpu_quota: i64,
-    #[serde(default)]
-    cpu_period: u64,
-    #[serde(default)]
-    cpu_rt_quota: i64,
-    #[serde(default)]
-    cpu_rt_period: u64,
-    #[serde(default)]
-    cpuset_cpus: String,
-    #[serde(default)]
-    cpuset_mems: String,
-    #[serde(default)]
-    pids_limit: i64,
-    #[serde(default)]
-    blkio_weight: u64,
-    #[serde(default)]
-    blkio_leaf_weight: u64,
-    #[serde(default)]
-    blkio_weight_device: Vec<&'a WeightDevice>,
-    #[serde(default)]
-    blkio_throttle_read_bps_device: Vec<&'a ThrottleDevice>,
-    #[serde(default)]
-    blkio_throttle_write_bps_device: Vec<&'a ThrottleDevice>,
-    #[serde(default)]
-    blkio_throttle_read_iops_device: Vec<&'a ThrottleDevice>,
-    #[serde(default)]
-    blkio_throttle_write_iops_device: Vec<&'a ThrottleDevice>,
-    #[serde(default)]
-    freezer: FreezerState,
-    #[serde(default)]
-    hugetlb_limit: Vec<&'a HugepageLimit>,
-    #[serde(default)]
-    oom_kill_disable: bool,
-    #[serde(default)]
-    memory_swapiness: u64,
-    #[serde(default)]
-    net_prio_ifpriomap: Vec<&'a IfPrioMap>,
-    #[serde(default)]
-    net_cls_classid_u: u32,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct Network {
-    #[serde(default)]
-    r#type: String,
-    #[serde(default)]
-    name: String,
-    #[serde(default)]
-    bridge: String,
-    #[serde(default)]
-    mac_address: String,
-    #[serde(default)]
-    address: String,
-    #[serde(default)]
-    gateway: String,
-    #[serde(default)]
-    ipv6_address: String,
-    #[serde(default)]
-    ipv6_gateway: String,
-    #[serde(default)]
-    mtu: i32,
-    #[serde(default)]
-    txqueuelen: i32,
-    #[serde(default)]
-    host_interface_name: String,
-    #[serde(default)]
-    hairpin_mode: bool,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct Route {
-    #[serde(default)]
-    destination: String,
-    #[serde(default)]
-    source: String,
-    #[serde(default)]
-    gateway: String,
-    #[serde(default)]
-    interface_name: String,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct IfPrioMap {
-    #[serde(default)]
-    interface: String,
-    #[serde(default)]
-    priority: i32,
-}
-
-impl IfPrioMap {
-    fn cgroup_string(&self) -> String {
-        format!("{} {}", self.interface, self.priority)
-    }
-}

src/agent/rustjail/src/configs/namespaces.rs

@@ -1,46 +0,0 @@
-// Copyright (c) 2019 Ant Financial
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-use serde;
-#[macro_use]
-use serde_derive;
-use serde_json;
-
-use std::collections::HashMap;
-#[macro_use]
-use lazy_static;
-
-pub type NamespaceType = String;
-pub type Namespaces = Vec<Namespace>;
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct Namespace {
-    #[serde(default)]
-    r#type: NamespaceType,
-    #[serde(default)]
-    path: String,
-}
-
-pub const NEWNET: &'static str = "NEWNET";
-pub const NEWPID: &'static str = "NEWPID";
-pub const NEWNS: &'static str = "NEWNS";
-pub const NEWUTS: &'static str = "NEWUTS";
-pub const NEWUSER: &'static str = "NEWUSER";
-pub const NEWCGROUP: &'static str = "NEWCGROUP";
-pub const NEWIPC: &'static str = "NEWIPC";
-
-lazy_static! {
-    static ref TYPETONAME: HashMap<&'static str, &'static str> = {
-        let mut m = HashMap::new();
-        m.insert("pid", "pid");
-        m.insert("network", "net");
-        m.insert("mount", "mnt");
-        m.insert("user", "user");
-        m.insert("uts", "uts");
-        m.insert("ipc", "ipc");
-        m.insert("cgroup", "cgroup");
-        m
-    };
-}

src/agent/rustjail/src/configs/validator.rs

@@ -1,23 +0,0 @@
-// Copyright (c) 2019 Ant Financial
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-use crate::configs::Config;
-use std::io::Result;
-
-pub trait Validator {
-    fn validate(&self, config: &Config) -> Result<()> {
-        Ok(())
-    }
-}
-
-pub struct ConfigValidator {}
-
-impl Validator for ConfigValidator {}
-
-impl ConfigValidator {
-    fn new() -> Self {
-        ConfigValidator {}
-    }
-}

src/agent/rustjail/src/container.rs

@@ -5,10 +5,10 @@
 
 use anyhow::{anyhow, Context, Result};
 use libc::pid_t;
-use oci::{ContainerState, LinuxDevice, LinuxIDMapping};
+use oci::{ContainerState, LinuxDevice, LinuxIdMapping};
 use oci::{Hook, Linux, LinuxNamespace, LinuxResources, Spec};
 use std::clone::Clone;
-use std::ffi::{CStr, CString};
+use std::ffi::CString;
 use std::fmt::Display;
 use std::fs;
 use std::os::unix::io::RawFd;
@@ -48,6 +48,7 @@ use oci::State as OCIState;
 use std::collections::HashMap;
 use std::os::unix::io::FromRawFd;
 use std::str::FromStr;
+use std::sync::Arc;
 
 use slog::{info, o, Logger};
 
@@ -57,13 +58,11 @@ use crate::sync_with_async::{read_async, write_async};
 use async_trait::async_trait;
 use rlimit::{setrlimit, Resource, Rlim};
 use tokio::io::AsyncBufReadExt;
+use tokio::sync::Mutex;
 
 use crate::utils;
 
-const STATE_FILENAME: &str = "state.json";
 const EXEC_FIFO_FILENAME: &str = "exec.fifo";
-const VER_MARKER: &str = "1.2.5";
-const PID_NS_PATH: &str = "/proc/self/ns/pid";
 
 const INIT: &str = "INIT";
 const NO_PIVOT: &str = "NO_PIVOT";
@@ -83,8 +82,8 @@ pub struct ContainerStatus {
 impl ContainerStatus {
     fn new() -> Self {
         ContainerStatus {
-            pre_status: ContainerState::CREATED,
-            cur_status: ContainerState::CREATED,
+            pre_status: ContainerState::Created,
+            cur_status: ContainerState::Created,
         }
     }
 
@@ -92,10 +91,6 @@ impl ContainerStatus {
         self.cur_status
     }
 
-    fn pre_status(&self) -> ContainerState {
-        self.pre_status
-    }
-
     fn transition(&mut self, to: ContainerState) {
         self.pre_status = self.status();
         self.cur_status = to;
@@ -106,6 +101,9 @@ pub type Config = CreateOpts;
 type NamespaceType = String;
 
 lazy_static! {
+    // This locker ensures the child exit signal will be received by the right receiver.
+    pub static ref WAIT_PID_LOCKER: Arc<Mutex<bool>> = Arc::new(Mutex::new(false));
+
     static ref NAMESPACES: HashMap<&'static str, CloneFlags> = {
         let mut m = HashMap::new();
         m.insert("user", CloneFlags::CLONE_NEWUSER);
@@ -132,62 +130,62 @@ lazy_static! {
     };
 
     pub static ref DEFAULT_DEVICES: Vec<LinuxDevice> = {
-        let mut v = Vec::new();
-        v.push(LinuxDevice {
-            path: "/dev/null".to_string(),
-            r#type: "c".to_string(),
-            major: 1,
-            minor: 3,
-            file_mode: Some(0o666),
-            uid: Some(0xffffffff),
-            gid: Some(0xffffffff),
-        });
-        v.push(LinuxDevice {
-            path: "/dev/zero".to_string(),
-            r#type: "c".to_string(),
-            major: 1,
-            minor: 5,
-            file_mode: Some(0o666),
-            uid: Some(0xffffffff),
-            gid: Some(0xffffffff),
-        });
-        v.push(LinuxDevice {
-            path: "/dev/full".to_string(),
-            r#type: String::from("c"),
-            major: 1,
-            minor: 7,
-            file_mode: Some(0o666),
-            uid: Some(0xffffffff),
-            gid: Some(0xffffffff),
-        });
-        v.push(LinuxDevice {
-            path: "/dev/tty".to_string(),
-            r#type: "c".to_string(),
-            major: 5,
-            minor: 0,
-            file_mode: Some(0o666),
-            uid: Some(0xffffffff),
-            gid: Some(0xffffffff),
-        });
-        v.push(LinuxDevice {
-            path: "/dev/urandom".to_string(),
-            r#type: "c".to_string(),
-            major: 1,
-            minor: 9,
-            file_mode: Some(0o666),
-            uid: Some(0xffffffff),
-            gid: Some(0xffffffff),
-        });
-        v.push(LinuxDevice {
-            path: "/dev/random".to_string(),
-            r#type: "c".to_string(),
-            major: 1,
-            minor: 8,
-            file_mode: Some(0o666),
-            uid: Some(0xffffffff),
-            gid: Some(0xffffffff),
-        });
-        v
+        vec![
+            LinuxDevice {
+                path: "/dev/null".to_string(),
+                r#type: "c".to_string(),
+                major: 1,
+                minor: 3,
+                file_mode: Some(0o666),
+                uid: Some(0xffffffff),
+                gid: Some(0xffffffff),
+            },
+            LinuxDevice {
+                path: "/dev/zero".to_string(),
+                r#type: "c".to_string(),
+                major: 1,
+                minor: 5,
+                file_mode: Some(0o666),
+                uid: Some(0xffffffff),
+                gid: Some(0xffffffff),
+            },
+            LinuxDevice {
+                path: "/dev/full".to_string(),
+                r#type: String::from("c"),
+                major: 1,
+                minor: 7,
+                file_mode: Some(0o666),
+                uid: Some(0xffffffff),
+                gid: Some(0xffffffff),
+            },
+            LinuxDevice {
+                path: "/dev/tty".to_string(),
+                r#type: "c".to_string(),
+                major: 5,
+                minor: 0,
+                file_mode: Some(0o666),
+                uid: Some(0xffffffff),
+                gid: Some(0xffffffff),
+            },
+            LinuxDevice {
+                path: "/dev/urandom".to_string(),
+                r#type: "c".to_string(),
+                major: 1,
+                minor: 9,
+                file_mode: Some(0o666),
+                uid: Some(0xffffffff),
+                gid: Some(0xffffffff),
+            },
+            LinuxDevice {
+                path: "/dev/random".to_string(),
+                r#type: "c".to_string(),
+                major: 1,
+                minor: 8,
+                file_mode: Some(0o666),
+                uid: Some(0xffffffff),
+                gid: Some(0xffffffff),
+            },
+        ]
     };
 }
 
@@ -255,7 +253,7 @@ pub struct State {
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct SyncPC {
+pub struct SyncPc {
     #[serde(default)]
     pid: pid_t,
 }
@@ -268,7 +266,7 @@ pub trait Container: BaseContainer {
 impl Container for LinuxContainer {
     fn pause(&mut self) -> Result<()> {
         let status = self.status();
-        if status != ContainerState::RUNNING && status != ContainerState::CREATED {
+        if status != ContainerState::Running && status != ContainerState::Created {
             return Err(anyhow!(
                 "failed to pause container: current status is: {:?}",
                 status
@@ -281,7 +279,7 @@ impl Container for LinuxContainer {
                 .unwrap()
                 .freeze(FreezerState::Frozen)?;
 
-            self.status.transition(ContainerState::PAUSED);
+            self.status.transition(ContainerState::Paused);
             return Ok(());
         }
         Err(anyhow!("failed to get container's cgroup manager"))
@@ -289,7 +287,7 @@ impl Container for LinuxContainer {
 
     fn resume(&mut self) -> Result<()> {
         let status = self.status();
-        if status != ContainerState::PAUSED {
+        if status != ContainerState::Paused {
             return Err(anyhow!("container status is: {:?}, not paused", status));
         }
 
@@ -299,7 +297,7 @@ impl Container for LinuxContainer {
                 .unwrap()
                 .freeze(FreezerState::Thawed)?;
 
-            self.status.transition(ContainerState::RUNNING);
+            self.status.transition(ContainerState::Running);
             return Ok(());
         }
         Err(anyhow!("failed to get container's cgroup manager"))
@@ -341,7 +339,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         Err(_e) => sched::unshare(CloneFlags::CLONE_NEWPID)?,
     }
 
-    match fork() {
+    match unsafe { fork() } {
         Ok(ForkResult::Parent { child, .. }) => {
             log_child!(
                 cfd_log,
@@ -464,7 +462,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     // Ref: https://github.com/opencontainers/runc/commit/50a19c6ff828c58e5dab13830bd3dacde268afe5
     //
     if !nses.is_empty() {
-        prctl::set_dumpable(false)
+        capctl::prctl::set_dumpable(false)
             .map_err(|e| anyhow!(e).context("set process non-dumpable failed"))?;
     }
 
@@ -540,7 +538,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         // notify parent to run prestart hooks
         write_sync(cwfd, SYNC_SUCCESS, "")?;
         // wait parent run prestart hooks
-        let _ = read_sync(crfd)?;
+        read_sync(crfd)?;
     }
 
     if mount_fd != -1 {
@@ -597,7 +595,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
 
     // NoNewPeiviledges, Drop capabilities
     if oci_process.no_new_privileges {
-        prctl::set_no_new_privileges(true).map_err(|_| anyhow!("cannot set no new privileges"))?;
+        capctl::prctl::set_no_new_privs().map_err(|_| anyhow!("cannot set no new privileges"))?;
     }
 
     if oci_process.capabilities.is_some() {
@@ -607,8 +605,6 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
 
     if init {
         // notify parent to run poststart hooks
-        // cfd is closed when return from join_namespaces
-        // should retunr cfile instead of cfd?
         write_sync(cwfd, SYNC_SUCCESS, "")?;
     }
 
@@ -634,12 +630,12 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         env::set_var(v[0], v[1]);
     }
 
-    // set the "HOME" env getting from "/etc/passwd"
+    // set the "HOME" env getting from "/etc/passwd", if
+    // there's no uid entry in /etc/passwd, set "/" as the
+    // home env.
     if env::var_os(HOME_ENV_KEY).is_none() {
-        match utils::home_dir(guser.uid) {
-            Ok(home_dir) => env::set_var(HOME_ENV_KEY, home_dir),
-            Err(e) => log_child!(cfd_log, "failed to get home dir: {:?}", e),
-        }
+        let home_dir = utils::home_dir(guser.uid).unwrap_or_else(|_| String::from("/"));
+        env::set_var(HOME_ENV_KEY, home_dir);
     }
 
     let exec_file = Path::new(&args[0]);
@@ -734,7 +730,7 @@ impl BaseContainer for LinuxContainer {
         };
 
         let status = self.status();
-        let pid = if status != ContainerState::STOPPED {
+        let pid = if status != ContainerState::Stopped {
             self.init_process_pid
         } else {
             0
@@ -817,7 +813,7 @@ impl BaseContainer for LinuxContainer {
             if stat::stat(fifo_file.as_str()).is_ok() {
                 return Err(anyhow!("exec fifo exists"));
             }
-            unistd::mkfifo(fifo_file.as_str(), Mode::from_bits(0o622).unwrap())?;
+            unistd::mkfifo(fifo_file.as_str(), Mode::from_bits(0o644).unwrap())?;
 
             fifofd = fcntl::open(
                 fifo_file.as_str(),
@@ -908,7 +904,7 @@ impl BaseContainer for LinuxContainer {
             child = child.env(PIDNS_FD, format!("{}", pidns.unwrap()));
         }
 
-        let child = child.spawn()?;
+        child.spawn()?;
 
         unistd::close(crfd)?;
         unistd::close(cwfd)?;
@@ -964,19 +960,6 @@ impl BaseContainer for LinuxContainer {
 
         self.created = SystemTime::now();
 
-        // create the pipes for notify process exited
-        let (exit_pipe_r, exit_pipe_w) = unistd::pipe2(OFlag::O_CLOEXEC)
-            .context("failed to create pipe")
-            .map_err(|e| {
-                let _ = signal::kill(Pid::from_raw(child.id() as i32), Some(Signal::SIGKILL))
-                    .map_err(|e| warn!(logger, "signal::kill creating pipe {:?}", e));
-
-                e
-            })?;
-
-        p.exit_pipe_w = Some(exit_pipe_w);
-        p.exit_pipe_r = Some(exit_pipe_r);
-
         if p.init {
             let spec = self.config.spec.as_mut().unwrap();
             update_namespaces(&self.logger, spec, p.pid)?;
@@ -997,7 +980,7 @@ impl BaseContainer for LinuxContainer {
 
         if init {
             self.exec()?;
-            self.status.transition(ContainerState::RUNNING);
+            self.status.transition(ContainerState::Running);
         }
 
         Ok(())
@@ -1019,7 +1002,7 @@ impl BaseContainer for LinuxContainer {
             }
         }
 
-        self.status.transition(ContainerState::STOPPED);
+        self.status.transition(ContainerState::Stopped);
         mount::umount2(
             spec.root.as_ref().unwrap().path.as_str(),
             MntFlags::MNT_DETACH,
@@ -1055,7 +1038,7 @@ impl BaseContainer for LinuxContainer {
             .unwrap()
             .as_secs();
 
-        self.status.transition(ContainerState::RUNNING);
+        self.status.transition(ContainerState::Running);
         unistd::close(fd)?;
 
         Ok(())
@@ -1089,9 +1072,8 @@ fn do_exec(args: &[String]) -> ! {
         .iter()
         .map(|s| CString::new(s.to_string()).unwrap_or_default())
         .collect();
-    let a: Vec<&CStr> = sa.iter().map(|s| s.as_c_str()).collect();
 
-    let _ = unistd::execvp(p.as_c_str(), a.as_slice()).map_err(|e| match e {
+    let _ = unistd::execvp(p.as_c_str(), &sa).map_err(|e| match e {
         nix::Error::Sys(errno) => {
             std::process::exit(errno as i32);
         }
@@ -1264,7 +1246,7 @@ async fn join_namespaces(
 
     if p.init {
         info!(logger, "notify child parent ready to run prestart hook!");
-        let _ = read_async(pipe_r).await?;
+        read_async(pipe_r).await?;
 
         info!(logger, "get ready to run prestart hook!");
 
@@ -1302,7 +1284,7 @@ async fn join_namespaces(
     Ok(())
 }
 
-fn write_mappings(logger: &Logger, path: &str, maps: &[LinuxIDMapping]) -> Result<()> {
+fn write_mappings(logger: &Logger, path: &str, maps: &[LinuxIdMapping]) -> Result<()> {
     let data = maps
         .iter()
         .filter(|m| m.size != 0)
@@ -1324,7 +1306,7 @@ fn write_mappings(logger: &Logger, path: &str, maps: &[LinuxIDMapping]) -> Resul
 
 fn setid(uid: Uid, gid: Gid) -> Result<()> {
     // set uid/gid
-    prctl::set_keep_capabilities(true)
+    capctl::prctl::set_keepcaps(true)
         .map_err(|e| anyhow!(e).context("set keep capabilities returned"))?;
 
     {
@@ -1338,7 +1320,7 @@ fn setid(uid: Uid, gid: Gid) -> Result<()> {
         capabilities::reset_effective()?;
     }
 
-    prctl::set_keep_capabilities(false)
+    capctl::prctl::set_keepcaps(false)
         .map_err(|e| anyhow!(e).context("set keep capabilities returned"))?;
 
     Ok(())
@@ -1412,18 +1394,8 @@ impl LinuxContainer {
             logger: logger.new(o!("module" => "rustjail", "subsystem" => "container", "cid" => id)),
         })
     }
-
-    fn load<T: Into<String>>(_id: T, _base: T) -> Result<Self> {
-        Err(anyhow!("not supported"))
-    }
 }
 
-// Handle the differing rlimit types for different targets
-#[cfg(target_env = "musl")]
-type RlimitsType = libc::c_int;
-#[cfg(target_env = "gnu")]
-type RlimitsType = libc::__rlimit_resource_t;
-
 fn setgroups(grps: &[libc::gid_t]) -> Result<()> {
     let ret = unsafe { libc::setgroups(grps.len(), grps.as_ptr() as *const libc::gid_t) };
     Errno::result(ret).map(drop)?;
@@ -1480,6 +1452,8 @@ async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
         })
         .collect();
 
+    // Avoid the exit signal to be reaped by the global reaper.
+    let _wait_locker = WAIT_PID_LOCKER.lock().await;
     let mut child = tokio::process::Command::new(path)
         .args(args.iter())
         .envs(env.iter())
@@ -1528,28 +1502,23 @@ async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
 
         match child.wait().await {
             Ok(exit) => {
-                let code = match exit.code() {
-                    Some(c) => c,
-                    None => {
-                        return Err(anyhow!("hook exit status has no status code"));
-                    }
-                };
+                let code = exit
+                    .code()
+                    .ok_or_else(|| anyhow!("hook exit status has no status code"))?;
 
-                if code == 0 {
-                    debug!(logger, "hook {} exit status is 0", &path);
-                    return Ok(());
-                } else {
+                if code != 0 {
                     error!(logger, "hook {} exit status is {}", &path, code);
                     return Err(anyhow!(nix::Error::from_errno(Errno::UnknownErrno)));
                 }
+
+                debug!(logger, "hook {} exit status is 0", &path);
+                Ok(())
             }
-            Err(e) => {
-                return Err(anyhow!(
-                    "wait child error: {} {}",
-                    e,
-                    e.raw_os_error().unwrap()
-                ));
-            }
+            Err(e) => Err(anyhow!(
+                "wait child error: {} {}",
+                e,
+                e.raw_os_error().unwrap()
+            )),
         }
     });
 
@@ -1568,6 +1537,7 @@ mod tests {
     use std::os::unix::fs::MetadataExt;
     use std::os::unix::io::AsRawFd;
     use tempfile::tempdir;
+    use tokio::process::Command;
 
     macro_rules! sl {
         () => {
@@ -1575,12 +1545,27 @@ mod tests {
         };
     }
 
+    async fn which(cmd: &str) -> String {
+        let output: std::process::Output = Command::new("which")
+            .arg(cmd)
+            .output()
+            .await
+            .expect("which command failed to run");
+
+        match String::from_utf8(output.stdout) {
+            Ok(v) => v.trim_end_matches('\n').to_string(),
+            Err(e) => panic!("Invalid UTF-8 sequence: {}", e),
+        }
+    }
+
     #[tokio::test]
     async fn test_execute_hook() {
+        let xargs = which("xargs").await;
+
         execute_hook(
             &slog_scope::logger(),
             &Hook {
-                path: "/usr/bin/xargs".to_string(),
+                path: xargs,
                 args: vec![],
                 env: vec![],
                 timeout: None,
@@ -1588,7 +1573,7 @@ mod tests {
             &OCIState {
                 version: "1.2.3".to_string(),
                 id: "321".to_string(),
-                status: ContainerState::RUNNING,
+                status: ContainerState::Running,
                 pid: 2,
                 bundle: "".to_string(),
                 annotations: Default::default(),
@@ -1600,10 +1585,12 @@ mod tests {
 
     #[tokio::test]
     async fn test_execute_hook_with_timeout() {
+        let sleep = which("sleep").await;
+
         let res = execute_hook(
             &slog_scope::logger(),
             &Hook {
-                path: "/usr/bin/sleep".to_string(),
+                path: sleep,
                 args: vec!["2".to_string()],
                 env: vec![],
                 timeout: Some(1),
@@ -1611,7 +1598,7 @@ mod tests {
             &OCIState {
                 version: "1.2.3".to_string(),
                 id: "321".to_string(),
-                status: ContainerState::RUNNING,
+                status: ContainerState::Running,
                 pid: 2,
                 bundle: "".to_string(),
                 annotations: Default::default(),
@@ -1630,17 +1617,17 @@ mod tests {
     fn test_status_transtition() {
         let mut status = ContainerStatus::new();
         let status_table: [ContainerState; 4] = [
-            ContainerState::CREATED,
-            ContainerState::RUNNING,
-            ContainerState::PAUSED,
-            ContainerState::STOPPED,
+            ContainerState::Created,
+            ContainerState::Running,
+            ContainerState::Paused,
+            ContainerState::Stopped,
         ];
 
         for s in status_table.iter() {
             let pre_status = status.status();
             status.transition(*s);
 
-            assert_eq!(pre_status, status.pre_status());
+            assert_eq!(pre_status, status.pre_status);
         }
     }
 
@@ -1770,7 +1757,7 @@ mod tests {
     fn test_linuxcontainer_pause_bad_status() {
         let ret = new_linux_container_and_then(|mut c: LinuxContainer| {
             // Change state to pause, c.pause() should fail
-            c.status.transition(ContainerState::PAUSED);
+            c.status.transition(ContainerState::Paused);
             c.pause().map_err(|e| anyhow!(e))
         });
 
@@ -1802,7 +1789,7 @@ mod tests {
     fn test_linuxcontainer_resume_bad_status() {
         let ret = new_linux_container_and_then(|mut c: LinuxContainer| {
             // Change state to created, c.resume() should fail
-            c.status.transition(ContainerState::CREATED);
+            c.status.transition(ContainerState::Created);
             c.resume().map_err(|e| anyhow!(e))
         });
 
@@ -1813,7 +1800,7 @@ mod tests {
     #[test]
     fn test_linuxcontainer_resume_cgroupmgr_is_none() {
         let ret = new_linux_container_and_then(|mut c: LinuxContainer| {
-            c.status.transition(ContainerState::PAUSED);
+            c.status.transition(ContainerState::Paused);
             c.cgroup_manager = None;
             c.resume().map_err(|e| anyhow!(e))
         });
@@ -1826,7 +1813,7 @@ mod tests {
         let ret = new_linux_container_and_then(|mut c: LinuxContainer| {
             c.cgroup_manager = FsManager::new("").ok();
             // Change status to paused, this way we can resume it
-            c.status.transition(ContainerState::PAUSED);
+            c.status.transition(ContainerState::Paused);
             c.resume().map_err(|e| anyhow!(e))
         });
 

src/agent/rustjail/src/lib.rs

@@ -3,15 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-// #![allow(unused_attributes)]
-// #![allow(unused_imports)]
-// #![allow(unused_variables)]
-// #![allow(unused_mut)]
-#![allow(dead_code)]
-// #![allow(deprecated)]
-// #![allow(unused_must_use)]
 #![allow(non_upper_case_globals)]
-// #![allow(unused_comparisons)]
 #[macro_use]
 #[cfg(test)]
 extern crate serial_test;
@@ -23,7 +15,7 @@ extern crate caps;
 extern crate protocols;
 #[macro_use]
 extern crate scopeguard;
-extern crate prctl;
+extern crate capctl;
 #[macro_use]
 extern crate lazy_static;
 extern crate libc;
@@ -47,35 +39,17 @@ pub mod sync;
 pub mod sync_with_async;
 pub mod utils;
 pub mod validator;
-// pub mod factory;
-//pub mod configs;
-// pub mod devices;
-// pub mod init;
-// pub mod rootfs;
-// pub mod capabilities;
-// pub mod console;
-// pub mod stats;
-// pub mod user;
-//pub mod intelrdt;
 
-// construtc ociSpec from grpcSpec, which is needed for hook
-// execution. since hooks read config.json
-
-use oci::{
-    Box as ociBox, Hooks as ociHooks, Linux as ociLinux, LinuxCapabilities as ociLinuxCapabilities,
-    Mount as ociMount, POSIXRlimit as ociPOSIXRlimit, Process as ociProcess, Root as ociRoot,
-    Spec as ociSpec, User as ociUser,
-};
-use protocols::oci::{
-    Hooks as grpcHooks, Linux as grpcLinux, Mount as grpcMount, Process as grpcProcess,
-    Root as grpcRoot, Spec as grpcSpec,
-};
 use std::collections::HashMap;
 
-pub fn process_grpc_to_oci(p: &grpcProcess) -> ociProcess {
+use protocols::oci as grpc;
+
+// construct ociSpec from grpc::Spec, which is needed for hook
+// execution. since hooks read config.json
+pub fn process_grpc_to_oci(p: &grpc::Process) -> oci::Process {
     let console_size = if p.ConsoleSize.is_some() {
         let c = p.ConsoleSize.as_ref().unwrap();
-        Some(ociBox {
+        Some(oci::Box {
             height: c.Height,
             width: c.Width,
         })
@@ -85,14 +59,14 @@ pub fn process_grpc_to_oci(p: &grpcProcess) -> ociProcess {
 
     let user = if p.User.is_some() {
         let u = p.User.as_ref().unwrap();
-        ociUser {
+        oci::User {
             uid: u.UID,
             gid: u.GID,
             additional_gids: u.AdditionalGids.clone(),
             username: u.Username.clone(),
         }
     } else {
-        ociUser {
+        oci::User {
             uid: 0,
             gid: 0,
             additional_gids: vec![],
@@ -103,7 +77,7 @@ pub fn process_grpc_to_oci(p: &grpcProcess) -> ociProcess {
     let capabilities = if p.Capabilities.is_some() {
         let cap = p.Capabilities.as_ref().unwrap();
 
-        Some(ociLinuxCapabilities {
+        Some(oci::LinuxCapabilities {
             bounding: cap.Bounding.clone().into_vec(),
             effective: cap.Effective.clone().into_vec(),
             inheritable: cap.Inheritable.clone().into_vec(),
@@ -117,7 +91,7 @@ pub fn process_grpc_to_oci(p: &grpcProcess) -> ociProcess {
     let rlimits = {
         let mut r = Vec::new();
         for lm in p.Rlimits.iter() {
-            r.push(ociPOSIXRlimit {
+            r.push(oci::PosixRlimit {
                 r#type: lm.Type.clone(),
                 hard: lm.Hard,
                 soft: lm.Soft,
@@ -126,7 +100,7 @@ pub fn process_grpc_to_oci(p: &grpcProcess) -> ociProcess {
         r
     };
 
-    ociProcess {
+    oci::Process {
         terminal: p.Terminal,
         console_size,
         user,
@@ -142,15 +116,15 @@ pub fn process_grpc_to_oci(p: &grpcProcess) -> ociProcess {
     }
 }
 
-fn root_grpc_to_oci(root: &grpcRoot) -> ociRoot {
-    ociRoot {
+fn root_grpc_to_oci(root: &grpc::Root) -> oci::Root {
+    oci::Root {
         path: root.Path.clone(),
         readonly: root.Readonly,
     }
 }
 
-fn mount_grpc_to_oci(m: &grpcMount) -> ociMount {
-    ociMount {
+fn mount_grpc_to_oci(m: &grpc::Mount) -> oci::Mount {
+    oci::Mount {
         destination: m.destination.clone(),
         r#type: m.field_type.clone(),
         source: m.source.clone(),
@@ -158,13 +132,12 @@ fn mount_grpc_to_oci(m: &grpcMount) -> ociMount {
     }
 }
 
-use oci::Hook as ociHook;
 use protocols::oci::Hook as grpcHook;
 
-fn hook_grpc_to_oci(h: &[grpcHook]) -> Vec<ociHook> {
+fn hook_grpc_to_oci(h: &[grpcHook]) -> Vec<oci::Hook> {
     let mut r = Vec::new();
     for e in h.iter() {
-        r.push(ociHook {
+        r.push(oci::Hook {
             path: e.Path.clone(),
             args: e.Args.clone().into_vec(),
             env: e.Env.clone().into_vec(),
@@ -174,39 +147,29 @@ fn hook_grpc_to_oci(h: &[grpcHook]) -> Vec<ociHook> {
     r
 }
 
-fn hooks_grpc_to_oci(h: &grpcHooks) -> ociHooks {
+fn hooks_grpc_to_oci(h: &grpc::Hooks) -> oci::Hooks {
     let prestart = hook_grpc_to_oci(h.Prestart.as_ref());
 
     let poststart = hook_grpc_to_oci(h.Poststart.as_ref());
 
     let poststop = hook_grpc_to_oci(h.Poststop.as_ref());
 
-    ociHooks {
+    oci::Hooks {
         prestart,
         poststart,
         poststop,
     }
 }
 
-use oci::{
-    LinuxDevice as ociLinuxDevice, LinuxIDMapping as ociLinuxIDMapping,
-    LinuxIntelRdt as ociLinuxIntelRdt, LinuxNamespace as ociLinuxNamespace,
-    LinuxResources as ociLinuxResources, LinuxSeccomp as ociLinuxSeccomp,
-};
-use protocols::oci::{
-    LinuxIDMapping as grpcLinuxIDMapping, LinuxResources as grpcLinuxResources,
-    LinuxSeccomp as grpcLinuxSeccomp,
-};
-
-fn idmap_grpc_to_oci(im: &grpcLinuxIDMapping) -> ociLinuxIDMapping {
-    ociLinuxIDMapping {
+fn idmap_grpc_to_oci(im: &grpc::LinuxIDMapping) -> oci::LinuxIdMapping {
+    oci::LinuxIdMapping {
         container_id: im.ContainerID,
         host_id: im.HostID,
         size: im.Size,
     }
 }
 
-fn idmaps_grpc_to_oci(ims: &[grpcLinuxIDMapping]) -> Vec<ociLinuxIDMapping> {
+fn idmaps_grpc_to_oci(ims: &[grpc::LinuxIDMapping]) -> Vec<oci::LinuxIdMapping> {
     let mut r = Vec::new();
     for im in ims.iter() {
         r.push(idmap_grpc_to_oci(im));
@@ -214,24 +177,13 @@ fn idmaps_grpc_to_oci(ims: &[grpcLinuxIDMapping]) -> Vec<ociLinuxIDMapping> {
     r
 }
 
-use oci::{
-    LinuxBlockIO as ociLinuxBlockIO, LinuxBlockIODevice as ociLinuxBlockIODevice,
-    LinuxCPU as ociLinuxCPU, LinuxDeviceCgroup as ociLinuxDeviceCgroup,
-    LinuxHugepageLimit as ociLinuxHugepageLimit,
-    LinuxInterfacePriority as ociLinuxInterfacePriority, LinuxMemory as ociLinuxMemory,
-    LinuxNetwork as ociLinuxNetwork, LinuxPids as ociLinuxPids,
-    LinuxThrottleDevice as ociLinuxThrottleDevice, LinuxWeightDevice as ociLinuxWeightDevice,
-};
-use protocols::oci::{
-    LinuxBlockIO as grpcLinuxBlockIO, LinuxThrottleDevice as grpcLinuxThrottleDevice,
-    LinuxWeightDevice as grpcLinuxWeightDevice,
-};
-
-fn throttle_devices_grpc_to_oci(tds: &[grpcLinuxThrottleDevice]) -> Vec<ociLinuxThrottleDevice> {
+fn throttle_devices_grpc_to_oci(
+    tds: &[grpc::LinuxThrottleDevice],
+) -> Vec<oci::LinuxThrottleDevice> {
     let mut r = Vec::new();
     for td in tds.iter() {
-        r.push(ociLinuxThrottleDevice {
-            blk: ociLinuxBlockIODevice {
+        r.push(oci::LinuxThrottleDevice {
+            blk: oci::LinuxBlockIoDevice {
                 major: td.Major,
                 minor: td.Minor,
             },
@@ -241,11 +193,11 @@ fn throttle_devices_grpc_to_oci(tds: &[grpcLinuxThrottleDevice]) -> Vec<ociLinux
     r
 }
 
-fn weight_devices_grpc_to_oci(wds: &[grpcLinuxWeightDevice]) -> Vec<ociLinuxWeightDevice> {
+fn weight_devices_grpc_to_oci(wds: &[grpc::LinuxWeightDevice]) -> Vec<oci::LinuxWeightDevice> {
     let mut r = Vec::new();
     for wd in wds.iter() {
-        r.push(ociLinuxWeightDevice {
-            blk: ociLinuxBlockIODevice {
+        r.push(oci::LinuxWeightDevice {
+            blk: oci::LinuxBlockIoDevice {
                 major: wd.Major,
                 minor: wd.Minor,
             },
@@ -256,7 +208,7 @@ fn weight_devices_grpc_to_oci(wds: &[grpcLinuxWeightDevice]) -> Vec<ociLinuxWeig
     r
 }
 
-fn blockio_grpc_to_oci(blk: &grpcLinuxBlockIO) -> ociLinuxBlockIO {
+fn blockio_grpc_to_oci(blk: &grpc::LinuxBlockIO) -> oci::LinuxBlockIo {
     let weight_device = weight_devices_grpc_to_oci(blk.WeightDevice.as_ref());
     let throttle_read_bps_device = throttle_devices_grpc_to_oci(blk.ThrottleReadBpsDevice.as_ref());
     let throttle_write_bps_device =
@@ -266,7 +218,7 @@ fn blockio_grpc_to_oci(blk: &grpcLinuxBlockIO) -> ociLinuxBlockIO {
     let throttle_write_iops_device =
         throttle_devices_grpc_to_oci(blk.ThrottleWriteIOPSDevice.as_ref());
 
-    ociLinuxBlockIO {
+    oci::LinuxBlockIo {
         weight: Some(blk.Weight as u16),
         leaf_weight: Some(blk.LeafWeight as u16),
         weight_device,
@@ -277,7 +229,7 @@ fn blockio_grpc_to_oci(blk: &grpcLinuxBlockIO) -> ociLinuxBlockIO {
     }
 }
 
-pub fn resources_grpc_to_oci(res: &grpcLinuxResources) -> ociLinuxResources {
+pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources {
     let devices = {
         let mut d = Vec::new();
         for dev in res.Devices.iter() {
@@ -292,7 +244,7 @@ pub fn resources_grpc_to_oci(res: &grpcLinuxResources) -> ociLinuxResources {
             } else {
                 Some(dev.Minor)
             };
-            d.push(ociLinuxDeviceCgroup {
+            d.push(oci::LinuxDeviceCgroup {
                 allow: dev.Allow,
                 r#type: dev.Type.clone(),
                 major,
@@ -305,7 +257,7 @@ pub fn resources_grpc_to_oci(res: &grpcLinuxResources) -> ociLinuxResources {
 
     let memory = if res.Memory.is_some() {
         let mem = res.Memory.as_ref().unwrap();
-        Some(ociLinuxMemory {
+        Some(oci::LinuxMemory {
             limit: Some(mem.Limit),
             reservation: Some(mem.Reservation),
             swap: Some(mem.Swap),
@@ -320,7 +272,7 @@ pub fn resources_grpc_to_oci(res: &grpcLinuxResources) -> ociLinuxResources {
 
     let cpu = if res.CPU.is_some() {
         let c = res.CPU.as_ref().unwrap();
-        Some(ociLinuxCPU {
+        Some(oci::LinuxCpu {
             shares: Some(c.Shares),
             quota: Some(c.Quota),
             period: Some(c.Period),
@@ -335,7 +287,7 @@ pub fn resources_grpc_to_oci(res: &grpcLinuxResources) -> ociLinuxResources {
 
     let pids = if res.Pids.is_some() {
         let p = res.Pids.as_ref().unwrap();
-        Some(ociLinuxPids { limit: p.Limit })
+        Some(oci::LinuxPids { limit: p.Limit })
     } else {
         None
     };
@@ -351,7 +303,7 @@ pub fn resources_grpc_to_oci(res: &grpcLinuxResources) -> ociLinuxResources {
     let hugepage_limits = {
         let mut r = Vec::new();
         for hl in res.HugepageLimits.iter() {
-            r.push(ociLinuxHugepageLimit {
+            r.push(oci::LinuxHugepageLimit {
                 page_size: hl.Pagesize.clone(),
                 limit: hl.Limit,
             });
@@ -364,14 +316,14 @@ pub fn resources_grpc_to_oci(res: &grpcLinuxResources) -> ociLinuxResources {
         let priorities = {
             let mut r = Vec::new();
             for pr in net.Priorities.iter() {
-                r.push(ociLinuxInterfacePriority {
+                r.push(oci::LinuxInterfacePriority {
                     name: pr.Name.clone(),
                     priority: pr.Priority,
                 });
             }
             r
         };
-        Some(ociLinuxNetwork {
+        Some(oci::LinuxNetwork {
             class_id: Some(net.ClassID),
             priorities,
         })
@@ -379,7 +331,7 @@ pub fn resources_grpc_to_oci(res: &grpcLinuxResources) -> ociLinuxResources {
         None
     };
 
-    ociLinuxResources {
+    oci::LinuxResources {
         devices,
         memory,
         cpu,
@@ -391,17 +343,22 @@ pub fn resources_grpc_to_oci(res: &grpcLinuxResources) -> ociLinuxResources {
     }
 }
 
-use oci::{LinuxSeccompArg as ociLinuxSeccompArg, LinuxSyscall as ociLinuxSyscall};
-
-fn seccomp_grpc_to_oci(sec: &grpcLinuxSeccomp) -> ociLinuxSeccomp {
+fn seccomp_grpc_to_oci(sec: &grpc::LinuxSeccomp) -> oci::LinuxSeccomp {
     let syscalls = {
         let mut r = Vec::new();
 
         for sys in sec.Syscalls.iter() {
             let mut args = Vec::new();
+            let errno_ret: u32;
+
+            if sys.has_errnoret() {
+                errno_ret = sys.get_errnoret();
+            } else {
+                errno_ret = libc::EPERM as u32;
+            }
 
             for arg in sys.Args.iter() {
-                args.push(ociLinuxSeccompArg {
+                args.push(oci::LinuxSeccompArg {
                     index: arg.Index as u32,
                     value: arg.Value,
                     value_two: arg.ValueTwo,
@@ -409,23 +366,25 @@ fn seccomp_grpc_to_oci(sec: &grpcLinuxSeccomp) -> ociLinuxSeccomp {
                 });
             }
 
-            r.push(ociLinuxSyscall {
+            r.push(oci::LinuxSyscall {
                 names: sys.Names.clone().into_vec(),
                 action: sys.Action.clone(),
+                errno_ret,
                 args,
             });
         }
         r
     };
 
-    ociLinuxSeccomp {
+    oci::LinuxSeccomp {
         default_action: sec.DefaultAction.clone(),
         architectures: sec.Architectures.clone().into_vec(),
+        flags: sec.Flags.clone().into_vec(),
         syscalls,
     }
 }
 
-fn linux_grpc_to_oci(l: &grpcLinux) -> ociLinux {
+fn linux_grpc_to_oci(l: &grpc::Linux) -> oci::Linux {
     let uid_mappings = idmaps_grpc_to_oci(l.UIDMappings.as_ref());
     let gid_mappings = idmaps_grpc_to_oci(l.GIDMappings.as_ref());
 
@@ -445,7 +404,7 @@ fn linux_grpc_to_oci(l: &grpcLinux) -> ociLinux {
         let mut r = Vec::new();
 
         for ns in l.Namespaces.iter() {
-            r.push(ociLinuxNamespace {
+            r.push(oci::LinuxNamespace {
                 r#type: ns.Type.clone(),
                 path: ns.Path.clone(),
             });
@@ -457,7 +416,7 @@ fn linux_grpc_to_oci(l: &grpcLinux) -> ociLinux {
         let mut r = Vec::new();
 
         for d in l.Devices.iter() {
-            r.push(ociLinuxDevice {
+            r.push(oci::LinuxDevice {
                 path: d.Path.clone(),
                 r#type: d.Type.clone(),
                 major: d.Major,
@@ -473,14 +432,14 @@ fn linux_grpc_to_oci(l: &grpcLinux) -> ociLinux {
     let intel_rdt = if l.IntelRdt.is_some() {
         let rdt = l.IntelRdt.as_ref().unwrap();
 
-        Some(ociLinuxIntelRdt {
+        Some(oci::LinuxIntelRdt {
             l3_cache_schema: rdt.L3CacheSchema.clone(),
         })
     } else {
         None
     };
 
-    ociLinux {
+    oci::Linux {
         uid_mappings,
         gid_mappings,
         sysctl: l.Sysctl.clone(),
@@ -497,11 +456,7 @@ fn linux_grpc_to_oci(l: &grpcLinux) -> ociLinux {
     }
 }
 
-fn linux_oci_to_grpc(_l: &ociLinux) -> grpcLinux {
-    grpcLinux::default()
-}
-
-pub fn grpc_to_oci(grpc: &grpcSpec) -> ociSpec {
+pub fn grpc_to_oci(grpc: &grpc::Spec) -> oci::Spec {
     // process
     let process = if grpc.Process.is_some() {
         Some(process_grpc_to_oci(grpc.Process.as_ref().unwrap()))
@@ -539,7 +494,7 @@ pub fn grpc_to_oci(grpc: &grpcSpec) -> ociSpec {
         None
     };
 
-    ociSpec {
+    oci::Spec {
         version: grpc.Version.clone(),
         process,
         root,
@@ -556,7 +511,6 @@ pub fn grpc_to_oci(grpc: &grpcSpec) -> ociSpec {
 
 #[cfg(test)]
 mod tests {
-    #[allow(unused_macros)]
     #[macro_export]
     macro_rules! skip_if_not_root {
         () => {

src/agent/rustjail/src/mount.rs

@@ -52,66 +52,66 @@ const MOUNTINFOFORMAT: &str = "{d} {d} {d}:{d} {} {} {} {}";
 const PROC_PATH: &str = "/proc";
 
 // since libc didn't defined this const for musl, thus redefined it here.
-#[cfg(all(target_os = "linux", target_env = "gnu"))]
+#[cfg(all(target_os = "linux", target_env = "gnu", not(target_arch = "s390x")))]
 const PROC_SUPER_MAGIC: libc::c_long = 0x00009fa0;
 #[cfg(all(target_os = "linux", target_env = "musl"))]
 const PROC_SUPER_MAGIC: libc::c_ulong = 0x00009fa0;
+#[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "s390x"))]
+const PROC_SUPER_MAGIC: libc::c_uint = 0x00009fa0;
 
 lazy_static! {
     static ref PROPAGATION: HashMap<&'static str, MsFlags> = {
         let mut m = HashMap::new();
-        m.insert("shared", MsFlags::MS_SHARED);
-        m.insert("rshared", MsFlags::MS_SHARED | MsFlags::MS_REC);
         m.insert("private", MsFlags::MS_PRIVATE);
         m.insert("rprivate", MsFlags::MS_PRIVATE | MsFlags::MS_REC);
-        m.insert("slave", MsFlags::MS_SLAVE);
+        m.insert("rshared", MsFlags::MS_SHARED | MsFlags::MS_REC);
         m.insert("rslave", MsFlags::MS_SLAVE | MsFlags::MS_REC);
+        m.insert("runbindable", MsFlags::MS_UNBINDABLE | MsFlags::MS_REC);
+        m.insert("shared", MsFlags::MS_SHARED);
+        m.insert("slave", MsFlags::MS_SLAVE);
+        m.insert("unbindable", MsFlags::MS_UNBINDABLE);
         m
     };
     static ref OPTIONS: HashMap<&'static str, (bool, MsFlags)> = {
         let mut m = HashMap::new();
+        m.insert("acl", (false, MsFlags::MS_POSIXACL));
+        m.insert("async", (true, MsFlags::MS_SYNCHRONOUS));
+        m.insert("atime", (true, MsFlags::MS_NOATIME));
+        m.insert("bind", (false, MsFlags::MS_BIND));
         m.insert("defaults", (false, MsFlags::empty()));
+        m.insert("dev", (true, MsFlags::MS_NODEV));
+        m.insert("diratime", (true, MsFlags::MS_NODIRATIME));
+        m.insert("dirsync", (false, MsFlags::MS_DIRSYNC));
+        m.insert("exec", (true, MsFlags::MS_NOEXEC));
+        m.insert("iversion", (false, MsFlags::MS_I_VERSION));
+        m.insert("lazytime", (false, MsFlags::MS_LAZYTIME));
+        m.insert("loud", (true, MsFlags::MS_SILENT));
+        m.insert("mand", (false, MsFlags::MS_MANDLOCK));
+        m.insert("noacl", (true, MsFlags::MS_POSIXACL));
+        m.insert("noatime", (false, MsFlags::MS_NOATIME));
+        m.insert("nodev", (false, MsFlags::MS_NODEV));
+        m.insert("nodiratime", (false, MsFlags::MS_NODIRATIME));
+        m.insert("noexec", (false, MsFlags::MS_NOEXEC));
+        m.insert("noiversion", (true, MsFlags::MS_I_VERSION));
+        m.insert("nolazytime", (true, MsFlags::MS_LAZYTIME));
+        m.insert("nomand", (true, MsFlags::MS_MANDLOCK));
+        m.insert("norelatime", (true, MsFlags::MS_RELATIME));
+        m.insert("nostrictatime", (true, MsFlags::MS_STRICTATIME));
+        m.insert("nosuid", (false, MsFlags::MS_NOSUID));
+        m.insert("rbind", (false, MsFlags::MS_BIND | MsFlags::MS_REC));
+        m.insert("relatime", (false, MsFlags::MS_RELATIME));
+        m.insert("remount", (false, MsFlags::MS_REMOUNT));
         m.insert("ro", (false, MsFlags::MS_RDONLY));
         m.insert("rw", (true, MsFlags::MS_RDONLY));
-        m.insert("suid", (true, MsFlags::MS_NOSUID));
-        m.insert("nosuid", (false, MsFlags::MS_NOSUID));
-        m.insert("dev", (true, MsFlags::MS_NODEV));
-        m.insert("nodev", (false, MsFlags::MS_NODEV));
-        m.insert("exec", (true, MsFlags::MS_NOEXEC));
-        m.insert("noexec", (false, MsFlags::MS_NOEXEC));
-        m.insert("sync", (false, MsFlags::MS_SYNCHRONOUS));
-        m.insert("async", (true, MsFlags::MS_SYNCHRONOUS));
-        m.insert("dirsync", (false, MsFlags::MS_DIRSYNC));
-        m.insert("remount", (false, MsFlags::MS_REMOUNT));
-        m.insert("mand", (false, MsFlags::MS_MANDLOCK));
-        m.insert("nomand", (true, MsFlags::MS_MANDLOCK));
-        m.insert("atime", (true, MsFlags::MS_NOATIME));
-        m.insert("noatime", (false, MsFlags::MS_NOATIME));
-        m.insert("diratime", (true, MsFlags::MS_NODIRATIME));
-        m.insert("nodiratime", (false, MsFlags::MS_NODIRATIME));
-        m.insert("bind", (false, MsFlags::MS_BIND));
-        m.insert("rbind", (false, MsFlags::MS_BIND | MsFlags::MS_REC));
-        m.insert("unbindable", (false, MsFlags::MS_UNBINDABLE));
-        m.insert(
-            "runbindable",
-            (false, MsFlags::MS_UNBINDABLE | MsFlags::MS_REC),
-        );
-        m.insert("private", (false, MsFlags::MS_PRIVATE));
-        m.insert("rprivate", (false, MsFlags::MS_PRIVATE | MsFlags::MS_REC));
-        m.insert("shared", (false, MsFlags::MS_SHARED));
-        m.insert("rshared", (false, MsFlags::MS_SHARED | MsFlags::MS_REC));
-        m.insert("slave", (false, MsFlags::MS_SLAVE));
-        m.insert("rslave", (false, MsFlags::MS_SLAVE | MsFlags::MS_REC));
-        m.insert("relatime", (false, MsFlags::MS_RELATIME));
-        m.insert("norelatime", (true, MsFlags::MS_RELATIME));
+        m.insert("silent", (false, MsFlags::MS_SILENT));
         m.insert("strictatime", (false, MsFlags::MS_STRICTATIME));
-        m.insert("nostrictatime", (true, MsFlags::MS_STRICTATIME));
+        m.insert("suid", (true, MsFlags::MS_NOSUID));
+        m.insert("sync", (false, MsFlags::MS_SYNCHRONOUS));
         m
     };
 }
 
 #[inline(always)]
-#[allow(unused_variables)]
 pub fn mount<
     P1: ?Sized + NixPath,
     P2: ?Sized + NixPath,
@@ -131,7 +131,6 @@ pub fn mount<
 }
 
 #[inline(always)]
-#[allow(unused_variables)]
 pub fn umount2<P: ?Sized + NixPath>(
     target: &P,
     flags: MntFlags,
@@ -190,7 +189,7 @@ pub fn init_rootfs(
 
     let mut bind_mount_dev = false;
     for m in &spec.mounts {
-        let (mut flags, data) = parse_mount(&m);
+        let (mut flags, pgflags, data) = parse_mount(&m);
         if !m.destination.starts_with('/') || m.destination.contains("..") {
             return Err(anyhow!(
                 "the mount destination {} is invalid",
@@ -232,13 +231,15 @@ pub fn init_rootfs(
             // effective.
             // first check that we have non-default options required before attempting a
             // remount
-            if m.r#type == "bind" {
-                for o in &m.options {
-                    if let Some(fl) = PROPAGATION.get(o.as_str()) {
-                        let dest = secure_join(rootfs, &m.destination);
-                        mount(None::<&str>, dest.as_str(), None::<&str>, *fl, None::<&str>)?;
-                    }
-                }
+            if m.r#type == "bind" && !pgflags.is_empty() {
+                let dest = secure_join(rootfs, &m.destination);
+                mount(
+                    None::<&str>,
+                    dest.as_str(),
+                    None::<&str>,
+                    pgflags,
+                    None::<&str>,
+                )?;
             }
         }
     }
@@ -449,7 +450,6 @@ fn mount_cgroups(
     Ok(())
 }
 
-#[allow(unused_variables)]
 fn pivot_root<P1: ?Sized + NixPath, P2: ?Sized + NixPath>(
     new_root: &P1,
     put_old: &P2,
@@ -582,7 +582,6 @@ fn parse_mount_table() -> Result<Vec<Info>> {
 }
 
 #[inline(always)]
-#[allow(unused_variables)]
 fn chroot<P: ?Sized + NixPath>(path: &P) -> Result<(), nix::Error> {
     #[cfg(not(test))]
     return unistd::chroot(path);
@@ -655,26 +654,27 @@ pub fn ms_move_root(rootfs: &str) -> Result<bool> {
     Ok(true)
 }
 
-fn parse_mount(m: &Mount) -> (MsFlags, String) {
+fn parse_mount(m: &Mount) -> (MsFlags, MsFlags, String) {
     let mut flags = MsFlags::empty();
+    let mut pgflags = MsFlags::empty();
     let mut data = Vec::new();
 
     for o in &m.options {
-        match OPTIONS.get(o.as_str()) {
-            Some(v) => {
-                let (clear, fl) = *v;
-                if clear {
-                    flags &= !fl;
-                } else {
-                    flags |= fl;
-                }
+        if let Some(v) = OPTIONS.get(o.as_str()) {
+            let (clear, fl) = *v;
+            if clear {
+                flags &= !fl;
+            } else {
+                flags |= fl;
             }
-
-            None => data.push(o.clone()),
+        } else if let Some(fl) = PROPAGATION.get(o.as_str()) {
+            pgflags |= *fl;
+        } else {
+            data.push(o.clone());
         }
     }
 
-    (flags, data.join(","))
+    (flags, pgflags, data.join(","))
 }
 
 // This function constructs a canonicalized path by combining the `rootfs` and `unsafe_path` elements.
@@ -920,7 +920,7 @@ pub fn finish_rootfs(cfd_log: RawFd, spec: &Spec) -> Result<()> {
 
     for m in spec.mounts.iter() {
         if m.destination == "/dev" {
-            let (flags, _) = parse_mount(m);
+            let (flags, _, _) = parse_mount(m);
             if flags.contains(MsFlags::MS_RDONLY) {
                 mount(
                     Some("/dev"),
@@ -1365,7 +1365,7 @@ mod tests {
             let msg = format!("{}, result: {:?}", msg, result);
 
             // Perform the checks
-            assert!(result == t.result, msg);
+            assert!(result == t.result, "{}", msg);
         }
     }
 }

src/agent/rustjail/src/pipestream.rs

@@ -77,10 +77,6 @@ impl PipeStream {
         Ok(Self(AsyncFd::new(StreamFd(fd))?))
     }
 
-    pub fn shutdown(&mut self) -> io::Result<()> {
-        self.0.get_mut().close()
-    }
-
     pub fn from_fd(fd: RawFd) -> Self {
         unsafe { Self::from_raw_fd(fd) }
     }
@@ -164,7 +160,44 @@ impl AsyncWrite for PipeStream {
     }
 
     fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
-        self.get_mut().shutdown()?;
+        // Do nothing in shutdown is very important
+        // The only right way to shutdown pipe is drop it
+        // Otherwise PipeStream will conflict with its twins
+        // Because they both have same fd, and both registered.
         Poll::Ready(Ok(()))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use nix::fcntl::OFlag;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+
+    #[tokio::test]
+    // Shutdown should never close the inner fd.
+    async fn test_pipestream_shutdown() {
+        let (_, wfd1) = unistd::pipe2(OFlag::O_CLOEXEC).unwrap();
+        let mut writer1 = PipeStream::new(wfd1).unwrap();
+
+        // if close fd in shutdown, the fd will be reused
+        // and the test will failed
+        let _ = writer1.shutdown().await.unwrap();
+
+        // let _ = unistd::close(wfd1);
+
+        let (rfd2, wfd2) = unistd::pipe2(OFlag::O_CLOEXEC).unwrap(); // reuse fd number, rfd2 == wfd1
+
+        let mut reader2 = PipeStream::new(rfd2).unwrap();
+        let mut writer2 = PipeStream::new(wfd2).unwrap();
+
+        // deregister writer1, then reader2 which has the same fd will be deregistered from epoll
+        drop(writer1);
+
+        let _ = writer2.write(b"1").await;
+
+        let mut content = vec![0u8; 1];
+        // Will Block here if shutdown close the fd.
+        let _ = reader2.read(&mut content).await;
+    }
+}

src/agent/rustjail/src/process.rs

@@ -29,7 +29,6 @@ pub enum StreamType {
     Stdin,
     Stdout,
     Stderr,
-    ExitPipeR,
     TermMaster,
     ParentStdin,
     ParentStdout,
@@ -45,8 +44,8 @@ pub struct Process {
     pub stdin: Option<RawFd>,
     pub stdout: Option<RawFd>,
     pub stderr: Option<RawFd>,
-    pub exit_pipe_r: Option<RawFd>,
-    pub exit_pipe_w: Option<RawFd>,
+    pub exit_tx: Option<tokio::sync::watch::Sender<bool>>,
+    pub exit_rx: Option<tokio::sync::watch::Receiver<bool>>,
     pub extra_files: Vec<File>,
     pub term_master: Option<RawFd>,
     pub tty: bool,
@@ -97,14 +96,15 @@ impl Process {
         pipe_size: i32,
     ) -> Result<Self> {
         let logger = logger.new(o!("subsystem" => "process"));
+        let (exit_tx, exit_rx) = tokio::sync::watch::channel(false);
 
         let mut p = Process {
             exec_id: String::from(id),
             stdin: None,
             stdout: None,
             stderr: None,
-            exit_pipe_w: None,
-            exit_pipe_r: None,
+            exit_tx: Some(exit_tx),
+            exit_rx: Some(exit_rx),
             extra_files: Vec::new(),
             tty: ocip.terminal,
             term_master: None,
@@ -152,7 +152,6 @@ impl Process {
             StreamType::Stdin => self.stdin,
             StreamType::Stdout => self.stdout,
             StreamType::Stderr => self.stderr,
-            StreamType::ExitPipeR => self.exit_pipe_r,
             StreamType::TermMaster => self.term_master,
             StreamType::ParentStdin => self.parent_stdin,
             StreamType::ParentStdout => self.parent_stdout,

src/agent/rustjail/src/sync_with_async.rs

@@ -117,28 +117,20 @@ pub async fn write_async(pipe_w: &mut PipeStream, msg_type: i32, data_str: &str)
     }
 
     match msg_type {
-        SYNC_FAILED => match write_count(pipe_w, data_str.as_bytes(), data_str.len()).await {
-            Ok(_) => pipe_w.shutdown()?,
-            Err(e) => {
-                pipe_w.shutdown()?;
+        SYNC_FAILED => {
+            if let Err(e) = write_count(pipe_w, data_str.as_bytes(), data_str.len()).await {
                 return Err(anyhow!(e).context("error in send message to process"));
             }
-        },
+        }
         SYNC_DATA => {
             let length: i32 = data_str.len() as i32;
             write_count(pipe_w, &length.to_be_bytes(), MSG_SIZE)
                 .await
-                .or_else(|e| {
-                    pipe_w.shutdown()?;
-                    Err(anyhow!(e).context("error in send message to process"))
-                })?;
+                .map_err(|e| anyhow!(e).context("error in send message to process"))?;
 
             write_count(pipe_w, data_str.as_bytes(), data_str.len())
                 .await
-                .or_else(|e| {
-                    pipe_w.shutdown()?;
-                    Err(anyhow!(e).context("error in send message to process"))
-                })?;
+                .map_err(|e| anyhow!(e).context("error in send message to process"))?;
         }
 
         _ => (),

src/agent/rustjail/src/validator.rs

@@ -6,7 +6,7 @@
 use crate::container::Config;
 use anyhow::{anyhow, Context, Error, Result};
 use nix::errno::Errno;
-use oci::{Linux, LinuxIDMapping, LinuxNamespace, Spec};
+use oci::{Linux, LinuxIdMapping, LinuxNamespace, Spec};
 use std::collections::HashMap;
 use std::path::{Component, PathBuf};
 
@@ -28,16 +28,6 @@ fn contain_namespace(nses: &[LinuxNamespace], key: &str) -> bool {
     false
 }
 
-fn get_namespace_path(nses: &[LinuxNamespace], key: &str) -> Result<String> {
-    for ns in nses {
-        if ns.r#type.as_str() == key {
-            return Ok(ns.path.clone());
-        }
-    }
-
-    Err(einval())
-}
-
 fn rootfs(root: &str) -> Result<()> {
     let path = PathBuf::from(root);
     // not absolute path or not exists
@@ -107,7 +97,7 @@ fn security(oci: &Spec) -> Result<()> {
     Ok(())
 }
 
-fn idmapping(maps: &[LinuxIDMapping]) -> Result<()> {
+fn idmapping(maps: &[LinuxIdMapping]) -> Result<()> {
     for map in maps {
         if map.size > 0 {
             return Ok(());
@@ -166,31 +156,6 @@ lazy_static! {
     };
 }
 
-fn check_host_ns(path: &str) -> Result<()> {
-    let cpath = PathBuf::from(path);
-    let hpath = PathBuf::from("/proc/self/ns/net");
-
-    let real_hpath = hpath
-        .read_link()
-        .context(format!("read link {:?}", hpath))?;
-    let meta = cpath
-        .symlink_metadata()
-        .context(format!("symlink metadata {:?}", cpath))?;
-    let file_type = meta.file_type();
-
-    if !file_type.is_symlink() {
-        return Ok(());
-    }
-    let real_cpath = cpath
-        .read_link()
-        .context(format!("read link {:?}", cpath))?;
-    if real_cpath == real_hpath {
-        return Err(einval());
-    }
-
-    Ok(())
-}
-
 fn sysctl(oci: &Spec) -> Result<()> {
     let linux = get_linux(oci)?;
 
@@ -238,7 +203,7 @@ fn rootless_euid_mapping(oci: &Spec) -> Result<()> {
     Ok(())
 }
 
-fn has_idmapping(maps: &[LinuxIDMapping], id: u32) -> bool {
+fn has_idmapping(maps: &[LinuxIdMapping], id: u32) -> bool {
     for map in maps {
         if id >= map.container_id && id < map.container_id + map.size {
             return true;
@@ -334,19 +299,6 @@ mod tests {
         assert_eq!(contain_namespace(&namespaces, ""), false);
         assert_eq!(contain_namespace(&namespaces, "Net"), false);
         assert_eq!(contain_namespace(&namespaces, "ipc"), false);
-
-        assert_eq!(
-            get_namespace_path(&namespaces, "net").unwrap(),
-            "/sys/cgroups/net"
-        );
-        assert_eq!(
-            get_namespace_path(&namespaces, "uts").unwrap(),
-            "/sys/cgroups/uts"
-        );
-
-        get_namespace_path(&namespaces, "").unwrap_err();
-        get_namespace_path(&namespaces, "Uts").unwrap_err();
-        get_namespace_path(&namespaces, "ipc").unwrap_err();
     }
 
     #[test]
@@ -441,7 +393,7 @@ mod tests {
         usernamespace(&spec).unwrap();
 
         let mut linux = Linux::default();
-        linux.uid_mappings = vec![LinuxIDMapping {
+        linux.uid_mappings = vec![LinuxIdMapping {
             container_id: 0,
             host_id: 1000,
             size: 0,
@@ -450,7 +402,7 @@ mod tests {
         usernamespace(&spec).unwrap_err();
 
         let mut linux = Linux::default();
-        linux.uid_mappings = vec![LinuxIDMapping {
+        linux.uid_mappings = vec![LinuxIdMapping {
             container_id: 0,
             host_id: 1000,
             size: 100,
@@ -497,12 +449,12 @@ mod tests {
                 path: "/sys/cgroups/user".to_owned(),
             },
         ];
-        linux.uid_mappings = vec![LinuxIDMapping {
+        linux.uid_mappings = vec![LinuxIdMapping {
             container_id: 0,
             host_id: 1000,
             size: 1000,
         }];
-        linux.gid_mappings = vec![LinuxIDMapping {
+        linux.gid_mappings = vec![LinuxIdMapping {
             container_id: 0,
             host_id: 1000,
             size: 1000,
@@ -528,12 +480,6 @@ mod tests {
         rootless_euid(&spec).unwrap();
     }
 
-    #[test]
-    fn test_check_host_ns() {
-        check_host_ns("/proc/self/ns/net").unwrap_err();
-        check_host_ns("/proc/sys/net/ipv4/tcp_sack").unwrap();
-    }
-
     #[test]
     fn test_sysctl() {
         let mut spec = Spec::default();

src/agent/src/ccw.rs

@@ -0,0 +1,140 @@
+// Copyright (c) IBM Corp. 2021
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::fmt;
+use std::str::FromStr;
+
+use anyhow::anyhow;
+
+// CCW bus ID follow the format <xx>.<d>.<xxxx> [1, p. 11], where
+//   - <xx> is the channel subsystem ID, which is always 0 from the guest side, but different from
+//     the host side, e.g. 0xfe for virtio-*-ccw [1, p. 435],
+//   - <d> is the subchannel set ID, which ranges from 0-3 [2], and
+//   - <xxxx> is the device number (0000-ffff; leading zeroes can be omitted,
+//      e.g. 3 instead of 0003).
+// [1] https://www.ibm.com/docs/en/linuxonibm/pdf/lku4dd04.pdf
+// [2] https://qemu.readthedocs.io/en/latest/system/s390x/css.html
+
+// Maximum subchannel set ID
+const SUBCHANNEL_SET_MAX: u8 = 3;
+
+// CCW device. From the guest side, the first field is always 0 and can therefore be omitted.
+#[derive(Copy, Clone, Debug)]
+pub struct Device {
+    subchannel_set_id: u8,
+    device_number: u16,
+}
+
+impl Device {
+    pub fn new(subchannel_set_id: u8, device_number: u16) -> anyhow::Result<Self> {
+        if subchannel_set_id > SUBCHANNEL_SET_MAX {
+            return Err(anyhow!(
+                "Subchannel set ID {:?} should be in range [0..{}]",
+                subchannel_set_id,
+                SUBCHANNEL_SET_MAX
+            ));
+        }
+
+        Ok(Device {
+            subchannel_set_id,
+            device_number,
+        })
+    }
+}
+
+impl FromStr for Device {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> anyhow::Result<Self> {
+        let split: Vec<&str> = s.split('.').collect();
+        if split.len() != 3 {
+            return Err(anyhow!(
+                "Wrong bus format. It needs to be in the form 0.<d>.<xxxx>, got {:?}",
+                s
+            ));
+        }
+
+        if split[0] != "0" {
+            return Err(anyhow!(
+                "Wrong bus format. First digit needs to be 0, but is {:?}",
+                split[0]
+            ));
+        }
+
+        let subchannel_set_id = match split[1].parse::<u8>() {
+            Ok(id) => id,
+            Err(_) => {
+                return Err(anyhow!(
+                    "Wrong bus format. Second digit needs to be 0-3, but is {:?}",
+                    split[1]
+                ))
+            }
+        };
+
+        let device_number = match u16::from_str_radix(split[2], 16) {
+            Ok(id) => id,
+            Err(_) => {
+                return Err(anyhow!(
+                    "Wrong bus format. Third digit needs to be 0-ffff, but is {:?}",
+                    split[2]
+                ))
+            }
+        };
+
+        Device::new(subchannel_set_id, device_number)
+    }
+}
+
+impl fmt::Display for Device {
+    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+        write!(f, "0.{}.{:04x}", self.subchannel_set_id, self.device_number)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::ccw::Device;
+    use std::str::FromStr;
+
+    #[test]
+    fn test_new_device() {
+        // Valid devices
+        let device = Device::new(0, 0).unwrap();
+        assert_eq!(format!("{}", device), "0.0.0000");
+
+        let device = Device::new(3, 0xffff).unwrap();
+        assert_eq!(format!("{}", device), "0.3.ffff");
+
+        // Invalid device
+        let device = Device::new(4, 0);
+        assert!(device.is_err());
+    }
+
+    #[test]
+    fn test_device_from_str() {
+        // Valid devices
+        let device = Device::from_str("0.0.0").unwrap();
+        assert_eq!(format!("{}", device), "0.0.0000");
+
+        let device = Device::from_str("0.0.0000").unwrap();
+        assert_eq!(format!("{}", device), "0.0.0000");
+
+        let device = Device::from_str("0.3.ffff").unwrap();
+        assert_eq!(format!("{}", device), "0.3.ffff");
+
+        // Invalid devices
+        let device = Device::from_str("0.0");
+        assert!(device.is_err());
+
+        let device = Device::from_str("1.0.0");
+        assert!(device.is_err());
+
+        let device = Device::from_str("0.not_a_subchannel_set_id.0");
+        assert!(device.is_err());
+
+        let device = Device::from_str("0.0.not_a_device_number");
+        assert!(device.is_err());
+    }
+}

src/agent/src/console.rs

@@ -0,0 +1,295 @@
+// Copyright (c) 2021 Ant Group
+// Copyright (c) 2021 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use crate::util;
+use anyhow::{anyhow, Result};
+use nix::fcntl::{self, FcntlArg, FdFlag, OFlag};
+use nix::libc::{STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO};
+use nix::pty::{openpty, OpenptyResult};
+use nix::sys::socket::{self, AddressFamily, SockAddr, SockFlag, SockType};
+use nix::sys::stat::Mode;
+use nix::sys::wait;
+use nix::unistd::{self, close, dup2, fork, setsid, ForkResult, Pid};
+use rustjail::pipestream::PipeStream;
+use slog::Logger;
+use std::ffi::CString;
+use std::os::unix::io::{FromRawFd, RawFd};
+use std::path::PathBuf;
+use std::process::Stdio;
+use std::sync::Arc;
+use std::sync::Mutex as SyncMutex;
+
+use futures::StreamExt;
+use tokio::io::{AsyncRead, AsyncWrite};
+use tokio::select;
+use tokio::sync::watch::Receiver;
+
+const CONSOLE_PATH: &str = "/dev/console";
+
+lazy_static! {
+    static ref SHELLS: Arc<SyncMutex<Vec<String>>> = {
+        let mut v = Vec::new();
+
+        if !cfg!(test) {
+            v.push("/bin/bash".to_string());
+            v.push("/bin/sh".to_string());
+        }
+
+        Arc::new(SyncMutex::new(v))
+    };
+}
+
+pub fn initialize() {
+    lazy_static::initialize(&SHELLS);
+}
+
+pub async fn debug_console_handler(
+    logger: Logger,
+    port: u32,
+    mut shutdown: Receiver<bool>,
+) -> Result<()> {
+    let logger = logger.new(o!("subsystem" => "debug-console"));
+
+    let shells = SHELLS.lock().unwrap().to_vec();
+
+    let shell = shells
+        .into_iter()
+        .find(|sh| PathBuf::from(sh).exists())
+        .ok_or_else(|| anyhow!("no shell found to launch debug console"))?;
+
+    if port > 0 {
+        let listenfd = socket::socket(
+            AddressFamily::Vsock,
+            SockType::Stream,
+            SockFlag::SOCK_CLOEXEC,
+            None,
+        )?;
+        let addr = SockAddr::new_vsock(libc::VMADDR_CID_ANY, port);
+        socket::bind(listenfd, &addr)?;
+        socket::listen(listenfd, 1)?;
+
+        let mut incoming = util::get_vsock_incoming(listenfd);
+
+        loop {
+            select! {
+                _ = shutdown.changed() => {
+                    info!(logger, "debug console got shutdown request");
+                    break;
+                }
+
+                conn = incoming.next() => {
+                    if let Some(conn) = conn {
+                        // Accept a new connection
+                        match conn {
+                            Ok(stream) => {
+                                let logger = logger.clone();
+                                let shell = shell.clone();
+                                // Do not block(await) here, or we'll never receive the shutdown signal
+                                tokio::spawn(async move {
+                                    let _ = run_debug_console_vsock(logger, shell, stream).await;
+                                });
+                            }
+                            Err(e) => {
+                                error!(logger, "{:?}", e);
+                            }
+                        }
+                    } else {
+                        break;
+                    }
+                }
+            }
+        }
+    } else {
+        let mut flags = OFlag::empty();
+        flags.insert(OFlag::O_RDWR);
+        flags.insert(OFlag::O_CLOEXEC);
+
+        let fd = fcntl::open(CONSOLE_PATH, flags, Mode::empty())?;
+
+        select! {
+            _ = shutdown.changed() => {
+                info!(logger, "debug console got shutdown request");
+            }
+
+            result = run_debug_console_serial(shell.clone(), fd) => {
+               match result {
+                   Ok(_) => {
+                       info!(logger, "run_debug_console_shell session finished");
+                   }
+                   Err(err) => {
+                       error!(logger, "run_debug_console_shell failed: {:?}", err);
+                   }
+               }
+            }
+        }
+    };
+
+    Ok(())
+}
+
+fn run_in_child(slave_fd: libc::c_int, shell: String) -> Result<()> {
+    // create new session with child as session leader
+    setsid()?;
+
+    // dup stdin, stdout, stderr to let child act as a terminal
+    dup2(slave_fd, STDIN_FILENO)?;
+    dup2(slave_fd, STDOUT_FILENO)?;
+    dup2(slave_fd, STDERR_FILENO)?;
+
+    // set tty
+    unsafe {
+        libc::ioctl(0, libc::TIOCSCTTY);
+    }
+
+    let cmd = CString::new(shell).unwrap();
+    let args: Vec<CString> = Vec::new();
+
+    // run shell
+    let _ = unistd::execvp(cmd.as_c_str(), &args).map_err(|e| match e {
+        nix::Error::Sys(errno) => {
+            std::process::exit(errno as i32);
+        }
+        _ => std::process::exit(-2),
+    });
+
+    Ok(())
+}
+
+async fn run_in_parent<T: AsyncRead + AsyncWrite>(
+    logger: Logger,
+    stream: T,
+    pseudo: OpenptyResult,
+    child_pid: Pid,
+) -> Result<()> {
+    info!(logger, "get debug shell pid {:?}", child_pid);
+
+    let master_fd = pseudo.master;
+    let _ = close(pseudo.slave);
+
+    let (mut socket_reader, mut socket_writer) = tokio::io::split(stream);
+    let (mut master_reader, mut master_writer) = tokio::io::split(PipeStream::from_fd(master_fd));
+
+    select! {
+        res = tokio::io::copy(&mut master_reader, &mut socket_writer) => {
+            debug!(
+                logger,
+                "master closed: {:?}", res
+            );
+        }
+        res = tokio::io::copy(&mut socket_reader, &mut master_writer) => {
+            info!(
+                logger,
+                "socket closed: {:?}", res
+            );
+        }
+    }
+
+    let wait_status = wait::waitpid(child_pid, None);
+    info!(logger, "debug console process exit code: {:?}", wait_status);
+
+    Ok(())
+}
+
+async fn run_debug_console_vsock<T: AsyncRead + AsyncWrite>(
+    logger: Logger,
+    shell: String,
+    stream: T,
+) -> Result<()> {
+    let logger = logger.new(o!("subsystem" => "debug-console-shell"));
+
+    let pseudo = openpty(None, None)?;
+    let _ = fcntl::fcntl(pseudo.master, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC));
+    let _ = fcntl::fcntl(pseudo.slave, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC));
+
+    let slave_fd = pseudo.slave;
+
+    match unsafe { fork() } {
+        Ok(ForkResult::Child) => run_in_child(slave_fd, shell),
+        Ok(ForkResult::Parent { child: child_pid }) => {
+            run_in_parent(logger.clone(), stream, pseudo, child_pid).await
+        }
+        Err(err) => Err(anyhow!("fork error: {:?}", err)),
+    }
+}
+
+async fn run_debug_console_serial(shell: String, fd: RawFd) -> Result<()> {
+    let mut child = match tokio::process::Command::new(shell)
+        .arg("-i")
+        .kill_on_drop(true)
+        .stdin(unsafe { Stdio::from_raw_fd(fd) })
+        .stdout(unsafe { Stdio::from_raw_fd(fd) })
+        .stderr(unsafe { Stdio::from_raw_fd(fd) })
+        .spawn()
+    {
+        Ok(c) => c,
+        Err(_) => return Err(anyhow!("failed to spawn shell")),
+    };
+
+    child.wait().await?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile::tempdir;
+    use tokio::sync::watch;
+
+    #[tokio::test]
+    async fn test_setup_debug_console_no_shells() {
+        {
+            // Guarantee no shells have been added
+            // (required to avoid racing with
+            // test_setup_debug_console_invalid_shell()).
+            let shells_ref = SHELLS.clone();
+            let mut shells = shells_ref.lock().unwrap();
+            shells.clear();
+        }
+
+        let logger = slog_scope::logger();
+
+        let (_, rx) = watch::channel(true);
+        let result = debug_console_handler(logger, 0, rx).await;
+
+        assert!(result.is_err());
+        assert_eq!(
+            result.unwrap_err().to_string(),
+            "no shell found to launch debug console"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_setup_debug_console_invalid_shell() {
+        {
+            let shells_ref = SHELLS.clone();
+            let mut shells = shells_ref.lock().unwrap();
+
+            let dir = tempdir().expect("failed to create tmpdir");
+
+            // Add an invalid shell
+            let shell = dir
+                .path()
+                .join("enoent")
+                .to_str()
+                .expect("failed to construct shell path")
+                .to_string();
+
+            shells.push(shell);
+        }
+
+        let logger = slog_scope::logger();
+
+        let (_, rx) = watch::channel(true);
+        let result = debug_console_handler(logger, 0, rx).await;
+
+        assert!(result.is_err());
+        assert_eq!(
+            result.unwrap_err().to_string(),
+            "no shell found to launch debug console"
+        );
+    }
+}

src/agent/src/device.rs

@@ -5,6 +5,7 @@
 
 use libc::{c_uint, major, minor};
 use nix::sys::stat;
+use regex::Regex;
 use std::collections::HashMap;
 use std::fs;
 use std::os::unix::fs::MetadataExt;
@@ -13,14 +14,20 @@ use std::str::FromStr;
 use std::sync::Arc;
 use tokio::sync::Mutex;
 
+#[cfg(target_arch = "s390x")]
+use crate::ccw;
 use crate::linux_abi::*;
-use crate::mount::{DRIVER_BLK_TYPE, DRIVER_MMIO_BLK_TYPE, DRIVER_NVDIMM_TYPE, DRIVER_SCSI_TYPE};
+use crate::mount::{
+    DRIVER_BLK_CCW_TYPE, DRIVER_BLK_TYPE, DRIVER_MMIO_BLK_TYPE, DRIVER_NVDIMM_TYPE,
+    DRIVER_SCSI_TYPE,
+};
 use crate::pci;
 use crate::sandbox::Sandbox;
-use crate::{AGENT_CONFIG, GLOBAL_DEVICE_WATCHER};
+use crate::uevent::{wait_for_uevent, Uevent, UeventMatcher};
 use anyhow::{anyhow, Result};
 use oci::{LinuxDeviceCgroup, LinuxResources, Spec};
 use protocols::agent::Device;
+use tracing::instrument;
 
 // Convenience macro to obtain the scope logger
 macro_rules! sl {
@@ -31,17 +38,21 @@ macro_rules! sl {
 
 const VM_ROOTFS: &str = "/";
 
+#[derive(Debug)]
 struct DevIndexEntry {
     idx: usize,
     residx: Vec<usize>,
 }
 
+#[derive(Debug)]
 struct DevIndex(HashMap<String, DevIndexEntry>);
 
+#[instrument]
 pub fn rescan_pci_bus() -> Result<()> {
     online_device(SYSFS_PCI_BUS_RESCAN_FILE)
 }
 
+#[instrument]
 pub fn online_device(path: &str) -> Result<()> {
     fs::write(path, "1")?;
     Ok(())
@@ -50,6 +61,7 @@ pub fn online_device(path: &str) -> Result<()> {
 // pcipath_to_sysfs fetches the sysfs path for a PCI path, relative to
 // the sysfs path for the PCI host bridge, based on the PCI path
 // provided.
+#[instrument]
 fn pcipath_to_sysfs(root_bus_sysfs: &str, pcipath: &pci::Path) -> Result<String> {
     let mut bus = "0000:00".to_string();
     let mut relpath = String::new();
@@ -87,78 +99,164 @@ fn pcipath_to_sysfs(root_bus_sysfs: &str, pcipath: &pci::Path) -> Result<String>
     Ok(relpath)
 }
 
-async fn get_device_name(sandbox: &Arc<Mutex<Sandbox>>, dev_addr: &str) -> Result<String> {
-    // Keep the same lock order as uevent::handle_block_add_event(), otherwise it may cause deadlock.
-    let mut w = GLOBAL_DEVICE_WATCHER.lock().await;
-    let sb = sandbox.lock().await;
-    for (key, value) in sb.pci_device_map.iter() {
-        if key.contains(dev_addr) {
-            info!(sl!(), "Device {} found in pci device map", dev_addr);
-            return Ok(format!("{}/{}", SYSTEM_DEV_PATH, value));
-        }
-    }
-    drop(sb);
-
-    // If device is not found in the device map, hotplug event has not
-    // been received yet, create and add channel to the watchers map.
-    // The key of the watchers map is the device we are interested in.
-    // Note this is done inside the lock, not to miss any events from the
-    // global udev listener.
-    let (tx, rx) = tokio::sync::oneshot::channel::<String>();
-    w.insert(dev_addr.to_string(), Some(tx));
-    drop(w);
-
-    info!(sl!(), "Waiting on channel for device notification\n");
-    let hotplug_timeout = AGENT_CONFIG.read().await.hotplug_timeout;
-
-    let dev_name = match tokio::time::timeout(hotplug_timeout, rx).await {
-        Ok(v) => v?,
-        Err(_) => {
-            let watcher = GLOBAL_DEVICE_WATCHER.clone();
-            let mut w = watcher.lock().await;
-            w.remove_entry(dev_addr);
-
-            return Err(anyhow!(
-                "Timeout reached after {:?} waiting for device {}",
-                hotplug_timeout,
-                dev_addr
-            ));
-        }
-    };
-
-    Ok(format!("{}/{}", SYSTEM_DEV_PATH, &dev_name))
+// FIXME: This matcher is only correct if the guest has at most one
+// SCSI host.
+#[derive(Debug)]
+struct ScsiBlockMatcher {
+    search: String,
 }
 
+impl ScsiBlockMatcher {
+    fn new(scsi_addr: &str) -> ScsiBlockMatcher {
+        let search = format!(r"/0:0:{}/block/", scsi_addr);
+
+        ScsiBlockMatcher { search }
+    }
+}
+
+impl UeventMatcher for ScsiBlockMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.subsystem == "block" && uev.devpath.contains(&self.search) && !uev.devname.is_empty()
+    }
+}
+
+#[instrument]
 pub async fn get_scsi_device_name(
     sandbox: &Arc<Mutex<Sandbox>>,
     scsi_addr: &str,
 ) -> Result<String> {
-    let dev_sub_path = format!("{}{}/{}", SCSI_HOST_CHANNEL, scsi_addr, SCSI_BLOCK_SUFFIX);
+    let matcher = ScsiBlockMatcher::new(scsi_addr);
 
     scan_scsi_bus(scsi_addr)?;
-    get_device_name(sandbox, &dev_sub_path).await
+    let uev = wait_for_uevent(sandbox, matcher).await?;
+    Ok(format!("{}/{}", SYSTEM_DEV_PATH, &uev.devname))
 }
 
-pub async fn get_pci_device_name(
+#[derive(Debug)]
+struct VirtioBlkPciMatcher {
+    rex: Regex,
+}
+
+impl VirtioBlkPciMatcher {
+    fn new(relpath: &str) -> VirtioBlkPciMatcher {
+        let root_bus = create_pci_root_bus_path();
+        let re = format!(r"^{}{}/virtio[0-9]+/block/", root_bus, relpath);
+        VirtioBlkPciMatcher {
+            rex: Regex::new(&re).unwrap(),
+        }
+    }
+}
+
+impl UeventMatcher for VirtioBlkPciMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.subsystem == "block" && self.rex.is_match(&uev.devpath) && !uev.devname.is_empty()
+    }
+}
+
+#[instrument]
+pub async fn get_virtio_blk_pci_device_name(
     sandbox: &Arc<Mutex<Sandbox>>,
     pcipath: &pci::Path,
 ) -> Result<String> {
     let root_bus_sysfs = format!("{}{}", SYSFS_DIR, create_pci_root_bus_path());
     let sysfs_rel_path = pcipath_to_sysfs(&root_bus_sysfs, pcipath)?;
+    let matcher = VirtioBlkPciMatcher::new(&sysfs_rel_path);
 
     rescan_pci_bus()?;
-    get_device_name(sandbox, &sysfs_rel_path).await
+
+    let uev = wait_for_uevent(sandbox, matcher).await?;
+    Ok(format!("{}/{}", SYSTEM_DEV_PATH, &uev.devname))
 }
 
-pub async fn get_pmem_device_name(
+#[cfg(target_arch = "s390x")]
+#[derive(Debug)]
+struct VirtioBlkCCWMatcher {
+    rex: Regex,
+}
+
+#[cfg(target_arch = "s390x")]
+impl VirtioBlkCCWMatcher {
+    fn new(root_bus_path: &str, device: &ccw::Device) -> Self {
+        let re = format!(
+            r"^{}/0\.[0-3]\.[0-9a-f]{{1,4}}/{}/virtio[0-9]+/block/",
+            root_bus_path, device
+        );
+        VirtioBlkCCWMatcher {
+            rex: Regex::new(&re).unwrap(),
+        }
+    }
+}
+
+#[cfg(target_arch = "s390x")]
+impl UeventMatcher for VirtioBlkCCWMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.action == "add" && self.rex.is_match(&uev.devpath) && !uev.devname.is_empty()
+    }
+}
+
+#[cfg(target_arch = "s390x")]
+#[instrument]
+pub async fn get_virtio_blk_ccw_device_name(
     sandbox: &Arc<Mutex<Sandbox>>,
-    pmem_devname: &str,
+    device: &ccw::Device,
 ) -> Result<String> {
-    let dev_sub_path = format!("/{}/{}", SCSI_BLOCK_SUFFIX, pmem_devname);
-    get_device_name(sandbox, &dev_sub_path).await
+    let matcher = VirtioBlkCCWMatcher::new(&create_ccw_root_bus_path(), device);
+    let uev = wait_for_uevent(sandbox, matcher).await?;
+    let devname = uev.devname;
+    return match Path::new(SYSTEM_DEV_PATH).join(&devname).to_str() {
+        Some(path) => Ok(String::from(path)),
+        None => Err(anyhow!("CCW device name {} is not valid UTF-8", &devname)),
+    };
+}
+
+#[derive(Debug)]
+struct PmemBlockMatcher {
+    suffix: String,
+}
+
+impl PmemBlockMatcher {
+    fn new(devname: &str) -> PmemBlockMatcher {
+        let suffix = format!(r"/block/{}", devname);
+
+        PmemBlockMatcher { suffix }
+    }
+}
+
+impl UeventMatcher for PmemBlockMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.subsystem == "block"
+            && uev.devpath.starts_with(ACPI_DEV_PATH)
+            && uev.devpath.ends_with(&self.suffix)
+            && !uev.devname.is_empty()
+    }
+}
+
+#[instrument]
+pub async fn wait_for_pmem_device(sandbox: &Arc<Mutex<Sandbox>>, devpath: &str) -> Result<()> {
+    let devname = match devpath.strip_prefix("/dev/") {
+        Some(dev) => dev,
+        None => {
+            return Err(anyhow!(
+                "Storage source '{}' must start with /dev/",
+                devpath
+            ))
+        }
+    };
+
+    let matcher = PmemBlockMatcher::new(devname);
+    let uev = wait_for_uevent(sandbox, matcher).await?;
+    if uev.devname != devname {
+        return Err(anyhow!(
+            "Unexpected device name {} for pmem device (expected {})",
+            uev.devname,
+            devname
+        ));
+    }
+    Ok(())
 }
 
 /// Scan SCSI bus for the given SCSI address(SCSI-Id and LUN)
+#[instrument]
 fn scan_scsi_bus(scsi_addr: &str) -> Result<()> {
     let tokens: Vec<&str> = scsi_addr.split(':').collect();
     if tokens.len() != 2 {
@@ -193,6 +291,7 @@ fn scan_scsi_bus(scsi_addr: &str) -> Result<()> {
 // the same device in the list of devices provided through the OCI spec.
 // This is needed to update information about minor/major numbers that cannot
 // be predicted from the caller.
+#[instrument]
 fn update_spec_device_list(device: &Device, spec: &mut Spec, devidx: &DevIndex) -> Result<()> {
     let major_id: c_uint;
     let minor_id: c_uint;
@@ -269,6 +368,7 @@ fn update_spec_device_list(device: &Device, spec: &mut Spec, devidx: &DevIndex)
 
 // device.Id should be the predicted device name (vda, vdb, ...)
 // device.VmPath already provides a way to send it in
+#[instrument]
 async fn virtiommio_blk_device_handler(
     device: &Device,
     spec: &mut Spec,
@@ -283,6 +383,7 @@ async fn virtiommio_blk_device_handler(
 }
 
 // device.Id should be a PCI path string
+#[instrument]
 async fn virtio_blk_device_handler(
     device: &Device,
     spec: &mut Spec,
@@ -290,19 +391,41 @@ async fn virtio_blk_device_handler(
     devidx: &DevIndex,
 ) -> Result<()> {
     let mut dev = device.clone();
+    let pcipath = pci::Path::from_str(&device.id)?;
 
-    // When "Id (PCI path)" is not set, we allow to use the predicted
-    // "VmPath" passed from kata-runtime Note this is a special code
-    // path for cloud-hypervisor when BDF information is not available
-    if !device.id.is_empty() {
-        let pcipath = pci::Path::from_str(&device.id)?;
-        dev.vm_path = get_pci_device_name(sandbox, &pcipath).await?;
-    }
+    dev.vm_path = get_virtio_blk_pci_device_name(sandbox, &pcipath).await?;
 
     update_spec_device_list(&dev, spec, devidx)
 }
 
+// device.id should be a CCW path string
+#[cfg(target_arch = "s390x")]
+#[instrument]
+async fn virtio_blk_ccw_device_handler(
+    device: &Device,
+    spec: &mut Spec,
+    sandbox: &Arc<Mutex<Sandbox>>,
+    devidx: &DevIndex,
+) -> Result<()> {
+    let mut dev = device.clone();
+    let ccw_device = ccw::Device::from_str(&device.id)?;
+    dev.vm_path = get_virtio_blk_ccw_device_name(sandbox, &ccw_device).await?;
+    update_spec_device_list(&dev, spec, devidx)
+}
+
+#[cfg(not(target_arch = "s390x"))]
+#[instrument]
+async fn virtio_blk_ccw_device_handler(
+    _: &Device,
+    _: &mut Spec,
+    _: &Arc<Mutex<Sandbox>>,
+    _: &DevIndex,
+) -> Result<()> {
+    Err(anyhow!("CCW is only supported on s390x"))
+}
+
 // device.Id should be the SCSI address of the disk in the format "scsiID:lunID"
+#[instrument]
 async fn virtio_scsi_device_handler(
     device: &Device,
     spec: &mut Spec,
@@ -314,6 +437,7 @@ async fn virtio_scsi_device_handler(
     update_spec_device_list(&dev, spec, devidx)
 }
 
+#[instrument]
 async fn virtio_nvdimm_device_handler(
     device: &Device,
     spec: &mut Spec,
@@ -352,6 +476,7 @@ impl DevIndex {
     }
 }
 
+#[instrument]
 pub async fn add_devices(
     devices: &[Device],
     spec: &mut Spec,
@@ -366,6 +491,7 @@ pub async fn add_devices(
     Ok(())
 }
 
+#[instrument]
 async fn add_device(
     device: &Device,
     spec: &mut Spec,
@@ -390,6 +516,7 @@ async fn add_device(
 
     match device.field_type.as_str() {
         DRIVER_BLK_TYPE => virtio_blk_device_handler(device, spec, sandbox, devidx).await,
+        DRIVER_BLK_CCW_TYPE => virtio_blk_ccw_device_handler(device, spec, sandbox, devidx).await,
         DRIVER_MMIO_BLK_TYPE => virtiommio_blk_device_handler(device, spec, sandbox, devidx).await,
         DRIVER_NVDIMM_TYPE => virtio_nvdimm_device_handler(device, spec, sandbox, devidx).await,
         DRIVER_SCSI_TYPE => virtio_scsi_device_handler(device, spec, sandbox, devidx).await,
@@ -400,6 +527,7 @@ async fn add_device(
 // update_device_cgroup update the device cgroup for container
 // to not allow access to the guest root partition. This prevents
 // the container from being able to access the VM rootfs.
+#[instrument]
 pub fn update_device_cgroup(spec: &mut Spec) -> Result<()> {
     let meta = fs::metadata(VM_ROOTFS)?;
     let rdev = meta.dev();
@@ -430,6 +558,7 @@ pub fn update_device_cgroup(spec: &mut Spec) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::uevent::spawn_test_watcher;
     use oci::Linux;
     use tempfile::tempdir;
 
@@ -776,4 +905,167 @@ mod tests {
         let relpath = pcipath_to_sysfs(rootbuspath, &path234);
         assert_eq!(relpath.unwrap(), "/0000:00:02.0/0000:01:03.0/0000:02:04.0");
     }
+
+    // We use device specific variants of this for real cases, but
+    // they have some complications that make them troublesome to unit
+    // test
+    async fn example_get_device_name(
+        sandbox: &Arc<Mutex<Sandbox>>,
+        relpath: &str,
+    ) -> Result<String> {
+        let matcher = VirtioBlkPciMatcher::new(relpath);
+
+        let uev = wait_for_uevent(sandbox, matcher).await?;
+
+        Ok(uev.devname)
+    }
+
+    #[tokio::test]
+    async fn test_get_device_name() {
+        let devname = "vda";
+        let root_bus = create_pci_root_bus_path();
+        let relpath = "/0000:00:0a.0/0000:03:0b.0";
+        let devpath = format!("{}{}/virtio4/block/{}", root_bus, relpath, devname);
+
+        let mut uev = crate::uevent::Uevent::default();
+        uev.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
+        uev.subsystem = "block".to_string();
+        uev.devpath = devpath.clone();
+        uev.devname = devname.to_string();
+
+        let logger = slog::Logger::root(slog::Discard, o!());
+        let sandbox = Arc::new(Mutex::new(Sandbox::new(&logger).unwrap()));
+
+        let mut sb = sandbox.lock().await;
+        sb.uevent_map.insert(devpath.clone(), uev);
+        drop(sb); // unlock
+
+        let name = example_get_device_name(&sandbox, relpath).await;
+        assert!(name.is_ok(), "{}", name.unwrap_err());
+        assert_eq!(name.unwrap(), devname);
+
+        let mut sb = sandbox.lock().await;
+        let uev = sb.uevent_map.remove(&devpath).unwrap();
+        drop(sb); // unlock
+
+        spawn_test_watcher(sandbox.clone(), uev);
+
+        let name = example_get_device_name(&sandbox, relpath).await;
+        assert!(name.is_ok(), "{}", name.unwrap_err());
+        assert_eq!(name.unwrap(), devname);
+    }
+
+    #[tokio::test]
+    async fn test_virtio_blk_matcher() {
+        let root_bus = create_pci_root_bus_path();
+        let devname = "vda";
+
+        let mut uev_a = crate::uevent::Uevent::default();
+        let relpath_a = "/0000:00:0a.0";
+        uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
+        uev_a.subsystem = "block".to_string();
+        uev_a.devname = devname.to_string();
+        uev_a.devpath = format!("{}{}/virtio4/block/{}", root_bus, relpath_a, devname);
+        let matcher_a = VirtioBlkPciMatcher::new(&relpath_a);
+
+        let mut uev_b = uev_a.clone();
+        let relpath_b = "/0000:00:0a.0/0000:00:0b.0";
+        uev_b.devpath = format!("{}{}/virtio0/block/{}", root_bus, relpath_b, devname);
+        let matcher_b = VirtioBlkPciMatcher::new(&relpath_b);
+
+        assert!(matcher_a.is_match(&uev_a));
+        assert!(matcher_b.is_match(&uev_b));
+        assert!(!matcher_b.is_match(&uev_a));
+        assert!(!matcher_a.is_match(&uev_b));
+    }
+
+    #[cfg(target_arch = "s390x")]
+    #[tokio::test]
+    async fn test_virtio_blk_ccw_matcher() {
+        let root_bus = create_ccw_root_bus_path();
+        let subsystem = "block";
+        let devname = "vda";
+        let relpath = "0.0.0002";
+
+        let mut uev = crate::uevent::Uevent::default();
+        uev.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
+        uev.subsystem = subsystem.to_string();
+        uev.devname = devname.to_string();
+        uev.devpath = format!(
+            "{}/0.0.0001/{}/virtio1/{}/{}",
+            root_bus, relpath, subsystem, devname
+        );
+
+        // Valid path
+        let device = ccw::Device::from_str(relpath).unwrap();
+        let matcher = VirtioBlkCCWMatcher::new(&root_bus, &device);
+        assert!(matcher.is_match(&uev));
+
+        // Invalid paths
+        uev.devpath = format!(
+            "{}/0.0.0001/0.0.0003/virtio1/{}/{}",
+            root_bus, subsystem, devname
+        );
+        assert!(!matcher.is_match(&uev));
+
+        uev.devpath = format!("0.0.0001/{}/virtio1/{}/{}", relpath, subsystem, devname);
+        assert!(!matcher.is_match(&uev));
+
+        uev.devpath = format!(
+            "{}/0.0.0001/{}/virtio/{}/{}",
+            root_bus, relpath, subsystem, devname
+        );
+        assert!(!matcher.is_match(&uev));
+
+        uev.devpath = format!("{}/0.0.0001/{}/virtio1", root_bus, relpath);
+        assert!(!matcher.is_match(&uev));
+
+        uev.devpath = format!(
+            "{}/1.0.0001/{}/virtio1/{}/{}",
+            root_bus, relpath, subsystem, devname
+        );
+        assert!(!matcher.is_match(&uev));
+
+        uev.devpath = format!(
+            "{}/0.4.0001/{}/virtio1/{}/{}",
+            root_bus, relpath, subsystem, devname
+        );
+        assert!(!matcher.is_match(&uev));
+
+        uev.devpath = format!(
+            "{}/0.0.10000/{}/virtio1/{}/{}",
+            root_bus, relpath, subsystem, devname
+        );
+        assert!(!matcher.is_match(&uev));
+    }
+
+    #[tokio::test]
+    async fn test_scsi_block_matcher() {
+        let root_bus = create_pci_root_bus_path();
+        let devname = "sda";
+
+        let mut uev_a = crate::uevent::Uevent::default();
+        let addr_a = "0:0";
+        uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
+        uev_a.subsystem = "block".to_string();
+        uev_a.devname = devname.to_string();
+        uev_a.devpath = format!(
+            "{}/0000:00:00.0/virtio0/host0/target0:0:0/0:0:{}/block/sda",
+            root_bus, addr_a
+        );
+        let matcher_a = ScsiBlockMatcher::new(&addr_a);
+
+        let mut uev_b = uev_a.clone();
+        let addr_b = "2:0";
+        uev_b.devpath = format!(
+            "{}/0000:00:00.0/virtio0/host0/target0:0:2/0:0:{}/block/sdb",
+            root_bus, addr_b
+        );
+        let matcher_b = ScsiBlockMatcher::new(&addr_b);
+
+        assert!(matcher_a.is_match(&uev_a));
+        assert!(matcher_b.is_match(&uev_b));
+        assert!(!matcher_b.is_match(&uev_a));
+        assert!(!matcher_a.is_match(&uev_b));
+    }
 }

src/agent/src/linux_abi.rs

@@ -65,6 +65,10 @@ pub fn create_pci_root_bus_path() -> String {
     ret
 }
 
+#[cfg(target_arch = "s390x")]
+pub fn create_ccw_root_bus_path() -> String {
+    String::from("/devices/css0")
+}
 // From https://www.kernel.org/doc/Documentation/acpi/namespace.txt
 // The Linux kernel's core ACPI subsystem creates struct acpi_device
 // objects for ACPI namespace objects representing devices, power resources
@@ -78,11 +82,6 @@ pub const SYSFS_MEMORY_BLOCK_SIZE_PATH: &str = "/sys/devices/system/memory/block
 pub const SYSFS_MEMORY_HOTPLUG_PROBE_PATH: &str = "/sys/devices/system/memory/probe";
 pub const SYSFS_MEMORY_ONLINE_PATH: &str = "/sys/devices/system/memory";
 
-// Here in "0:0", the first number is the SCSI host number because
-// only one SCSI controller has been plugged, while the second number
-// is always 0.
-pub const SCSI_HOST_CHANNEL: &str = "0:0:";
-pub const SCSI_BLOCK_SUFFIX: &str = "block";
 pub const SYSFS_SCSI_HOST_PATH: &str = "/sys/class/scsi_host";
 
 pub const SYSFS_CGROUPPATH: &str = "/sys/fs/cgroup";

src/agent/src/main.rs

@@ -5,8 +5,8 @@
 
 #[macro_use]
 extern crate lazy_static;
+extern crate capctl;
 extern crate oci;
-extern crate prctl;
 extern crate prometheus;
 extern crate protocols;
 extern crate regex;
@@ -20,27 +20,24 @@ extern crate scopeguard;
 extern crate slog;
 
 use anyhow::{anyhow, Context, Result};
-use nix::fcntl::{self, OFlag};
-use nix::fcntl::{FcntlArg, FdFlag};
-use nix::libc::{STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO};
-use nix::pty;
-use nix::sys::select::{select, FdSet};
+use nix::fcntl::OFlag;
 use nix::sys::socket::{self, AddressFamily, SockAddr, SockFlag, SockType};
-use nix::sys::wait;
-use nix::unistd::{self, close, dup, dup2, fork, setsid, ForkResult};
-use std::collections::HashMap;
+use nix::unistd::{self, dup, Pid};
 use std::env;
-use std::ffi::{CStr, CString, OsStr};
+use std::ffi::OsStr;
 use std::fs::{self, File};
-use std::io::{Read, Write};
 use std::os::unix::ffi::OsStrExt;
 use std::os::unix::fs as unixfs;
 use std::os::unix::io::AsRawFd;
 use std::path::Path;
+use std::process::exit;
 use std::sync::Arc;
-use unistd::Pid;
+use tracing::{instrument, span};
 
+#[cfg(target_arch = "s390x")]
+mod ccw;
 mod config;
+mod console;
 mod device;
 mod linux_abi;
 mod metrics;
@@ -57,44 +54,37 @@ mod test_utils;
 mod uevent;
 mod util;
 mod version;
+mod watcher;
 
 use mount::{cgroups_mount, general_mount};
 use sandbox::Sandbox;
 use signal::setup_signal_handler;
-use slog::Logger;
+use slog::{error, info, o, warn, Logger};
 use uevent::watch_uevents;
 
-use std::sync::Mutex as SyncMutex;
-
 use futures::future::join_all;
-use futures::StreamExt as _;
 use rustjail::pipestream::PipeStream;
 use tokio::{
     io::AsyncWrite,
     sync::{
-        oneshot::Sender,
         watch::{channel, Receiver},
         Mutex, RwLock,
     },
     task::JoinHandle,
 };
-use tokio_vsock::{Incoming, VsockListener, VsockStream};
 
 mod rpc;
+mod tracer;
 
 const NAME: &str = "kata-agent";
 const KERNEL_CMDLINE_FILE: &str = "/proc/cmdline";
-const CONSOLE_PATH: &str = "/dev/console";
-
-const DEFAULT_BUF_SIZE: usize = 8 * 1024;
 
 lazy_static! {
-    static ref GLOBAL_DEVICE_WATCHER: Arc<Mutex<HashMap<String, Option<Sender<String>>>>> =
-        Arc::new(Mutex::new(HashMap::new()));
     static ref AGENT_CONFIG: Arc<RwLock<AgentConfig>> =
         Arc::new(RwLock::new(config::AgentConfig::new()));
 }
 
+#[instrument]
 fn announce(logger: &Logger, config: &AgentConfig) {
     info!(logger, "announce";
     "agent-commit" => version::VERSION_COMMIT,
@@ -108,27 +98,6 @@ fn announce(logger: &Logger, config: &AgentConfig) {
     );
 }
 
-fn set_fd_close_exec(fd: RawFd) -> Result<RawFd> {
-    if let Err(e) = fcntl::fcntl(fd, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)) {
-        return Err(anyhow!("failed to set fd: {} as close-on-exec: {}", fd, e));
-    }
-    Ok(fd)
-}
-
-fn get_vsock_incoming(fd: RawFd) -> Incoming {
-    let incoming;
-    unsafe {
-        incoming = VsockListener::from_raw_fd(fd).incoming();
-    }
-    incoming
-}
-
-async fn get_vsock_stream(fd: RawFd) -> Result<VsockStream> {
-    let stream = get_vsock_incoming(fd).next().await.unwrap().unwrap();
-    set_fd_close_exec(stream.as_raw_fd())?;
-    Ok(stream)
-}
-
 // Create a thread to handle reading from the logger pipe. The thread will
 // output to the vsock port specified, or stdout.
 async fn create_logger_task(rfd: RawFd, vsock_port: u32, shutdown: Receiver<bool>) -> Result<()> {
@@ -147,7 +116,7 @@ async fn create_logger_task(rfd: RawFd, vsock_port: u32, shutdown: Receiver<bool
         socket::bind(listenfd, &addr).unwrap();
         socket::listen(listenfd, 1).unwrap();
 
-        writer = Box::new(get_vsock_stream(listenfd).await.unwrap());
+        writer = Box::new(util::get_vsock_stream(listenfd).await.unwrap());
     } else {
         writer = Box::new(tokio::io::stdout());
     }
@@ -163,7 +132,7 @@ async fn real_main() -> std::result::Result<(), Box<dyn std::error::Error>> {
     // List of tasks that need to be stopped for a clean shutdown
     let mut tasks: Vec<JoinHandle<Result<()>>> = vec![];
 
-    lazy_static::initialize(&SHELLS);
+    console::initialize();
 
     lazy_static::initialize(&AGENT_CONFIG);
 
@@ -236,6 +205,17 @@ async fn real_main() -> std::result::Result<(), Box<dyn std::error::Error>> {
         ttrpc_log_guard = Ok(slog_stdlog::init().map_err(|e| e)?);
     }
 
+    if config.tracing != tracer::TraceType::Disabled {
+        let _ = tracer::setup_tracing(NAME, &logger, &config)?;
+    }
+
+    let root = span!(tracing::Level::TRACE, "root-span", work_units = 2);
+
+    // XXX: Start the root trace transaction.
+    //
+    // XXX: Note that *ALL* spans needs to start after this point!!
+    let _enter = root.enter();
+
     // Start the sandbox and wait for its ttRPC server to end
     start_sandbox(&logger, &config, init_mode, &mut tasks, shutdown_rx.clone()).await?;
 
@@ -264,6 +244,10 @@ async fn real_main() -> std::result::Result<(), Box<dyn std::error::Error>> {
         }
     }
 
+    if config.tracing != tracer::TraceType::Disabled {
+        tracer::end_tracing();
+    }
+
     eprintln!("{} shutdown complete", NAME);
 
     Ok(())
@@ -285,6 +269,7 @@ fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
     }
 
     if args.len() == 2 && args[1] == "init" {
+        reset_sigpipe();
         rustjail::container::init_child();
         exit(0);
     }
@@ -296,6 +281,7 @@ fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
     rt.block_on(real_main())
 }
 
+#[instrument]
 async fn start_sandbox(
     logger: &Logger,
     config: &AgentConfig,
@@ -303,26 +289,17 @@ async fn start_sandbox(
     tasks: &mut Vec<JoinHandle<Result<()>>>,
     shutdown: Receiver<bool>,
 ) -> Result<()> {
-    let shells = SHELLS.clone();
     let debug_console_vport = config.debug_console_vport as u32;
 
-    let shell_handle = if config.debug_console {
-        let thread_logger = logger.clone();
-        let shells = shells.lock().unwrap().to_vec();
+    if config.debug_console {
+        let debug_console_task = tokio::task::spawn(console::debug_console_handler(
+            logger.clone(),
+            debug_console_vport,
+            shutdown.clone(),
+        ));
 
-        let handle = tokio::task::spawn_blocking(move || {
-            let result = setup_debug_console(&thread_logger, shells, debug_console_vport);
-            if result.is_err() {
-                // Report error, but don't fail
-                warn!(thread_logger, "failed to setup debug console";
-                    "error" => format!("{}", result.unwrap_err()));
-            }
-        });
-
-        Some(handle)
-    } else {
-        None
-    };
+        tasks.push(debug_console_task);
+    }
 
     // Initialize unique sandbox structure.
     let s = Sandbox::new(&logger).context("Failed to create sandbox")?;
@@ -351,13 +328,9 @@ async fn start_sandbox(
     let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str());
     server.start().await?;
 
-    let _ = rx.await?;
+    rx.await?;
     server.shutdown().await?;
 
-    if let Some(handle) = shell_handle {
-        handle.await.map_err(|e| anyhow!("{:?}", e))?;
-    }
-
     Ok(())
 }
 
@@ -395,6 +368,7 @@ fn init_agent_as_init(logger: &Logger, unified_cgroup_hierarchy: bool) -> Result
     Ok(())
 }
 
+#[instrument]
 fn sethostname(hostname: &OsStr) -> Result<()> {
     let size = hostname.len() as usize;
 
@@ -408,284 +382,16 @@ fn sethostname(hostname: &OsStr) -> Result<()> {
     }
 }
 
-lazy_static! {
-    static ref SHELLS: Arc<SyncMutex<Vec<String>>> = {
-        let mut v = Vec::new();
-
-        if !cfg!(test) {
-            v.push("/bin/bash".to_string());
-            v.push("/bin/sh".to_string());
-        }
-
-        Arc::new(SyncMutex::new(v))
-    };
+// The Rust standard library had suppressed the default SIGPIPE behavior,
+// see https://github.com/rust-lang/rust/pull/13158.
+// Since the parent's signal handler would be inherited by it's child process,
+// thus we should re-enable the standard SIGPIPE behavior as a workaround to
+// fix the issue of https://github.com/kata-containers/kata-containers/issues/1887.
+fn reset_sigpipe() {
+    unsafe {
+        libc::signal(libc::SIGPIPE, libc::SIG_DFL);
+    }
 }
 
 use crate::config::AgentConfig;
-use nix::sys::stat::Mode;
 use std::os::unix::io::{FromRawFd, RawFd};
-use std::path::PathBuf;
-use std::process::exit;
-
-fn setup_debug_console(logger: &Logger, shells: Vec<String>, port: u32) -> Result<()> {
-    let shell = shells
-        .iter()
-        .find(|sh| PathBuf::from(sh).exists())
-        .ok_or_else(|| anyhow!("no shell found to launch debug console"))?;
-
-    if port > 0 {
-        let listenfd = socket::socket(
-            AddressFamily::Vsock,
-            SockType::Stream,
-            SockFlag::SOCK_CLOEXEC,
-            None,
-        )?;
-        let addr = SockAddr::new_vsock(libc::VMADDR_CID_ANY, port);
-        socket::bind(listenfd, &addr)?;
-        socket::listen(listenfd, 1)?;
-        loop {
-            let f: RawFd = socket::accept4(listenfd, SockFlag::SOCK_CLOEXEC)?;
-            match run_debug_console_shell(logger, shell, f) {
-                Ok(_) => {
-                    info!(logger, "run_debug_console_shell session finished");
-                }
-                Err(err) => {
-                    error!(logger, "run_debug_console_shell failed: {:?}", err);
-                }
-            }
-        }
-    } else {
-        let mut flags = OFlag::empty();
-        flags.insert(OFlag::O_RDWR);
-        flags.insert(OFlag::O_CLOEXEC);
-        loop {
-            let f: RawFd = fcntl::open(CONSOLE_PATH, flags, Mode::empty())?;
-            match run_debug_console_shell(logger, shell, f) {
-                Ok(_) => {
-                    info!(logger, "run_debug_console_shell session finished");
-                }
-                Err(err) => {
-                    error!(logger, "run_debug_console_shell failed: {:?}", err);
-                }
-            }
-        }
-    };
-}
-
-fn io_copy<R: ?Sized, W: ?Sized>(reader: &mut R, writer: &mut W) -> std::io::Result<u64>
-where
-    R: Read,
-    W: Write,
-{
-    let mut buf = [0; DEFAULT_BUF_SIZE];
-    let buf_len;
-
-    match reader.read(&mut buf) {
-        Ok(0) => return Ok(0),
-        Ok(len) => buf_len = len,
-        Err(err) => return Err(err),
-    };
-
-    // write and return
-    match writer.write_all(&buf[..buf_len]) {
-        Ok(_) => Ok(buf_len as u64),
-        Err(err) => Err(err),
-    }
-}
-
-fn run_debug_console_shell(logger: &Logger, shell: &str, socket_fd: RawFd) -> Result<()> {
-    let pseduo = pty::openpty(None, None)?;
-    let _ = fcntl::fcntl(pseduo.master, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC));
-    let _ = fcntl::fcntl(pseduo.slave, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC));
-
-    let slave_fd = pseduo.slave;
-
-    match fork() {
-        Ok(ForkResult::Child) => {
-            // create new session with child as session leader
-            setsid()?;
-
-            // dup stdin, stdout, stderr to let child act as a terminal
-            dup2(slave_fd, STDIN_FILENO)?;
-            dup2(slave_fd, STDOUT_FILENO)?;
-            dup2(slave_fd, STDERR_FILENO)?;
-
-            // set tty
-            unsafe {
-                libc::ioctl(0, libc::TIOCSCTTY);
-            }
-
-            let cmd = CString::new(shell).unwrap();
-            let args: Vec<&CStr> = vec![];
-
-            // run shell
-            let _ = unistd::execvp(cmd.as_c_str(), args.as_slice()).map_err(|e| match e {
-                nix::Error::Sys(errno) => {
-                    std::process::exit(errno as i32);
-                }
-                _ => std::process::exit(-2),
-            });
-        }
-
-        Ok(ForkResult::Parent { child: child_pid }) => {
-            info!(logger, "get debug shell pid {:?}", child_pid);
-
-            let (rfd, wfd) = unistd::pipe2(OFlag::O_CLOEXEC)?;
-            let master_fd = pseduo.master;
-            let debug_shell_logger = logger.clone();
-
-            // channel that used to sync between thread and main process
-            let (tx, rx) = std::sync::mpsc::channel::<i32>();
-
-            // start a thread to do IO copy between socket and pseduo.master
-            std::thread::spawn(move || {
-                let mut master_reader = unsafe { File::from_raw_fd(master_fd) };
-                let mut master_writer = unsafe { File::from_raw_fd(master_fd) };
-                let mut socket_reader = unsafe { File::from_raw_fd(socket_fd) };
-                let mut socket_writer = unsafe { File::from_raw_fd(socket_fd) };
-
-                loop {
-                    let mut fd_set = FdSet::new();
-                    fd_set.insert(rfd);
-                    fd_set.insert(master_fd);
-                    fd_set.insert(socket_fd);
-
-                    match select(
-                        Some(fd_set.highest().unwrap() + 1),
-                        &mut fd_set,
-                        None,
-                        None,
-                        None,
-                    ) {
-                        Ok(_) => (),
-                        Err(e) => {
-                            if e == nix::Error::from(nix::errno::Errno::EINTR) {
-                                continue;
-                            } else {
-                                error!(debug_shell_logger, "select error {:?}", e);
-                                tx.send(1).unwrap();
-                                break;
-                            }
-                        }
-                    }
-
-                    if fd_set.contains(rfd) {
-                        info!(
-                            debug_shell_logger,
-                            "debug shell process {} exited", child_pid
-                        );
-                        tx.send(1).unwrap();
-                        break;
-                    }
-
-                    if fd_set.contains(master_fd) {
-                        match io_copy(&mut master_reader, &mut socket_writer) {
-                            Ok(0) => {
-                                debug!(debug_shell_logger, "master fd closed");
-                                tx.send(1).unwrap();
-                                break;
-                            }
-                            Ok(_) => {}
-                            Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
-                            Err(e) => {
-                                error!(debug_shell_logger, "read master fd error {:?}", e);
-                                tx.send(1).unwrap();
-                                break;
-                            }
-                        }
-                    }
-
-                    if fd_set.contains(socket_fd) {
-                        match io_copy(&mut socket_reader, &mut master_writer) {
-                            Ok(0) => {
-                                debug!(debug_shell_logger, "socket fd closed");
-                                tx.send(1).unwrap();
-                                break;
-                            }
-                            Ok(_) => {}
-                            Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
-                            Err(e) => {
-                                error!(debug_shell_logger, "read socket fd error {:?}", e);
-                                tx.send(1).unwrap();
-                                break;
-                            }
-                        }
-                    }
-                }
-            });
-
-            let wait_status = wait::waitpid(child_pid, None);
-            info!(logger, "debug console process exit code: {:?}", wait_status);
-
-            info!(logger, "notify debug monitor thread to exit");
-            // close pipe to exit select loop
-            let _ = close(wfd);
-
-            // wait for thread exit.
-            let _ = rx.recv().unwrap();
-            info!(logger, "debug monitor thread has exited");
-
-            // close files
-            let _ = close(rfd);
-            let _ = close(master_fd);
-            let _ = close(slave_fd);
-        }
-        Err(err) => {
-            return Err(anyhow!("fork error: {:?}", err));
-        }
-    }
-
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use tempfile::tempdir;
-
-    #[test]
-    fn test_setup_debug_console_no_shells() {
-        // Guarantee no shells have been added
-        // (required to avoid racing with
-        // test_setup_debug_console_invalid_shell()).
-        let shells_ref = SHELLS.clone();
-        let mut shells = shells_ref.lock().unwrap();
-        shells.clear();
-        let logger = slog_scope::logger();
-
-        let result = setup_debug_console(&logger, shells.to_vec(), 0);
-
-        assert!(result.is_err());
-        assert_eq!(
-            result.unwrap_err().to_string(),
-            "no shell found to launch debug console"
-        );
-    }
-
-    #[test]
-    fn test_setup_debug_console_invalid_shell() {
-        let shells_ref = SHELLS.clone();
-        let mut shells = shells_ref.lock().unwrap();
-
-        let dir = tempdir().expect("failed to create tmpdir");
-
-        // Add an invalid shell
-        let shell = dir
-            .path()
-            .join("enoent")
-            .to_str()
-            .expect("failed to construct shell path")
-            .to_string();
-
-        shells.push(shell);
-        let logger = slog_scope::logger();
-
-        let result = setup_debug_console(&logger, shells.to_vec(), 0);
-
-        assert!(result.is_err());
-        assert_eq!(
-            result.unwrap_err().to_string(),
-            "no shell found to launch debug console"
-        );
-    }
-}

src/agent/src/metrics.rs

@@ -8,6 +8,7 @@ extern crate procfs;
 use prometheus::{Encoder, Gauge, GaugeVec, IntCounter, TextEncoder};
 
 use anyhow::Result;
+use tracing::instrument;
 
 const NAMESPACE_KATA_AGENT: &str = "kata_agent";
 const NAMESPACE_KATA_GUEST: &str = "kata_guest";
@@ -68,6 +69,7 @@ lazy_static! {
     prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"meminfo").as_ref() , "Statistics about memory usage in the system.", &["item"]).unwrap();
 }
 
+#[instrument]
 pub fn get_metrics(_: &protocols::agent::GetMetricsRequest) -> Result<String> {
     AGENT_SCRAPE_COUNT.inc();
 
@@ -87,6 +89,7 @@ pub fn get_metrics(_: &protocols::agent::GetMetricsRequest) -> Result<String> {
     Ok(String::from_utf8(buffer).unwrap())
 }
 
+#[instrument]
 fn update_agent_metrics() {
     let me = procfs::process::Process::myself();
 
@@ -136,6 +139,7 @@ fn update_agent_metrics() {
     }
 }
 
+#[instrument]
 fn update_guest_metrics() {
     // try get load and task info
     match procfs::LoadAverage::new() {
@@ -218,6 +222,7 @@ fn update_guest_metrics() {
     }
 }
 
+#[instrument]
 fn set_gauge_vec_meminfo(gv: &prometheus::GaugeVec, meminfo: &procfs::Meminfo) {
     gv.with_label_values(&["mem_total"])
         .set(meminfo.mem_total as f64);
@@ -332,6 +337,7 @@ fn set_gauge_vec_meminfo(gv: &prometheus::GaugeVec, meminfo: &procfs::Meminfo) {
         .set(meminfo.k_reclaimable.unwrap_or(0) as f64);
 }
 
+#[instrument]
 fn set_gauge_vec_cpu_time(gv: &prometheus::GaugeVec, cpu: &str, cpu_time: &procfs::CpuTime) {
     gv.with_label_values(&[cpu, "user"])
         .set(cpu_time.user as f64);
@@ -355,6 +361,7 @@ fn set_gauge_vec_cpu_time(gv: &prometheus::GaugeVec, cpu: &str, cpu_time: &procf
         .set(cpu_time.guest_nice.unwrap_or(0.0) as f64);
 }
 
+#[instrument]
 fn set_gauge_vec_diskstat(gv: &prometheus::GaugeVec, diskstat: &procfs::DiskStat) {
     gv.with_label_values(&[diskstat.name.as_str(), "reads"])
         .set(diskstat.reads as f64);
@@ -393,6 +400,7 @@ fn set_gauge_vec_diskstat(gv: &prometheus::GaugeVec, diskstat: &procfs::DiskStat
 }
 
 // set_gauge_vec_netdev set gauge for NetDevLine
+#[instrument]
 fn set_gauge_vec_netdev(gv: &prometheus::GaugeVec, status: &procfs::net::DeviceStatus) {
     gv.with_label_values(&[status.name.as_str(), "recv_bytes"])
         .set(status.recv_bytes as f64);
@@ -429,6 +437,7 @@ fn set_gauge_vec_netdev(gv: &prometheus::GaugeVec, status: &procfs::net::DeviceS
 }
 
 // set_gauge_vec_proc_status set gauge for ProcStatus
+#[instrument]
 fn set_gauge_vec_proc_status(gv: &prometheus::GaugeVec, status: &procfs::process::Status) {
     gv.with_label_values(&["vmpeak"])
         .set(status.vmpeak.unwrap_or(0) as f64);
@@ -469,6 +478,7 @@ fn set_gauge_vec_proc_status(gv: &prometheus::GaugeVec, status: &procfs::process
 }
 
 // set_gauge_vec_proc_io set gauge for ProcIO
+#[instrument]
 fn set_gauge_vec_proc_io(gv: &prometheus::GaugeVec, io_stat: &procfs::process::Io) {
     gv.with_label_values(&["rchar"]).set(io_stat.rchar as f64);
     gv.with_label_values(&["wchar"]).set(io_stat.wchar as f64);
@@ -483,6 +493,7 @@ fn set_gauge_vec_proc_io(gv: &prometheus::GaugeVec, io_stat: &procfs::process::I
 }
 
 // set_gauge_vec_proc_stat set gauge for ProcStat
+#[instrument]
 fn set_gauge_vec_proc_stat(gv: &prometheus::GaugeVec, stat: &procfs::process::Stat) {
     gv.with_label_values(&["utime"]).set(stat.utime as f64);
     gv.with_label_values(&["stime"]).set(stat.stime as f64);

src/agent/src/mount.rs

@@ -6,45 +6,55 @@
 use std::collections::HashMap;
 use std::ffi::CString;
 use std::fs;
+use std::fs::File;
 use std::io;
-use std::os::unix::fs::PermissionsExt;
-
+use std::io::{BufRead, BufReader};
+use std::iter;
+use std::os::unix::fs::{MetadataExt, PermissionsExt};
 use std::path::Path;
 use std::ptr::null;
 use std::str::FromStr;
 use std::sync::Arc;
+
 use tokio::sync::Mutex;
 
 use libc::{c_void, mount};
 use nix::mount::{self, MsFlags};
+use nix::unistd::Gid;
 
 use regex::Regex;
-use std::fs::File;
-use std::io::{BufRead, BufReader};
 
 use crate::device::{
-    get_pci_device_name, get_pmem_device_name, get_scsi_device_name, online_device,
+    get_scsi_device_name, get_virtio_blk_pci_device_name, online_device, wait_for_pmem_device,
 };
 use crate::linux_abi::*;
 use crate::pci;
 use crate::protocols::agent::Storage;
 use crate::Sandbox;
+#[cfg(target_arch = "s390x")]
+use crate::{ccw, device::get_virtio_blk_ccw_device_name};
 use anyhow::{anyhow, Context, Result};
 use slog::Logger;
+use tracing::instrument;
 
 pub const DRIVER_9P_TYPE: &str = "9p";
 pub const DRIVER_VIRTIOFS_TYPE: &str = "virtio-fs";
 pub const DRIVER_BLK_TYPE: &str = "blk";
+pub const DRIVER_BLK_CCW_TYPE: &str = "blk-ccw";
 pub const DRIVER_MMIO_BLK_TYPE: &str = "mmioblk";
 pub const DRIVER_SCSI_TYPE: &str = "scsi";
 pub const DRIVER_NVDIMM_TYPE: &str = "nvdimm";
 pub const DRIVER_EPHEMERAL_TYPE: &str = "ephemeral";
 pub const DRIVER_LOCAL_TYPE: &str = "local";
+pub const DRIVER_WATCHABLE_BIND_TYPE: &str = "watchable-bind";
 
 pub const TYPE_ROOTFS: &str = "rootfs";
 
 pub const MOUNT_GUEST_TAG: &str = "kataShared";
 
+// Allocating an FSGroup that owns the pod's volumes
+const FS_GID: &str = "fsgid";
+
 #[rustfmt::skip]
 lazy_static! {
     pub static ref FLAGS: HashMap<&'static str, (bool, MsFlags)> = {
@@ -127,7 +137,7 @@ lazy_static! {
     ];
 }
 
-pub const STORAGE_HANDLER_LIST: [&str; 8] = [
+pub const STORAGE_HANDLER_LIST: &[&str] = &[
     DRIVER_BLK_TYPE,
     DRIVER_9P_TYPE,
     DRIVER_VIRTIOFS_TYPE,
@@ -136,6 +146,7 @@ pub const STORAGE_HANDLER_LIST: [&str; 8] = [
     DRIVER_LOCAL_TYPE,
     DRIVER_SCSI_TYPE,
     DRIVER_NVDIMM_TYPE,
+    DRIVER_WATCHABLE_BIND_TYPE,
 ];
 
 #[derive(Debug, Clone)]
@@ -152,6 +163,7 @@ pub struct BareMount<'a> {
 // * evaluate all symlinks
 // * ensure the source exists
 impl<'a> BareMount<'a> {
+    #[instrument]
     pub fn new(
         s: &'a str,
         d: &'a str,
@@ -170,6 +182,7 @@ impl<'a> BareMount<'a> {
         }
     }
 
+    #[instrument]
     pub fn mount(&self) -> Result<()> {
         let source;
         let dest;
@@ -228,6 +241,7 @@ impl<'a> BareMount<'a> {
     }
 }
 
+#[instrument]
 async fn ephemeral_storage_handler(
     logger: &Logger,
     storage: &Storage,
@@ -241,11 +255,40 @@ async fn ephemeral_storage_handler(
     }
 
     fs::create_dir_all(Path::new(&storage.mount_point))?;
-    common_storage_handler(logger, storage)?;
+
+    // By now we only support one option field: "fsGroup" which
+    // isn't an valid mount option, thus we should remove it when
+    // do mount.
+    if storage.options.len() > 0 {
+        // ephemeral_storage didn't support mount options except fsGroup.
+        let mut new_storage = storage.clone();
+        new_storage.options = protobuf::RepeatedField::default();
+        common_storage_handler(logger, &new_storage)?;
+
+        let opts_vec: Vec<String> = storage.options.to_vec();
+
+        let opts = parse_options(opts_vec);
+
+        if let Some(fsgid) = opts.get(FS_GID) {
+            let gid = fsgid.parse::<u32>()?;
+
+            nix::unistd::chown(storage.mount_point.as_str(), None, Some(Gid::from_raw(gid)))?;
+
+            let meta = fs::metadata(&storage.mount_point)?;
+            let mut permission = meta.permissions();
+
+            let o_mode = meta.mode() | 0o2000;
+            permission.set_mode(o_mode);
+            fs::set_permissions(&storage.mount_point, permission)?;
+        }
+    } else {
+        common_storage_handler(logger, &storage)?;
+    }
 
     Ok("".to_string())
 }
 
+#[instrument]
 async fn local_storage_handler(
     _logger: &Logger,
     storage: &Storage,
@@ -266,11 +309,24 @@ async fn local_storage_handler(
     let opts_vec: Vec<String> = storage.options.to_vec();
 
     let opts = parse_options(opts_vec);
-    let mode = opts.get("mode");
-    if let Some(mode) = mode {
+
+    let mut need_set_fsgid = false;
+    if let Some(fsgid) = opts.get(FS_GID) {
+        let gid = fsgid.parse::<u32>()?;
+
+        nix::unistd::chown(storage.mount_point.as_str(), None, Some(Gid::from_raw(gid)))?;
+        need_set_fsgid = true;
+    }
+
+    if let Some(mode) = opts.get("mode") {
         let mut permission = fs::metadata(&storage.mount_point)?.permissions();
 
-        let o_mode = u32::from_str_radix(mode, 8)?;
+        let mut o_mode = u32::from_str_radix(mode, 8)?;
+
+        if need_set_fsgid {
+            // set SetGid mode mask.
+            o_mode |= 0o2000;
+        }
         permission.set_mode(o_mode);
 
         fs::set_permissions(&storage.mount_point, permission)?;
@@ -279,6 +335,7 @@ async fn local_storage_handler(
     Ok("".to_string())
 }
 
+#[instrument]
 async fn virtio9p_storage_handler(
     logger: &Logger,
     storage: &Storage,
@@ -288,6 +345,7 @@ async fn virtio9p_storage_handler(
 }
 
 // virtiommio_blk_storage_handler handles the storage for mmio blk driver.
+#[instrument]
 async fn virtiommio_blk_storage_handler(
     logger: &Logger,
     storage: &Storage,
@@ -298,6 +356,7 @@ async fn virtiommio_blk_storage_handler(
 }
 
 // virtiofs_storage_handler handles the storage for virtio-fs.
+#[instrument]
 async fn virtiofs_storage_handler(
     logger: &Logger,
     storage: &Storage,
@@ -307,6 +366,7 @@ async fn virtiofs_storage_handler(
 }
 
 // virtio_blk_storage_handler handles the storage for blk driver.
+#[instrument]
 async fn virtio_blk_storage_handler(
     logger: &Logger,
     storage: &Storage,
@@ -325,14 +385,40 @@ async fn virtio_blk_storage_handler(
         }
     } else {
         let pcipath = pci::Path::from_str(&storage.source)?;
-        let dev_path = get_pci_device_name(&sandbox, &pcipath).await?;
+        let dev_path = get_virtio_blk_pci_device_name(&sandbox, &pcipath).await?;
         storage.source = dev_path;
     }
 
     common_storage_handler(logger, &storage)
 }
 
-// virtio_scsi_storage_handler handles the storage for scsi driver.
+// virtio_blk_ccw_storage_handler handles storage for the blk-ccw driver (s390x)
+#[cfg(target_arch = "s390x")]
+#[instrument]
+async fn virtio_blk_ccw_storage_handler(
+    logger: &Logger,
+    storage: &Storage,
+    sandbox: Arc<Mutex<Sandbox>>,
+) -> Result<String> {
+    let mut storage = storage.clone();
+    let ccw_device = ccw::Device::from_str(&storage.source)?;
+    let dev_path = get_virtio_blk_ccw_device_name(&sandbox, &ccw_device).await?;
+    storage.source = dev_path;
+    common_storage_handler(logger, &storage)
+}
+
+#[cfg(not(target_arch = "s390x"))]
+#[instrument]
+async fn virtio_blk_ccw_storage_handler(
+    _: &Logger,
+    _: &Storage,
+    _: Arc<Mutex<Sandbox>>,
+) -> Result<String> {
+    Err(anyhow!("CCW is only supported on s390x"))
+}
+
+// virtio_scsi_storage_handler handles the  storage for scsi driver.
+#[instrument]
 async fn virtio_scsi_storage_handler(
     logger: &Logger,
     storage: &Storage,
@@ -347,6 +433,7 @@ async fn virtio_scsi_storage_handler(
     common_storage_handler(logger, &storage)
 }
 
+#[instrument]
 fn common_storage_handler(logger: &Logger, storage: &Storage) -> Result<String> {
     // Mount the storage device.
     let mount_point = storage.mount_point.to_string();
@@ -355,32 +442,36 @@ fn common_storage_handler(logger: &Logger, storage: &Storage) -> Result<String>
 }
 
 // nvdimm_storage_handler handles the storage for NVDIMM driver.
+#[instrument]
 async fn nvdimm_storage_handler(
     logger: &Logger,
     storage: &Storage,
     sandbox: Arc<Mutex<Sandbox>>,
 ) -> Result<String> {
-    let mut storage = storage.clone();
-    // If hot-plugged, get the device node path based on the PCI address else
-    // use the virt path provided in Storage Source
-    let pmem_devname = match storage.source.strip_prefix("/dev/") {
-        Some(dev) => dev,
-        None => {
-            return Err(anyhow!(
-                "Storage source '{}' must start with /dev/",
-                storage.source
-            ))
-        }
-    };
+    let storage = storage.clone();
 
     // Retrieve the device path from NVDIMM address.
-    let dev_path = get_pmem_device_name(&sandbox, pmem_devname).await?;
-    storage.source = dev_path;
+    wait_for_pmem_device(&sandbox, &storage.source).await?;
 
     common_storage_handler(logger, &storage)
 }
 
+async fn bind_watcher_storage_handler(
+    logger: &Logger,
+    storage: &Storage,
+    sandbox: Arc<Mutex<Sandbox>>,
+) -> Result<()> {
+    let mut locked = sandbox.lock().await;
+    let container_id = locked.id.clone();
+
+    locked
+        .bind_watcher
+        .add_container(container_id, iter::once(storage.clone()), logger)
+        .await
+}
+
 // mount_storage performs the mount described by the storage structure.
+#[instrument]
 fn mount_storage(logger: &Logger, storage: &Storage) -> Result<()> {
     let logger = logger.new(o!("subsystem" => "mount"));
 
@@ -388,7 +479,10 @@ fn mount_storage(logger: &Logger, storage: &Storage) -> Result<()> {
     // If so, skip doing the mount. This facilitates mounting the sharedfs automatically
     // in the guest before the agent service starts.
     if storage.source == MOUNT_GUEST_TAG && is_mounted(&storage.mount_point)? {
-        warn!(logger, "kataShared already mounted, ignoring...");
+        warn!(
+            logger,
+            "{} already mounted on {}, ignoring...", MOUNT_GUEST_TAG, &storage.mount_point
+        );
         return Ok(());
     }
 
@@ -428,7 +522,8 @@ fn mount_storage(logger: &Logger, storage: &Storage) -> Result<()> {
 }
 
 /// Looks for `mount_point` entry in the /proc/mounts.
-fn is_mounted(mount_point: &str) -> Result<bool> {
+#[instrument]
+pub fn is_mounted(mount_point: &str) -> Result<bool> {
     let mount_point = mount_point.trim_end_matches('/');
     let found = fs::metadata(mount_point).is_ok()
         // Looks through /proc/mounts and check if the mount exists
@@ -445,6 +540,7 @@ fn is_mounted(mount_point: &str) -> Result<bool> {
     Ok(found)
 }
 
+#[instrument]
 fn parse_mount_flags_and_options(options_vec: Vec<&str>) -> (MsFlags, String) {
     let mut flags = MsFlags::empty();
     let mut options: String = "".to_string();
@@ -473,6 +569,7 @@ fn parse_mount_flags_and_options(options_vec: Vec<&str>) -> (MsFlags, String) {
 // associated operations such as waiting for the device to show up, and mount
 // it to a specific location, according to the type of handler chosen, and for
 // each storage.
+#[instrument]
 pub async fn add_storages(
     logger: Logger,
     storages: Vec<Storage>,
@@ -488,6 +585,9 @@ pub async fn add_storages(
 
         let res = match handler_name.as_str() {
             DRIVER_BLK_TYPE => virtio_blk_storage_handler(&logger, &storage, sandbox.clone()).await,
+            DRIVER_BLK_CCW_TYPE => {
+                virtio_blk_ccw_storage_handler(&logger, &storage, sandbox.clone()).await
+            }
             DRIVER_9P_TYPE => virtio9p_storage_handler(&logger, &storage, sandbox.clone()).await,
             DRIVER_VIRTIOFS_TYPE => {
                 virtiofs_storage_handler(&logger, &storage, sandbox.clone()).await
@@ -503,6 +603,11 @@ pub async fn add_storages(
                 virtio_scsi_storage_handler(&logger, &storage, sandbox.clone()).await
             }
             DRIVER_NVDIMM_TYPE => nvdimm_storage_handler(&logger, &storage, sandbox.clone()).await,
+            DRIVER_WATCHABLE_BIND_TYPE => {
+                bind_watcher_storage_handler(&logger, &storage, sandbox.clone()).await?;
+                // Don't register watch mounts, they're hanlded separately by the watcher.
+                Ok(String::new())
+            }
             _ => {
                 return Err(anyhow!(
                     "Failed to find the storage handler {}",
@@ -522,6 +627,7 @@ pub async fn add_storages(
     Ok(mount_list)
 }
 
+#[instrument]
 fn mount_to_rootfs(logger: &Logger, m: &InitMount) -> Result<()> {
     let options_vec: Vec<&str> = m.options.clone();
 
@@ -547,6 +653,7 @@ fn mount_to_rootfs(logger: &Logger, m: &InitMount) -> Result<()> {
     Ok(())
 }
 
+#[instrument]
 pub fn general_mount(logger: &Logger) -> Result<()> {
     let logger = logger.new(o!("subsystem" => "mount"));
 
@@ -564,6 +671,7 @@ pub fn get_mount_fs_type(mount_point: &str) -> Result<String> {
 
 // get_mount_fs_type_from_file returns the FS type corresponding to the passed mount point and
 // any error ecountered.
+#[instrument]
 pub fn get_mount_fs_type_from_file(mount_file: &str, mount_point: &str) -> Result<String> {
     if mount_point.is_empty() {
         return Err(anyhow!("Invalid mount point {}", mount_point));
@@ -594,6 +702,7 @@ pub fn get_mount_fs_type_from_file(mount_file: &str, mount_point: &str) -> Resul
     ))
 }
 
+#[instrument]
 pub fn get_cgroup_mounts(
     logger: &Logger,
     cg_path: &str,
@@ -684,6 +793,7 @@ pub fn get_cgroup_mounts(
     Ok(cg_mounts)
 }
 
+#[instrument]
 pub fn cgroups_mount(logger: &Logger, unified_cgroup_hierarchy: bool) -> Result<()> {
     let logger = logger.new(o!("subsystem" => "mount"));
 
@@ -699,6 +809,7 @@ pub fn cgroups_mount(logger: &Logger, unified_cgroup_hierarchy: bool) -> Result<
     Ok(())
 }
 
+#[instrument]
 pub fn remove_mounts(mounts: &[String]) -> Result<()> {
     for m in mounts.iter() {
         mount::umount(m.as_str()).context(format!("failed to umount {:?}", m))?;
@@ -708,6 +819,7 @@ pub fn remove_mounts(mounts: &[String]) -> Result<()> {
 
 // ensure_destination_exists will recursively create a given mountpoint. If directories
 // are created, their permissions are initialized to mountPerm(0755)
+#[instrument]
 fn ensure_destination_exists(destination: &str, fs_type: &str) -> Result<()> {
     let d = Path::new(destination);
     if !d.exists() {
@@ -728,6 +840,7 @@ fn ensure_destination_exists(destination: &str, fs_type: &str) -> Result<()> {
     Ok(())
 }
 
+#[instrument]
 fn parse_options(option_list: Vec<String>) -> HashMap<String, String> {
     let mut options = HashMap::new();
     for opt in option_list.iter() {
@@ -900,7 +1013,7 @@ mod tests {
             let msg = format!("{}: result: {:?}", msg, result);
 
             if d.error_contains.is_empty() {
-                assert!(result.is_ok(), msg);
+                assert!(result.is_ok(), "{}", msg);
 
                 // Cleanup
                 unsafe {
@@ -912,7 +1025,7 @@ mod tests {
 
                     let msg = format!("{}: umount result: {:?}", msg, result);
 
-                    assert!(ret == 0, msg);
+                    assert!(ret == 0, "{}", msg);
                 };
 
                 continue;
@@ -920,7 +1033,7 @@ mod tests {
 
             let err = result.unwrap_err();
             let error_msg = format!("{}", err);
-            assert!(error_msg.contains(d.error_contains), msg);
+            assert!(error_msg.contains(d.error_contains), "{}", msg);
         }
     }
 
@@ -1026,13 +1139,13 @@ mod tests {
             let msg = format!("{}: result: {:?}", msg, result);
 
             if d.error_contains.is_empty() {
-                assert!(result.is_ok(), msg);
+                assert!(result.is_ok(), "{}", msg);
                 continue;
             }
 
             let error_msg = format!("{:#}", result.unwrap_err());
 
-            assert!(error_msg.contains(d.error_contains), msg);
+            assert!(error_msg.contains(d.error_contains), "{}", msg);
         }
     }
 
@@ -1108,6 +1221,7 @@ mod tests {
 
             assert!(
                 format!("{}", err).contains("No such file or directory"),
+                "{}",
                 msg
             );
         }
@@ -1136,13 +1250,13 @@ mod tests {
             if d.error_contains.is_empty() {
                 let fs_type = result.unwrap();
 
-                assert!(d.fs_type == fs_type, msg);
+                assert!(d.fs_type == fs_type, "{}", msg);
 
                 continue;
             }
 
             let error_msg = format!("{}", result.unwrap_err());
-            assert!(error_msg.contains(d.error_contains), msg);
+            assert!(error_msg.contains(d.error_contains), "{}", msg);
         }
     }
 
@@ -1291,34 +1405,34 @@ mod tests {
             let msg = format!("{}: result: {:?}", msg, result);
 
             if !d.error_contains.is_empty() {
-                assert!(result.is_err(), msg);
+                assert!(result.is_err(), "{}", msg);
 
                 let error_msg = format!("{}", result.unwrap_err());
-                assert!(error_msg.contains(d.error_contains), msg);
+                assert!(error_msg.contains(d.error_contains), "{}", msg);
                 continue;
             }
 
-            assert!(result.is_ok(), msg);
+            assert!(result.is_ok(), "{}", msg);
 
             let mounts = result.unwrap();
             let count = mounts.len();
 
             if !d.devices_cgroup {
-                assert!(count == 0, msg);
+                assert!(count == 0, "{}", msg);
                 continue;
             }
 
             // get_cgroup_mounts() adds the device cgroup plus two other mounts.
-            assert!(count == (1 + 2), msg);
+            assert!(count == (1 + 2), "{}", msg);
 
             // First mount
-            assert!(mounts[0].eq(&first_mount), msg);
+            assert!(mounts[0].eq(&first_mount), "{}", msg);
 
             // Last mount
-            assert!(mounts[2].eq(&last_mount), msg);
+            assert!(mounts[2].eq(&last_mount), "{}", msg);
 
             // Devices cgroup
-            assert!(mounts[1].eq(&cg_devices_mount), msg);
+            assert!(mounts[1].eq(&cg_devices_mount), "{}", msg);
         }
     }
 }

src/agent/src/namespace.rs

@@ -11,6 +11,7 @@ use std::fmt;
 use std::fs;
 use std::fs::File;
 use std::path::{Path, PathBuf};
+use tracing::instrument;
 
 use crate::mount::{BareMount, FLAGS};
 use slog::Logger;
@@ -20,6 +21,7 @@ pub const NSTYPEIPC: &str = "ipc";
 pub const NSTYPEUTS: &str = "uts";
 pub const NSTYPEPID: &str = "pid";
 
+#[instrument]
 pub fn get_current_thread_ns_path(ns_type: &str) -> String {
     format!(
         "/proc/{}/task/{}/ns/{}",
@@ -40,31 +42,35 @@ pub struct Namespace {
 }
 
 impl Namespace {
+    #[instrument]
     pub fn new(logger: &Logger) -> Self {
         Namespace {
             logger: logger.clone(),
             path: String::from(""),
             persistent_ns_dir: String::from(PERSISTENT_NS_DIR),
-            ns_type: NamespaceType::IPC,
+            ns_type: NamespaceType::Ipc,
             hostname: None,
         }
     }
 
+    #[instrument]
     pub fn get_ipc(mut self) -> Self {
-        self.ns_type = NamespaceType::IPC;
+        self.ns_type = NamespaceType::Ipc;
         self
     }
 
+    #[instrument]
     pub fn get_uts(mut self, hostname: &str) -> Self {
-        self.ns_type = NamespaceType::UTS;
+        self.ns_type = NamespaceType::Uts;
         if !hostname.is_empty() {
             self.hostname = Some(String::from(hostname));
         }
         self
     }
 
+    #[instrument]
     pub fn get_pid(mut self) -> Self {
-        self.ns_type = NamespaceType::PID;
+        self.ns_type = NamespaceType::Pid;
         self
     }
 
@@ -76,12 +82,13 @@ impl Namespace {
 
     // setup creates persistent namespace without switching to it.
     // Note, pid namespaces cannot be persisted.
+    #[instrument]
     pub async fn setup(mut self) -> Result<Self> {
         fs::create_dir_all(&self.persistent_ns_dir)?;
 
         let ns_path = PathBuf::from(&self.persistent_ns_dir);
         let ns_type = self.ns_type;
-        if ns_type == NamespaceType::PID {
+        if ns_type == NamespaceType::Pid {
             return Err(anyhow!("Cannot persist namespace of PID type"));
         }
         let logger = self.logger.clone();
@@ -104,7 +111,7 @@ impl Namespace {
 
                 unshare(cf)?;
 
-                if ns_type == NamespaceType::UTS && hostname.is_some() {
+                if ns_type == NamespaceType::Uts && hostname.is_some() {
                     nix::unistd::sethostname(hostname.unwrap())?;
                 }
                 // Bind mount the new namespace from the current thread onto the mount point to persist it.
@@ -147,27 +154,27 @@ impl Namespace {
 /// Represents the Namespace type.
 #[derive(Clone, Copy, PartialEq)]
 enum NamespaceType {
-    IPC,
-    UTS,
-    PID,
+    Ipc,
+    Uts,
+    Pid,
 }
 
 impl NamespaceType {
     /// Get the string representation of the namespace type.
     pub fn get(&self) -> &str {
         match *self {
-            Self::IPC => "ipc",
-            Self::UTS => "uts",
-            Self::PID => "pid",
+            Self::Ipc => "ipc",
+            Self::Uts => "uts",
+            Self::Pid => "pid",
         }
     }
 
     /// Get the associate flags with the namespace type.
     pub fn get_flags(&self) -> CloneFlags {
         match *self {
-            Self::IPC => CloneFlags::CLONE_NEWIPC,
-            Self::UTS => CloneFlags::CLONE_NEWUTS,
-            Self::PID => CloneFlags::CLONE_NEWPID,
+            Self::Ipc => CloneFlags::CLONE_NEWIPC,
+            Self::Uts => CloneFlags::CLONE_NEWUTS,
+            Self::Pid => CloneFlags::CLONE_NEWPID,
         }
     }
 }
@@ -178,12 +185,6 @@ impl fmt::Debug for NamespaceType {
     }
 }
 
-impl Default for NamespaceType {
-    fn default() -> Self {
-        NamespaceType::IPC
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::{Namespace, NamespaceType};
@@ -234,15 +235,15 @@ mod tests {
 
     #[test]
     fn test_namespace_type() {
-        let ipc = NamespaceType::IPC;
+        let ipc = NamespaceType::Ipc;
         assert_eq!("ipc", ipc.get());
         assert_eq!(CloneFlags::CLONE_NEWIPC, ipc.get_flags());
 
-        let uts = NamespaceType::UTS;
+        let uts = NamespaceType::Uts;
         assert_eq!("uts", uts.get());
         assert_eq!(CloneFlags::CLONE_NEWUTS, uts.get_flags());
 
-        let pid = NamespaceType::PID;
+        let pid = NamespaceType::Pid;
         assert_eq!("pid", pid.get());
         assert_eq!(CloneFlags::CLONE_NEWPID, pid.get_flags());
     }

src/agent/src/netlink.rs

@@ -542,12 +542,10 @@ impl Handle {
                 ntype: NDA_UNSPEC as u8,
             },
             nlas: {
-                let mut nlas = vec![];
-
-                nlas.push(Nla::Destination(match ip {
+                let mut nlas = vec![Nla::Destination(match ip {
                     IpAddr::V4(v4) => v4.octets().to_vec(),
                     IpAddr::V6(v6) => v6.octets().to_vec(),
-                }));
+                })];
 
                 if !neigh.lladdr.is_empty() {
                     nlas.push(Nla::LinkLocalAddress(

src/agent/src/random.rs

@@ -9,6 +9,7 @@ use nix::fcntl::{self, OFlag};
 use nix::sys::stat::Mode;
 use std::fs;
 use std::os::unix::io::{AsRawFd, FromRawFd};
+use tracing::instrument;
 
 pub const RNGDEV: &str = "/dev/random";
 pub const RNDADDTOENTCNT: libc::c_int = 0x40045201;
@@ -20,6 +21,7 @@ type IoctlRequestType = libc::c_int;
 #[cfg(target_env = "gnu")]
 type IoctlRequestType = libc::c_ulong;
 
+#[instrument]
 pub fn reseed_rng(data: &[u8]) -> Result<()> {
     let len = data.len() as libc::c_long;
     fs::write(RNGDEV, data)?;
@@ -37,10 +39,10 @@ pub fn reseed_rng(data: &[u8]) -> Result<()> {
             &len as *const libc::c_long,
         )
     };
-    let _ = Errno::result(ret).map(drop)?;
+    Errno::result(ret).map(drop)?;
 
     let ret = unsafe { libc::ioctl(f.as_raw_fd(), RNDRESEEDRNG as IoctlRequestType, 0) };
-    let _ = Errno::result(ret).map(drop)?;
+    Errno::result(ret).map(drop)?;
 
     Ok(())
 }

src/agent/src/rpc.rs

@@ -20,9 +20,8 @@ use anyhow::{anyhow, Context, Result};
 use oci::{LinuxNamespace, Root, Spec};
 use protobuf::{RepeatedField, SingularPtrField};
 use protocols::agent::{
-    AgentDetails, CopyFileRequest, GuestDetailsResponse, Interfaces, ListProcessesResponse,
-    Metrics, OOMEvent, ReadStreamResponse, Routes, StatsContainerResponse, WaitProcessResponse,
-    WriteStreamResponse,
+    AgentDetails, CopyFileRequest, GuestDetailsResponse, Interfaces, Metrics, OOMEvent,
+    ReadStreamResponse, Routes, StatsContainerResponse, WaitProcessResponse, WriteStreamResponse,
 };
 use protocols::empty::Empty;
 use protocols::health::{
@@ -52,6 +51,14 @@ use crate::sandbox::Sandbox;
 use crate::version::{AGENT_VERSION, API_VERSION};
 use crate::AGENT_CONFIG;
 
+use crate::trace_rpc_call;
+use crate::tracer::extract_carrier_from_ttrpc;
+use opentelemetry::global;
+use tracing::span;
+use tracing_opentelemetry::OpenTelemetrySpanExt;
+
+use tracing::instrument;
+
 use libc::{self, c_ushort, pid_t, winsize, TIOCSWINSZ};
 use std::convert::TryFrom;
 use std::fs;
@@ -75,7 +82,7 @@ macro_rules! sl {
     };
 }
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct AgentService {
     sandbox: Arc<Mutex<Sandbox>>,
 }
@@ -98,13 +105,14 @@ fn verify_cid(id: &str) -> Result<()> {
 }
 
 impl AgentService {
+    #[instrument]
     async fn do_create_container(
         &self,
         req: protocols::agent::CreateContainerRequest,
     ) -> Result<()> {
         let cid = req.container_id.clone();
 
-        let _ = verify_cid(&cid)?;
+        verify_cid(&cid)?;
 
         let mut oci_spec = req.OCI.clone();
         let use_sandbox_pidns = req.get_sandbox_pidns();
@@ -197,6 +205,7 @@ impl AgentService {
         Ok(())
     }
 
+    #[instrument]
     async fn do_start_container(&self, req: protocols::agent::StartContainerRequest) -> Result<()> {
         let cid = req.container_id;
 
@@ -222,6 +231,7 @@ impl AgentService {
         Ok(())
     }
 
+    #[instrument]
     async fn do_remove_container(
         &self,
         req: protocols::agent::RemoveContainerRequest,
@@ -254,11 +264,14 @@ impl AgentService {
         if req.timeout == 0 {
             let s = Arc::clone(&self.sandbox);
             let mut sandbox = s.lock().await;
-            let ctr = sandbox
-                .get_container(&cid)
-                .ok_or_else(|| anyhow!("Invalid container id"))?;
 
-            ctr.destroy().await?;
+            sandbox.bind_watcher.remove_container(&cid).await;
+
+            sandbox
+                .get_container(&cid)
+                .ok_or_else(|| anyhow!("Invalid container id"))?
+                .destroy()
+                .await?;
 
             remove_container_resources(&mut sandbox)?;
 
@@ -274,6 +287,7 @@ impl AgentService {
             let mut sandbox = s.lock().await;
             if let Some(ctr) = sandbox.get_container(&cid2) {
                 ctr.destroy().await.unwrap();
+                sandbox.bind_watcher.remove_container(&cid2).await;
                 tx.send(1).unwrap();
             };
         });
@@ -299,6 +313,7 @@ impl AgentService {
         Ok(())
     }
 
+    #[instrument]
     async fn do_exec_process(&self, req: protocols::agent::ExecProcessRequest) -> Result<()> {
         let cid = req.container_id.clone();
         let exec_id = req.exec_id.clone();
@@ -327,6 +342,7 @@ impl AgentService {
         Ok(())
     }
 
+    #[instrument]
     async fn do_signal_process(&self, req: protocols::agent::SignalProcessRequest) -> Result<()> {
         let cid = req.container_id.clone();
         let eid = req.exec_id.clone();
@@ -361,6 +377,7 @@ impl AgentService {
         Ok(())
     }
 
+    #[instrument]
     async fn do_wait_process(
         &self,
         req: protocols::agent::WaitProcessRequest,
@@ -370,7 +387,6 @@ impl AgentService {
         let s = self.sandbox.clone();
         let mut resp = WaitProcessResponse::new();
         let pid: pid_t;
-        let stream;
 
         let (exit_send, mut exit_recv) = tokio::sync::mpsc::channel(100);
 
@@ -381,22 +397,20 @@ impl AgentService {
             "exec-id" => eid.clone()
         );
 
-        {
+        let exit_rx = {
             let mut sandbox = s.lock().await;
             let p = find_process(&mut sandbox, cid.as_str(), eid.as_str(), false)?;
 
-            stream = p.get_reader(StreamType::ExitPipeR);
-
             p.exit_watchers.push(exit_send);
             pid = p.pid;
-        }
 
-        if stream.is_some() {
-            info!(sl!(), "reading exit pipe");
+            p.exit_rx.clone()
+        };
 
-            let reader = stream.unwrap();
-            let mut content: Vec<u8> = vec![0, 1];
-            let _ = reader.lock().await.read(&mut content).await;
+        if let Some(mut exit_rx) = exit_rx {
+            info!(sl!(), "cid {} eid {} waiting for exit signal", &cid, &eid);
+            while exit_rx.changed().await.is_ok() {}
+            info!(sl!(), "cid {} eid {} received exit signal", &cid, &eid);
         }
 
         let mut sandbox = s.lock().await;
@@ -513,9 +527,10 @@ impl AgentService {
 impl protocols::agent_ttrpc::AgentService for AgentService {
     async fn create_container(
         &self,
-        _ctx: &TtrpcContext,
+        ctx: &TtrpcContext,
         req: protocols::agent::CreateContainerRequest,
     ) -> ttrpc::Result<Empty> {
+        trace_rpc_call!(ctx, "create_container", req);
         match self.do_create_container(req).await {
             Err(e) => Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string())),
             Ok(_) => Ok(Empty::new()),
@@ -524,9 +539,10 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
     async fn start_container(
         &self,
-        _ctx: &TtrpcContext,
+        ctx: &TtrpcContext,
         req: protocols::agent::StartContainerRequest,
     ) -> ttrpc::Result<Empty> {
+        trace_rpc_call!(ctx, "start_container", req);
         match self.do_start_container(req).await {
             Err(e) => Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string())),
             Ok(_) => Ok(Empty::new()),
@@ -535,9 +551,10 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
     async fn remove_container(
         &self,
-        _ctx: &TtrpcContext,
+        ctx: &TtrpcContext,
         req: protocols::agent::RemoveContainerRequest,
     ) -> ttrpc::Result<Empty> {
+        trace_rpc_call!(ctx, "remove_container", req);
         match self.do_remove_container(req).await {
             Err(e) => Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string())),
             Ok(_) => Ok(Empty::new()),
@@ -546,9 +563,10 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
     async fn exec_process(
         &self,
-        _ctx: &TtrpcContext,
+        ctx: &TtrpcContext,
         req: protocols::agent::ExecProcessRequest,
     ) -> ttrpc::Result<Empty> {
+        trace_rpc_call!(ctx, "exec_process", req);
         match self.do_exec_process(req).await {
             Err(e) => Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string())),
             Ok(_) => Ok(Empty::new()),
@@ -557,9 +575,10 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
     async fn signal_process(
         &self,
-        _ctx: &TtrpcContext,
+        ctx: &TtrpcContext,
         req: protocols::agent::SignalProcessRequest,
     ) -> ttrpc::Result<Empty> {
+        trace_rpc_call!(ctx, "signal_process", req);
         match self.do_signal_process(req).await {
             Err(e) => Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string())),
             Ok(_) => Ok(Empty::new()),
@@ -568,104 +587,21 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
     async fn wait_process(
         &self,
-        _ctx: &TtrpcContext,
+        ctx: &TtrpcContext,
         req: protocols::agent::WaitProcessRequest,
     ) -> ttrpc::Result<WaitProcessResponse> {
+        trace_rpc_call!(ctx, "wait_process", req);
         self.do_wait_process(req)
             .await
             .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))
     }
 
-    async fn list_processes(
-        &self,
-        _ctx: &TtrpcContext,
-        req: protocols::agent::ListProcessesRequest,
-    ) -> ttrpc::Result<ListProcessesResponse> {
-        let cid = req.container_id.clone();
-        let format = req.format.clone();
-        let mut args = req.args.into_vec();
-        let mut resp = ListProcessesResponse::new();
-
-        let s = Arc::clone(&self.sandbox);
-        let mut sandbox = s.lock().await;
-
-        let ctr = sandbox.get_container(&cid).ok_or_else(|| {
-            ttrpc_error(
-                ttrpc::Code::INVALID_ARGUMENT,
-                "invalid container id".to_string(),
-            )
-        })?;
-
-        let pids = ctr.processes().unwrap();
-
-        match format.as_str() {
-            "table" => {}
-            "json" => {
-                resp.process_list = serde_json::to_vec(&pids).unwrap();
-                return Ok(resp);
-            }
-            _ => {
-                return Err(ttrpc_error(
-                    ttrpc::Code::INVALID_ARGUMENT,
-                    "invalid format!".to_string(),
-                ));
-            }
-        }
-
-        // format "table"
-        if args.is_empty() {
-            // default argument
-            args = vec!["-ef".to_string()];
-        }
-
-        let output = tokio::process::Command::new("ps")
-            .args(args.as_slice())
-            .stdout(Stdio::piped())
-            .output()
-            .await
-            .expect("ps failed");
-
-        let out: String = String::from_utf8(output.stdout).unwrap();
-        let mut lines: Vec<String> = out.split('\n').map(|v| v.to_string()).collect();
-
-        let pid_index = lines[0]
-            .split_whitespace()
-            .position(|v| v == "PID")
-            .unwrap();
-
-        let mut result = String::new();
-        result.push_str(lines[0].as_str());
-
-        lines.remove(0);
-        for line in &lines {
-            if line.trim().is_empty() {
-                continue;
-            }
-
-            let fields: Vec<String> = line.split_whitespace().map(|v| v.to_string()).collect();
-
-            if fields.len() < pid_index + 1 {
-                warn!(sl!(), "corrupted output?");
-                continue;
-            }
-            let pid = fields[pid_index].trim().parse::<i32>().unwrap();
-
-            for p in &pids {
-                if pid == *p {
-                    result.push_str(line.as_str());
-                }
-            }
-        }
-
-        resp.process_list = Vec::from(result);
-        Ok(resp)
-    }
-
     async fn update_container(
         &self,
-        _ctx: &TtrpcContext,
+        ctx: &TtrpcContext,
         req: protocols::agent::UpdateContainerRequest,
     ) -> ttrpc::Result<Empty> {
+        trace_rpc_call!(ctx, "update_container", req);
         let cid = req.container_id.clone();
         let res = req.resources;
 
@@ -697,9 +633,10 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
     async fn stats_container(
         &self,
-        _ctx: &TtrpcContext,
+        ctx: &TtrpcContext,
         req: protocols::agent::StatsContainerRequest,
     ) -> ttrpc::Result<StatsContainerResponse> {
+        trace_rpc_call!(ctx, "stats_container", req);
         let cid = req.container_id;
         let s = Arc::clone(&self.sandbox);
         let mut sandbox = s.lock().await;
@@ -717,9 +654,10 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
     async fn pause_container(
         &self,
-        _ctx: &TtrpcContext,
+        ctx: &TtrpcContext,
         req: protocols::agent::PauseContainerRequest,
     ) -> ttrpc::Result<protocols::empty::Empty> {
+        trace_rpc_call!(ctx, "pause_container", req);
         let cid = req.get_container_id();
         let s = Arc::clone(&self.sandbox);
         let mut sandbox = s.lock().await;
@@ -739,9 +677,10 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
     async fn resume_container(
         &self,
-        _ctx: &TtrpcContext,
+        ctx: &TtrpcContext,
         req: protocols::agent::ResumeContainerRequest,
     ) -> ttrpc::Result<protocols::empty::Empty> {
+        trace_rpc_call!(ctx, "resume_container", req);
         let cid = req.get_container_id();
         let s = Arc::clone(&self.sandbox);
         let mut sandbox = s.lock().await;
@@ -791,9 +730,11 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
     async fn close_stdin(
         &self,
-        _ctx: &TtrpcContext,
+        ctx: &TtrpcContext,
         req: protocols::agent::CloseStdinRequest,
     ) -> ttrpc::Result<Empty> {
+        trace_rpc_call!(ctx, "close_stdin", req);
+
         let cid = req.container_id.clone();
         let eid = req.exec_id;
         let s = Arc::clone(&self.sandbox);
@@ -825,9 +766,11 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
     async fn tty_win_resize(
         &self,
-        _ctx: &TtrpcContext,
+        ctx: &TtrpcContext,
         req: protocols::agent::TtyWinResizeRequest,
     ) -> ttrpc::Result<Empty> {
+        trace_rpc_call!(ctx, "tty_win_resize", req);
+
         let cid = req.container_id.clone();
         let eid = req.exec_id.clone();
         let s = Arc::clone(&self.sandbox);
@@ -863,9 +806,11 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
     async fn update_interface(
         &self,
-        _ctx: &TtrpcContext,
+        ctx: &TtrpcContext,
         req: protocols::agent::UpdateInterfaceRequest,
     ) -> ttrpc::Result<Interface> {
+        trace_rpc_call!(ctx, "update_interface", req);
+
         let interface = req.interface.into_option().ok_or_else(|| {
             ttrpc_error(
                 ttrpc::Code::INVALID_ARGUMENT,
@@ -888,9 +833,11 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
     async fn update_routes(
         &self,
-        _ctx: &TtrpcContext,
+        ctx: &TtrpcContext,
         req: protocols::agent::UpdateRoutesRequest,
     ) -> ttrpc::Result<Routes> {
+        trace_rpc_call!(ctx, "update_routes", req);
+
         let new_routes = req
             .routes
             .into_option()
@@ -926,9 +873,11 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
     async fn list_interfaces(
         &self,
-        _ctx: &TtrpcContext,
-        _req: protocols::agent::ListInterfacesRequest,
+        ctx: &TtrpcContext,
+        req: protocols::agent::ListInterfacesRequest,
     ) -> ttrpc::Result<Interfaces> {
+        trace_rpc_call!(ctx, "list_interfaces", req);
+
         let list = self
             .sandbox
             .lock()
@@ -951,9 +900,11 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
     async fn list_routes(
         &self,
-        _ctx: &TtrpcContext,
-        _req: protocols::agent::ListRoutesRequest,
+        ctx: &TtrpcContext,
+        req: protocols::agent::ListRoutesRequest,
     ) -> ttrpc::Result<Routes> {
+        trace_rpc_call!(ctx, "list_routes", req);
+
         let list = self
             .sandbox
             .lock()
@@ -988,9 +939,11 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
     async fn create_sandbox(
         &self,
-        _ctx: &TtrpcContext,
+        ctx: &TtrpcContext,
         req: protocols::agent::CreateSandboxRequest,
     ) -> ttrpc::Result<Empty> {
+        trace_rpc_call!(ctx, "create_sandbox", req);
+
         {
             let sandbox = self.sandbox.clone();
             let mut s = sandbox.lock().await;
@@ -1015,7 +968,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
             }
 
             for m in req.kernel_modules.iter() {
-                let _ = load_kernel_module(m)
+                load_kernel_module(m)
                     .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
             }
 
@@ -1051,9 +1004,11 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
     async fn destroy_sandbox(
         &self,
-        _ctx: &TtrpcContext,
-        _req: protocols::agent::DestroySandboxRequest,
+        ctx: &TtrpcContext,
+        req: protocols::agent::DestroySandboxRequest,
     ) -> ttrpc::Result<Empty> {
+        trace_rpc_call!(ctx, "destroy_sandbox", req);
+
         let s = Arc::clone(&self.sandbox);
         let mut sandbox = s.lock().await;
         // destroy all containers, clean up, notify agent to exit
@@ -1070,9 +1025,11 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
     async fn add_arp_neighbors(
         &self,
-        _ctx: &TtrpcContext,
+        ctx: &TtrpcContext,
         req: protocols::agent::AddARPNeighborsRequest,
     ) -> ttrpc::Result<Empty> {
+        trace_rpc_call!(ctx, "add_arp_neighbors", req);
+
         let neighs = req
             .neighbors
             .into_option()
@@ -1102,11 +1059,12 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
     async fn online_cpu_mem(
         &self,
-        _ctx: &TtrpcContext,
+        ctx: &TtrpcContext,
         req: protocols::agent::OnlineCPUMemRequest,
     ) -> ttrpc::Result<Empty> {
         let s = Arc::clone(&self.sandbox);
         let sandbox = s.lock().await;
+        trace_rpc_call!(ctx, "online_cpu_mem", req);
 
         sandbox
             .online_cpu_memory(&req)
@@ -1117,9 +1075,11 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
     async fn reseed_random_dev(
         &self,
-        _ctx: &TtrpcContext,
+        ctx: &TtrpcContext,
         req: protocols::agent::ReseedRandomDevRequest,
     ) -> ttrpc::Result<Empty> {
+        trace_rpc_call!(ctx, "reseed_random_dev", req);
+
         random::reseed_rng(req.data.as_slice())
             .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
 
@@ -1128,9 +1088,11 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
     async fn get_guest_details(
         &self,
-        _ctx: &TtrpcContext,
+        ctx: &TtrpcContext,
         req: protocols::agent::GuestDetailsRequest,
     ) -> ttrpc::Result<GuestDetailsResponse> {
+        trace_rpc_call!(ctx, "get_guest_details", req);
+
         info!(sl!(), "get guest details!");
         let mut resp = GuestDetailsResponse::new();
         // to get memory block size
@@ -1154,9 +1116,11 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
     async fn mem_hotplug_by_probe(
         &self,
-        _ctx: &TtrpcContext,
+        ctx: &TtrpcContext,
         req: protocols::agent::MemHotplugByProbeRequest,
     ) -> ttrpc::Result<Empty> {
+        trace_rpc_call!(ctx, "mem_hotplug_by_probe", req);
+
         do_mem_hotplug_by_probe(&req.memHotplugProbeAddr)
             .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
 
@@ -1165,9 +1129,11 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
     async fn set_guest_date_time(
         &self,
-        _ctx: &TtrpcContext,
+        ctx: &TtrpcContext,
         req: protocols::agent::SetGuestDateTimeRequest,
     ) -> ttrpc::Result<Empty> {
+        trace_rpc_call!(ctx, "set_guest_date_time", req);
+
         do_set_guest_date_time(req.Sec, req.Usec)
             .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
 
@@ -1176,9 +1142,11 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
     async fn copy_file(
         &self,
-        _ctx: &TtrpcContext,
+        ctx: &TtrpcContext,
         req: protocols::agent::CopyFileRequest,
     ) -> ttrpc::Result<Empty> {
+        trace_rpc_call!(ctx, "copy_file", req);
+
         do_copy_file(&req).map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
 
         Ok(Empty::new())
@@ -1186,9 +1154,11 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
     async fn get_metrics(
         &self,
-        _ctx: &TtrpcContext,
+        ctx: &TtrpcContext,
         req: protocols::agent::GetMetricsRequest,
     ) -> ttrpc::Result<Metrics> {
+        trace_rpc_call!(ctx, "get_metrics", req);
+
         match get_metrics(&req) {
             Err(e) => Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string())),
             Ok(s) => {
@@ -1618,27 +1588,22 @@ fn setup_bundle(cid: &str, spec: &mut Spec) -> Result<PathBuf> {
 fn cleanup_process(p: &mut Process) -> Result<()> {
     if p.parent_stdin.is_some() {
         p.close_stream(StreamType::ParentStdin);
-        let _ = unistd::close(p.parent_stdin.unwrap())?;
+        unistd::close(p.parent_stdin.unwrap())?;
     }
 
     if p.parent_stdout.is_some() {
         p.close_stream(StreamType::ParentStdout);
-        let _ = unistd::close(p.parent_stdout.unwrap())?;
+        unistd::close(p.parent_stdout.unwrap())?;
     }
 
     if p.parent_stderr.is_some() {
         p.close_stream(StreamType::ParentStderr);
-        let _ = unistd::close(p.parent_stderr.unwrap())?;
+        unistd::close(p.parent_stderr.unwrap())?;
     }
 
     if p.term_master.is_some() {
         p.close_stream(StreamType::TermMaster);
-        let _ = unistd::close(p.term_master.unwrap())?;
-    }
-
-    if p.exit_pipe_r.is_some() {
-        p.close_stream(StreamType::ExitPipeR);
-        let _ = unistd::close(p.exit_pipe_r.unwrap())?;
+        unistd::close(p.term_master.unwrap())?;
     }
 
     p.notify_term_close();
@@ -2018,9 +1983,9 @@ mod tests {
             let msg = format!("{}, result: {:?}", msg, result);
 
             if result.is_ok() {
-                assert!(!d.expect_error, msg);
+                assert!(!d.expect_error, "{}", msg);
             } else {
-                assert!(d.expect_error, msg);
+                assert!(d.expect_error, "{}", msg);
             }
         }
     }

src/agent/src/sandbox.rs

@@ -8,6 +8,8 @@ use crate::mount::{get_mount_fs_type, remove_mounts, TYPE_ROOTFS};
 use crate::namespace::Namespace;
 use crate::netlink::Handle;
 use crate::network::Network;
+use crate::uevent::{Uevent, UeventMatcher};
+use crate::watcher::BindWatcher;
 use anyhow::{anyhow, Context, Result};
 use libc::pid_t;
 use oci::{Hook, Hooks};
@@ -25,7 +27,11 @@ use std::path::Path;
 use std::sync::Arc;
 use std::{thread, time};
 use tokio::sync::mpsc::{channel, Receiver, Sender};
+use tokio::sync::oneshot;
 use tokio::sync::Mutex;
+use tracing::instrument;
+
+type UeventWatcher = (Box<dyn UeventMatcher>, oneshot::Sender<Uevent>);
 
 #[derive(Debug)]
 pub struct Sandbox {
@@ -36,7 +42,8 @@ pub struct Sandbox {
     pub network: Network,
     pub mounts: Vec<String>,
     pub container_mounts: HashMap<String, Vec<String>>,
-    pub pci_device_map: HashMap<String, String>,
+    pub uevent_map: HashMap<String, Uevent>,
+    pub uevent_watchers: Vec<Option<UeventWatcher>>,
     pub shared_utsns: Namespace,
     pub shared_ipcns: Namespace,
     pub sandbox_pidns: Option<Namespace>,
@@ -48,9 +55,11 @@ pub struct Sandbox {
     pub hooks: Option<Hooks>,
     pub event_rx: Arc<Mutex<Receiver<String>>>,
     pub event_tx: Option<Sender<String>>,
+    pub bind_watcher: BindWatcher,
 }
 
 impl Sandbox {
+    #[instrument]
     pub fn new(logger: &Logger) -> Result<Self> {
         let fs_type = get_mount_fs_type("/")?;
         let logger = logger.new(o!("subsystem" => "sandbox"));
@@ -65,7 +74,8 @@ impl Sandbox {
             containers: HashMap::new(),
             mounts: Vec::new(),
             container_mounts: HashMap::new(),
-            pci_device_map: HashMap::new(),
+            uevent_map: HashMap::new(),
+            uevent_watchers: Vec::new(),
             shared_utsns: Namespace::new(&logger),
             shared_ipcns: Namespace::new(&logger),
             sandbox_pidns: None,
@@ -77,6 +87,7 @@ impl Sandbox {
             hooks: None,
             event_rx,
             event_tx: Some(tx),
+            bind_watcher: BindWatcher::new(),
         })
     }
 
@@ -88,6 +99,7 @@ impl Sandbox {
     //
     // It's assumed that caller is calling this method after
     // acquiring a lock on sandbox.
+    #[instrument]
     pub fn set_sandbox_storage(&mut self, path: &str) -> bool {
         match self.storages.get_mut(path) {
             None => {
@@ -110,6 +122,7 @@ impl Sandbox {
     //
     // It's assumed that caller is calling this method after
     // acquiring a lock on sandbox.
+    #[instrument]
     pub fn unset_sandbox_storage(&mut self, path: &str) -> Result<bool> {
         match self.storages.get_mut(path) {
             None => Err(anyhow!("Sandbox storage with path {} not found", path)),
@@ -129,6 +142,7 @@ impl Sandbox {
     //
     // It's assumed that caller is calling this method after
     // acquiring a lock on sandbox.
+    #[instrument]
     pub fn remove_sandbox_storage(&self, path: &str) -> Result<()> {
         let mounts = vec![path.to_string()];
         remove_mounts(&mounts)?;
@@ -142,6 +156,7 @@ impl Sandbox {
     //
     // It's assumed that caller is calling this method after
     // acquiring a lock on sandbox.
+    #[instrument]
     pub fn unset_and_remove_sandbox_storage(&mut self, path: &str) -> Result<()> {
         if self.unset_sandbox_storage(path)? {
             return self.remove_sandbox_storage(path);
@@ -150,6 +165,7 @@ impl Sandbox {
         Ok(())
     }
 
+    #[instrument]
     pub async fn setup_shared_namespaces(&mut self) -> Result<bool> {
         // Set up shared IPC namespace
         self.shared_ipcns = Namespace::new(&self.logger)
@@ -172,6 +188,7 @@ impl Sandbox {
         self.containers.insert(c.id.clone(), c);
     }
 
+    #[instrument]
     pub fn update_shared_pidns(&mut self, c: &LinuxContainer) -> Result<()> {
         // Populate the shared pid path only if this is an infra container and
         // sandbox_pidns has not been passed in the create_sandbox request.
@@ -209,6 +226,7 @@ impl Sandbox {
         None
     }
 
+    #[instrument]
     pub async fn destroy(&mut self) -> Result<()> {
         for ctr in self.containers.values_mut() {
             ctr.destroy().await?;
@@ -216,6 +234,7 @@ impl Sandbox {
         Ok(())
     }
 
+    #[instrument]
     pub fn online_cpu_memory(&self, req: &OnlineCPUMemRequest) -> Result<()> {
         if req.nb_cpus > 0 {
             // online cpus
@@ -259,6 +278,7 @@ impl Sandbox {
         Ok(())
     }
 
+    #[instrument]
     pub fn add_hooks(&mut self, dir: &str) -> Result<()> {
         let mut hooks = Hooks::default();
         if let Ok(hook) = self.find_hooks(dir, "prestart") {
@@ -274,6 +294,7 @@ impl Sandbox {
         Ok(())
     }
 
+    #[instrument]
     fn find_hooks(&self, hook_path: &str, hook_type: &str) -> Result<Vec<Hook>> {
         let mut hooks = Vec::new();
         for entry in fs::read_dir(Path::new(hook_path).join(hook_type))? {
@@ -310,6 +331,7 @@ impl Sandbox {
         Ok(hooks)
     }
 
+    #[instrument]
     pub async fn run_oom_event_monitor(&self, mut rx: Receiver<String>, container_id: String) {
         let logger = self.logger.clone();
 
@@ -342,6 +364,7 @@ impl Sandbox {
     }
 }
 
+#[instrument]
 fn online_resources(logger: &Logger, path: &str, pattern: &str, num: i32) -> Result<i32> {
     let mut count = 0;
     let re = Regex::new(pattern)?;
@@ -387,6 +410,7 @@ fn online_resources(logger: &Logger, path: &str, pattern: &str, num: i32) -> Res
 const ONLINE_CPUMEM_WATI_MILLIS: u64 = 50;
 const ONLINE_CPUMEM_MAX_RETRIES: u32 = 100;
 
+#[instrument]
 fn online_cpus(logger: &Logger, num: i32) -> Result<i32> {
     let mut onlined_count: i32 = 0;
 
@@ -416,6 +440,7 @@ fn online_cpus(logger: &Logger, num: i32) -> Result<i32> {
     ))
 }
 
+#[instrument]
 fn online_memory(logger: &Logger) -> Result<()> {
     online_resources(logger, SYSFS_MEMORY_ONLINE_PATH, r"memory[0-9]+", -1)?;
     Ok(())
@@ -531,7 +556,6 @@ mod tests {
     }
 
     #[tokio::test]
-    #[allow(unused_assignments)]
     async fn unset_and_remove_sandbox_storage() {
         skip_if_not_root!();
 
@@ -565,7 +589,7 @@ mod tests {
         assert_eq!(s.set_sandbox_storage(&destdir_path), true);
         assert!(s.unset_and_remove_sandbox_storage(&destdir_path).is_ok());
 
-        let mut other_dir_str = String::new();
+        let other_dir_str;
         {
             // Create another folder in a separate scope to ensure that is
             // deleted

src/agent/src/signal.rs

@@ -6,10 +6,10 @@
 
 use crate::sandbox::Sandbox;
 use anyhow::{anyhow, Result};
+use capctl::prctl::set_subreaper;
 use nix::sys::wait::WaitPidFlag;
 use nix::sys::wait::{self, WaitStatus};
 use nix::unistd;
-use prctl::set_child_subreaper;
 use slog::{error, info, o, Logger};
 use std::sync::Arc;
 use tokio::select;
@@ -22,6 +22,9 @@ async fn handle_sigchild(logger: Logger, sandbox: Arc<Mutex<Sandbox>>) -> Result
     info!(logger, "handling signal"; "signal" => "SIGCHLD");
 
     loop {
+        // Avoid reaping the undesirable child's signal, e.g., execute_hook's
+        // The lock should be released immediately.
+        rustjail::container::WAIT_PID_LOCKER.lock().await;
         let result = wait::waitpid(
             Some(Pid::from_raw(-1)),
             Some(WaitPidFlag::WNOHANG | WaitPidFlag::__WALL),
@@ -55,13 +58,6 @@ async fn handle_sigchild(logger: Logger, sandbox: Arc<Mutex<Sandbox>>) -> Result
             }
 
             let mut p = process.unwrap();
-
-            if p.exit_pipe_w.is_none() {
-                info!(logger, "process exit pipe not set");
-                continue;
-            }
-
-            let pipe_write = p.exit_pipe_w.unwrap();
             let ret: i32;
 
             match wait_status {
@@ -75,7 +71,7 @@ async fn handle_sigchild(logger: Logger, sandbox: Arc<Mutex<Sandbox>>) -> Result
             }
 
             p.exit_code = ret;
-            let _ = unistd::close(pipe_write);
+            let _ = p.exit_tx.take();
 
             info!(logger, "notify term to close");
             // close the socket file to notify readStdio to close terminal specifically
@@ -92,7 +88,7 @@ pub async fn setup_signal_handler(
 ) -> Result<()> {
     let logger = logger.new(o!("subsystem" => "signals"));
 
-    set_child_subreaper(true)
+    set_subreaper(true)
         .map_err(|err| anyhow!(err).context("failed to setup agent as a child subreaper"))?;
 
     let mut sigchild_stream = signal(SignalKind::child())?;

src/agent/src/test_utils.rs

@@ -7,7 +7,6 @@
 #[cfg(test)]
 mod test_utils {
     #[macro_export]
-    #[allow(unused_macros)]
     macro_rules! skip_if_root {
         () => {
             if nix::unistd::Uid::effective().is_root() {
@@ -18,7 +17,6 @@ mod test_utils {
     }
 
     #[macro_export]
-    #[allow(unused_macros)]
     macro_rules! skip_if_not_root {
         () => {
             if !nix::unistd::Uid::effective().is_root() {
@@ -29,7 +27,6 @@ mod test_utils {
     }
 
     #[macro_export]
-    #[allow(unused_macros)]
     macro_rules! skip_loop_if_root {
         ($msg:expr) => {
             if nix::unistd::Uid::effective().is_root() {
@@ -44,7 +41,6 @@ mod test_utils {
     }
 
     #[macro_export]
-    #[allow(unused_macros)]
     macro_rules! skip_loop_if_not_root {
         ($msg:expr) => {
             if !nix::unistd::Uid::effective().is_root() {

src/agent/src/tracer.rs

@@ -0,0 +1,122 @@
+// Copyright (c) 2020-2021 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use crate::config::AgentConfig;
+use anyhow::Result;
+use opentelemetry::sdk::propagation::TraceContextPropagator;
+use opentelemetry::{global, sdk::trace::Config, trace::TracerProvider};
+use slog::{info, o, Logger};
+use std::collections::HashMap;
+use std::error::Error;
+use std::fmt;
+use std::str::FromStr;
+use tracing_opentelemetry::OpenTelemetryLayer;
+use tracing_subscriber::layer::SubscriberExt;
+use tracing_subscriber::Registry;
+use ttrpc::r#async::TtrpcContext;
+
+#[derive(Debug, PartialEq)]
+pub enum TraceType {
+    Disabled,
+    Isolated,
+}
+
+#[derive(Debug)]
+pub struct TraceTypeError {
+    details: String,
+}
+
+impl TraceTypeError {
+    fn new(msg: &str) -> TraceTypeError {
+        TraceTypeError {
+            details: msg.into(),
+        }
+    }
+}
+
+impl Error for TraceTypeError {}
+
+impl fmt::Display for TraceTypeError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.details)
+    }
+}
+
+impl FromStr for TraceType {
+    type Err = TraceTypeError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "isolated" => Ok(TraceType::Isolated),
+            "disabled" => Ok(TraceType::Disabled),
+            _ => Err(TraceTypeError::new("invalid trace type")),
+        }
+    }
+}
+
+pub fn setup_tracing(name: &'static str, logger: &Logger, _agent_cfg: &AgentConfig) -> Result<()> {
+    let logger = logger.new(o!("subsystem" => "vsock-tracer"));
+
+    let exporter = vsock_exporter::Exporter::builder()
+        .with_logger(&logger)
+        .init();
+
+    let config = Config::default();
+
+    let builder = opentelemetry::sdk::trace::TracerProvider::builder()
+        .with_simple_exporter(exporter)
+        .with_config(config);
+
+    let provider = builder.build();
+
+    // We don't need a versioned tracer.
+    let version = None;
+
+    let tracer = provider.get_tracer(name, version);
+
+    let _global_provider = global::set_tracer_provider(provider);
+
+    let layer = OpenTelemetryLayer::new(tracer);
+
+    let subscriber = Registry::default().with(layer);
+
+    tracing::subscriber::set_global_default(subscriber)?;
+
+    global::set_text_map_propagator(TraceContextPropagator::new());
+
+    info!(logger, "tracing setup");
+
+    Ok(())
+}
+
+pub fn end_tracing() {
+    global::shutdown_tracer_provider();
+}
+
+pub fn extract_carrier_from_ttrpc(ttrpc_context: &TtrpcContext) -> HashMap<String, String> {
+    let mut carrier = HashMap::new();
+    for (k, v) in &ttrpc_context.metadata {
+        carrier.insert(k.clone(), v.join(","));
+    }
+
+    carrier
+}
+
+#[macro_export]
+macro_rules! trace_rpc_call {
+    ($ctx: ident, $name:literal, $req: ident) => {
+        // extract context from request context
+        let parent_context = global::get_text_map_propagator(|propagator| {
+            propagator.extract(&extract_carrier_from_ttrpc($ctx))
+        });
+
+        // generate tracing span
+        let rpc_span = span!(tracing::Level::INFO, $name, "mod"="rpc.rs", req=?$req);
+
+        // assign parent span from external context
+        rpc_span.set_parent(parent_context);
+        let _enter = rpc_span.enter();
+    };
+}

src/agent/src/uevent.rs

@@ -6,26 +6,39 @@
 use crate::device::online_device;
 use crate::linux_abi::*;
 use crate::sandbox::Sandbox;
-use crate::GLOBAL_DEVICE_WATCHER;
+use crate::AGENT_CONFIG;
 use slog::Logger;
 
-use anyhow::Result;
+use anyhow::{anyhow, Result};
 use netlink_sys::{protocols, SocketAddr, TokioSocket};
 use nix::errno::Errno;
+use std::fmt::Debug;
 use std::os::unix::io::FromRawFd;
 use std::sync::Arc;
 use tokio::select;
 use tokio::sync::watch::Receiver;
 use tokio::sync::Mutex;
+use tracing::instrument;
 
-#[derive(Debug, Default)]
-struct Uevent {
-    action: String,
-    devpath: String,
-    devname: String,
-    subsystem: String,
+// Convenience macro to obtain the scope logger
+macro_rules! sl {
+    () => {
+        slog_scope::logger().new(o!("subsystem" => "uevent"))
+    };
+}
+
+#[derive(Debug, Default, Clone, PartialEq, Eq)]
+pub struct Uevent {
+    pub action: String,
+    pub devpath: String,
+    pub devname: String,
+    pub subsystem: String,
     seqnum: String,
-    interface: String,
+    pub interface: String,
+}
+
+pub trait UeventMatcher: Sync + Send + Debug + 'static {
+    fn is_match(&self, uev: &Uevent) -> bool;
 }
 
 impl Uevent {
@@ -52,89 +65,91 @@ impl Uevent {
         event
     }
 
-    // Check whether this is a block device hot-add event.
-    fn is_block_add_event(&self) -> bool {
-        let pci_root_bus_path = create_pci_root_bus_path();
-        self.action == U_EVENT_ACTION_ADD
-            && self.subsystem == "block"
-            && {
-                self.devpath.starts_with(pci_root_bus_path.as_str())
-                    || self.devpath.starts_with(ACPI_DEV_PATH) // NVDIMM/PMEM devices
-            }
-            && !self.devname.is_empty()
-    }
+    #[instrument]
+    async fn process_add(&self, logger: &Logger, sandbox: &Arc<Mutex<Sandbox>>) {
+        // Special case for memory hot-adds first
+        let online_path = format!("{}/{}/online", SYSFS_DIR, &self.devpath);
+        if online_path.starts_with(SYSFS_MEMORY_ONLINE_PATH) {
+            let _ = online_device(online_path.as_ref()).map_err(|e| {
+                error!(
+                    *logger,
+                    "failed to online device";
+                    "device" => &self.devpath,
+                    "error" => format!("{}", e),
+                )
+            });
+            return;
+        }
 
-    async fn handle_block_add_event(&self, sandbox: &Arc<Mutex<Sandbox>>) {
-        let pci_root_bus_path = create_pci_root_bus_path();
-
-        // Keep the same lock order as device::get_device_name(), otherwise it may cause deadlock.
-        let watcher = GLOBAL_DEVICE_WATCHER.clone();
-        let mut w = watcher.lock().await;
         let mut sb = sandbox.lock().await;
 
-        // Add the device node name to the pci device map.
-        sb.pci_device_map
-            .insert(self.devpath.clone(), self.devname.clone());
+        // Record the event by sysfs path
+        sb.uevent_map.insert(self.devpath.clone(), self.clone());
 
         // Notify watchers that are interested in the udev event.
-        // Close the channel after watcher has been notified.
-        let devpath = self.devpath.clone();
-        let empties: Vec<_> = w
-            .iter_mut()
-            .filter(|(dev_addr, _)| {
-                let pci_p = format!("{}/{}", pci_root_bus_path, *dev_addr);
-
-                // blk block device
-                devpath.starts_with(pci_p.as_str()) ||
-                // scsi block device
-                {
-                    (*dev_addr).ends_with(SCSI_BLOCK_SUFFIX) &&
-                        devpath.contains(*dev_addr)
-                } ||
-                // nvdimm/pmem device
-                {
-                    let pmem_suffix = format!("/{}/{}", SCSI_BLOCK_SUFFIX, self.devname);
-                    devpath.starts_with(ACPI_DEV_PATH) &&
-                        devpath.ends_with(pmem_suffix.as_str()) &&
-                        dev_addr.ends_with(pmem_suffix.as_str())
+        for watch in &mut sb.uevent_watchers {
+            if let Some((matcher, _)) = watch {
+                if matcher.is_match(&self) {
+                    let (_, sender) = watch.take().unwrap();
+                    let _ = sender.send(self.clone());
                 }
-            })
-            .map(|(k, sender)| {
-                let devname = self.devname.clone();
-                let sender = sender.take().unwrap();
-                let _ = sender.send(devname);
-                k.clone()
-            })
-            .collect();
-
-        // Remove notified nodes from the watcher map.
-        for empty in empties {
-            w.remove(&empty);
+            }
         }
     }
 
+    #[instrument]
     async fn process(&self, logger: &Logger, sandbox: &Arc<Mutex<Sandbox>>) {
-        if self.is_block_add_event() {
-            return self.handle_block_add_event(sandbox).await;
-        } else if self.action == U_EVENT_ACTION_ADD {
-            let online_path = format!("{}/{}/online", SYSFS_DIR, &self.devpath);
-            // It's a memory hot-add event.
-            if online_path.starts_with(SYSFS_MEMORY_ONLINE_PATH) {
-                let _ = online_device(online_path.as_ref()).map_err(|e| {
-                    error!(
-                        *logger,
-                        "failed to online device";
-                        "device" => &self.devpath,
-                        "error" => format!("{}", e),
-                    )
-                });
-                return;
-            }
+        if self.action == U_EVENT_ACTION_ADD {
+            return self.process_add(logger, sandbox).await;
         }
         debug!(*logger, "ignoring event"; "uevent" => format!("{:?}", self));
     }
 }
 
+#[instrument]
+pub async fn wait_for_uevent(
+    sandbox: &Arc<Mutex<Sandbox>>,
+    matcher: impl UeventMatcher,
+) -> Result<Uevent> {
+    let mut sb = sandbox.lock().await;
+    for uev in sb.uevent_map.values() {
+        if matcher.is_match(uev) {
+            info!(sl!(), "Device {:?} found in device map", uev);
+            return Ok(uev.clone());
+        }
+    }
+
+    // If device is not found in the device map, hotplug event has not
+    // been received yet, create and add channel to the watchers map.
+    // The key of the watchers map is the device we are interested in.
+    // Note this is done inside the lock, not to miss any events from the
+    // global udev listener.
+    let (tx, rx) = tokio::sync::oneshot::channel::<Uevent>();
+    let idx = sb.uevent_watchers.len();
+    sb.uevent_watchers.push(Some((Box::new(matcher), tx)));
+    drop(sb); // unlock
+
+    info!(sl!(), "Waiting on channel for uevent notification\n");
+    let hotplug_timeout = AGENT_CONFIG.read().await.hotplug_timeout;
+
+    let uev = match tokio::time::timeout(hotplug_timeout, rx).await {
+        Ok(v) => v?,
+        Err(_) => {
+            let mut sb = sandbox.lock().await;
+            let matcher = sb.uevent_watchers[idx].take().unwrap().0;
+
+            return Err(anyhow!(
+                "Timeout after {:?} waiting for uevent {:?}",
+                hotplug_timeout,
+                &matcher
+            ));
+        }
+    };
+
+    Ok(uev)
+}
+
+#[instrument]
 pub async fn watch_uevents(
     sandbox: Arc<Mutex<Sandbox>>,
     mut shutdown: Receiver<bool>,
@@ -199,3 +214,71 @@ pub async fn watch_uevents(
 
     Ok(())
 }
+
+// Used in the device module unit tests
+#[cfg(test)]
+pub(crate) fn spawn_test_watcher(sandbox: Arc<Mutex<Sandbox>>, uev: Uevent) {
+    tokio::spawn(async move {
+        loop {
+            let mut sb = sandbox.lock().await;
+            for w in &mut sb.uevent_watchers {
+                if let Some((matcher, _)) = w {
+                    if matcher.is_match(&uev) {
+                        let (_, sender) = w.take().unwrap();
+                        let _ = sender.send(uev);
+                        return;
+                    }
+                }
+            }
+            drop(sb); // unlock
+        }
+    });
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[derive(Debug, Clone, Copy)]
+    struct AlwaysMatch();
+
+    impl UeventMatcher for AlwaysMatch {
+        fn is_match(&self, _: &Uevent) -> bool {
+            true
+        }
+    }
+
+    #[tokio::test]
+    async fn test_wait_for_uevent() {
+        let uev = Uevent {
+            action: crate::linux_abi::U_EVENT_ACTION_ADD.to_string(),
+            subsystem: "test".to_string(),
+            devpath: "/test/sysfs/path".to_string(),
+            devname: "testdevname".to_string(),
+            ..Default::default()
+        };
+
+        let matcher = AlwaysMatch();
+
+        let logger = slog::Logger::root(slog::Discard, o!());
+        let sandbox = Arc::new(Mutex::new(Sandbox::new(&logger).unwrap()));
+
+        let mut sb = sandbox.lock().await;
+        sb.uevent_map.insert(uev.devpath.clone(), uev.clone());
+        drop(sb); // unlock
+
+        let uev2 = wait_for_uevent(&sandbox, matcher).await;
+        assert!(uev2.is_ok());
+        assert_eq!(uev2.unwrap(), uev);
+
+        let mut sb = sandbox.lock().await;
+        sb.uevent_map.remove(&uev.devpath).unwrap();
+        drop(sb); // unlock
+
+        spawn_test_watcher(sandbox.clone(), uev.clone());
+
+        let uev2 = wait_for_uevent(&sandbox, matcher).await;
+        assert!(uev2.is_ok());
+        assert_eq!(uev2.unwrap(), uev);
+    }
+}

src/agent/src/util.rs

@@ -3,10 +3,15 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+use anyhow::Result;
+use futures::StreamExt;
 use std::io;
 use std::io::ErrorKind;
+use std::os::unix::io::{FromRawFd, RawFd};
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::sync::watch::Receiver;
+use tokio_vsock::{Incoming, VsockListener, VsockStream};
+use tracing::instrument;
 
 // Size of I/O read buffer
 const BUF_SIZE: usize = 8192;
@@ -52,6 +57,17 @@ where
     Ok(total_bytes)
 }
 
+#[instrument]
+pub fn get_vsock_incoming(fd: RawFd) -> Incoming {
+    unsafe { VsockListener::from_raw_fd(fd).incoming() }
+}
+
+#[instrument]
+pub async fn get_vsock_stream(fd: RawFd) -> Result<VsockStream> {
+    let stream = get_vsock_incoming(fd).next().await.unwrap()?;
+    Ok(stream)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/agent/src/watcher.rs

@@ -0,0 +1,771 @@
+// Copyright (c) 2021 Apple Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+use std::time::SystemTime;
+
+use tokio::fs;
+use tokio::sync::Mutex;
+use tokio::task;
+use tokio::time::{self, Duration};
+
+use anyhow::{ensure, Context, Result};
+use async_recursion::async_recursion;
+use nix::mount::{umount, MsFlags};
+use slog::{debug, error, Logger};
+
+use crate::mount::BareMount;
+use crate::protocols::agent as protos;
+
+/// The maximum number of file system entries agent will watch for each mount.
+const MAX_ENTRIES_PER_STORAGE: usize = 8;
+
+/// The maximum size of a watchable mount in bytes.
+const MAX_SIZE_PER_WATCHABLE_MOUNT: u64 = 1024 * 1024;
+
+/// How often to check for modified files.
+const WATCH_INTERVAL_SECS: u64 = 2;
+
+/// Destination path for tmpfs
+const WATCH_MOUNT_POINT_PATH: &str = "/run/kata-containers/shared/containers/watchable/";
+
+/// Represents a single watched storage entry which may have multiple files to watch.
+#[derive(Default, Debug, Clone)]
+struct Storage {
+    /// A mount point without inotify capabilities.
+    source_mount_point: PathBuf,
+
+    /// The target mount point, where the watched files will be copied/mirrored
+    /// when being changed, added or removed. This will be subdirectory of a tmpfs
+    target_mount_point: PathBuf,
+
+    /// Flag to indicate that the Storage should be watched. Storage will be watched until
+    /// the source becomes too large, either in number of files (>8) or total size (>1MB).
+    watch: bool,
+
+    /// The list of files to watch from the source mount point and updated in the target one.
+    watched_files: HashMap<PathBuf, SystemTime>,
+}
+
+impl Drop for Storage {
+    fn drop(&mut self) {
+        let _ = std::fs::remove_dir_all(&self.target_mount_point);
+    }
+}
+
+impl Storage {
+    async fn new(storage: protos::Storage) -> Result<Storage> {
+        let entry = Storage {
+            source_mount_point: PathBuf::from(&storage.source),
+            target_mount_point: PathBuf::from(&storage.mount_point),
+            watch: true,
+            watched_files: HashMap::new(),
+        };
+
+        Ok(entry)
+    }
+
+    async fn update_target(&self, logger: &Logger, source_path: impl AsRef<Path>) -> Result<()> {
+        let source_file_path = source_path.as_ref();
+
+        let dest_file_path = if self.source_mount_point.is_file() {
+            // Simple file to file copy
+            // Assume target mount is a file path
+            self.target_mount_point.clone()
+        } else {
+            let dest_file_path = self.make_target_path(&source_file_path)?;
+
+            if let Some(path) = dest_file_path.parent() {
+                debug!(logger, "Creating destination directory: {}", path.display());
+                fs::create_dir_all(path)
+                    .await
+                    .with_context(|| format!("Unable to mkdir all for {}", path.display()))?;
+            }
+
+            dest_file_path
+        };
+
+        debug!(
+            logger,
+            "Copy from {} to {}",
+            source_file_path.display(),
+            dest_file_path.display()
+        );
+        fs::copy(&source_file_path, &dest_file_path)
+            .await
+            .with_context(|| {
+                format!(
+                    "Copy from {} to {} failed",
+                    source_file_path.display(),
+                    dest_file_path.display()
+                )
+            })?;
+
+        Ok(())
+    }
+
+    async fn scan(&mut self, logger: &Logger) -> Result<usize> {
+        debug!(logger, "Scanning for changes");
+
+        let mut remove_list = Vec::new();
+        let mut updated_files: Vec<PathBuf> = Vec::new();
+
+        // Remove deleted files for tracking list
+        self.watched_files.retain(|st, _| {
+            if st.exists() {
+                true
+            } else {
+                remove_list.push(st.to_path_buf());
+                false
+            }
+        });
+
+        // Delete from target
+        for path in remove_list {
+            // File has been deleted, remove it from target mount
+            let target = self.make_target_path(path)?;
+            debug!(logger, "Removing file from mount: {}", target.display());
+            let _ = fs::remove_file(target).await;
+        }
+
+        // Scan new & changed files
+        self.scan_path(
+            logger,
+            self.source_mount_point.clone().as_path(),
+            &mut updated_files,
+        )
+        .await
+        .with_context(|| "Scan path failed")?;
+
+        // Update identified files:
+        for path in &updated_files {
+            self.update_target(logger, path.as_path()).await?;
+        }
+
+        Ok(updated_files.len())
+    }
+
+    #[async_recursion]
+    async fn scan_path(
+        &mut self,
+        logger: &Logger,
+        path: &Path,
+        update_list: &mut Vec<PathBuf>,
+    ) -> Result<u64> {
+        let mut size: u64 = 0;
+        debug!(logger, "Scanning path: {}", path.display());
+
+        if path.is_file() {
+            let metadata = path
+                .metadata()
+                .with_context(|| format!("Failed to query metadata for: {}", path.display()))?;
+
+            let modified = metadata
+                .modified()
+                .with_context(|| format!("Failed to get modified date for: {}", path.display()))?;
+
+            size += metadata.len();
+
+            ensure!(
+                self.watched_files.len() <= MAX_ENTRIES_PER_STORAGE,
+                "Too many file system entries to watch (must be < {})",
+                MAX_ENTRIES_PER_STORAGE
+            );
+
+            // Insert will return old entry if any
+            if let Some(old_st) = self.watched_files.insert(path.to_path_buf(), modified) {
+                if modified > old_st {
+                    update_list.push(PathBuf::from(&path))
+                }
+            } else {
+                // Storage just added, copy to target
+                debug!(logger, "New entry: {}", path.display());
+                update_list.push(PathBuf::from(&path))
+            }
+        } else {
+            // Scan dir recursively
+            let mut entries = fs::read_dir(path)
+                .await
+                .with_context(|| format!("Failed to read dir: {}", path.display()))?;
+
+            while let Some(entry) = entries.next_entry().await? {
+                let path = entry.path();
+                let res_size = self
+                    .scan_path(logger, path.as_path(), update_list)
+                    .await
+                    .with_context(|| format!("Unable to scan inner path: {}", path.display()))?;
+                size += res_size;
+            }
+        }
+        ensure!(
+            size <= MAX_SIZE_PER_WATCHABLE_MOUNT,
+            "Too many file system entries to watch (must be < {})",
+            MAX_SIZE_PER_WATCHABLE_MOUNT,
+        );
+
+        Ok(size)
+    }
+
+    fn make_target_path(&self, source_file_path: impl AsRef<Path>) -> Result<PathBuf> {
+        let relative_path = source_file_path
+            .as_ref()
+            .strip_prefix(&self.source_mount_point)
+            .with_context(|| {
+                format!(
+                    "Failed to strip prefix: {} - {}",
+                    source_file_path.as_ref().display().to_string(),
+                    &self.source_mount_point.display()
+                )
+            })?;
+
+        let dest_file_path = Path::new(&self.target_mount_point).join(relative_path);
+        Ok(dest_file_path)
+    }
+}
+
+#[derive(Default, Debug)]
+struct SandboxStorages(Vec<Storage>);
+
+impl SandboxStorages {
+    async fn add(
+        &mut self,
+        list: impl IntoIterator<Item = protos::Storage>,
+
+        logger: &Logger,
+    ) -> Result<()> {
+        for storage in list.into_iter() {
+            let entry = Storage::new(storage)
+                .await
+                .with_context(|| "Failed to add storage")?;
+            self.0.push(entry);
+        }
+
+        // Perform initial copy
+        self.check(logger)
+            .await
+            .with_context(|| "Failed to perform initial check")?;
+
+        Ok(())
+    }
+
+    async fn check(&mut self, logger: &Logger) -> Result<()> {
+        for entry in self.0.iter_mut().filter(|e| e.watch) {
+            if let Err(e) = entry.scan(logger).await {
+                // If an error was observed, we will stop treating this Storage as being watchable, and
+                // instead clean up the target-mount files on the tmpfs and bind mount the source_mount_point
+                // to target_mount_point.
+                error!(logger, "error observed when watching: {:?}", e);
+                entry.watch = false;
+
+                // Remove destination contents, but not the directory itself, since this is
+                // assumed to be bind-mounted into a container. If source/mount is a file, no need to cleanup
+                if entry.target_mount_point.as_path().is_dir() {
+                    for dir_entry in std::fs::read_dir(entry.target_mount_point.as_path())? {
+                        let dir_entry = dir_entry?;
+                        let path = dir_entry.path();
+                        if dir_entry.file_type()?.is_dir() {
+                            tokio::fs::remove_dir_all(path).await?;
+                        } else {
+                            tokio::fs::remove_file(path).await?;
+                        }
+                    }
+                }
+
+                //  - Create bind mount from source to destination
+                BareMount::new(
+                    entry.source_mount_point.to_str().unwrap(),
+                    entry.target_mount_point.to_str().unwrap(),
+                    "bind",
+                    MsFlags::MS_BIND,
+                    "bind",
+                    logger,
+                )
+                .mount()?;
+            }
+        }
+        Ok(())
+    }
+}
+
+/// Handles watchable mounts. The watcher will manage one or more mounts for one or more containers. For each
+/// mount that is added, the watcher will maintain a list of files to monitor, and periodically checks for new,
+/// removed or changed (modified date) files. When a change is identified, the watcher will either copy the new
+/// or updated file to a target mount point, or remove the removed file from the target mount point.  All WatchableStorage
+/// target mount points are expected to reside within a single tmpfs, whose root is created by the BindWatcher.
+///
+/// This is a temporary workaround to handle config map updates until we get inotify on 9p/virtio-fs.
+/// More context on this:
+/// - https://github.com/kata-containers/runtime/issues/1505
+/// - https://github.com/kata-containers/kata-containers/issues/1879
+#[derive(Debug, Default)]
+pub struct BindWatcher {
+    /// Container ID -> Vec of watched entries
+    sandbox_storages: Arc<Mutex<HashMap<String, SandboxStorages>>>,
+    watch_thread: Option<task::JoinHandle<()>>,
+}
+
+impl Drop for BindWatcher {
+    fn drop(&mut self) {
+        self.cleanup();
+    }
+}
+
+impl BindWatcher {
+    pub fn new() -> BindWatcher {
+        Default::default()
+    }
+
+    pub async fn add_container(
+        &mut self,
+        id: String,
+        mounts: impl IntoIterator<Item = protos::Storage>,
+        logger: &Logger,
+    ) -> Result<()> {
+        if self.watch_thread.is_none() {
+            // Virtio-fs shared path is RO by default, so we back the target-mounts by tmpfs.
+            self.mount(logger).await?;
+
+            // Spawn background thread to monitor changes
+            self.watch_thread = Some(Self::spawn_watcher(
+                logger.clone(),
+                Arc::clone(&self.sandbox_storages),
+                WATCH_INTERVAL_SECS,
+            ));
+        }
+
+        self.sandbox_storages
+            .lock()
+            .await
+            .entry(id)
+            .or_insert_with(SandboxStorages::default)
+            .add(mounts, logger)
+            .await
+            .with_context(|| "Failed to add container")?;
+
+        Ok(())
+    }
+
+    pub async fn remove_container(&self, id: &str) {
+        self.sandbox_storages.lock().await.remove(id);
+    }
+
+    fn spawn_watcher(
+        logger: Logger,
+        sandbox_storages: Arc<Mutex<HashMap<String, SandboxStorages>>>,
+        interval_secs: u64,
+    ) -> tokio::task::JoinHandle<()> {
+        tokio::spawn(async move {
+            let mut interval = time::interval(Duration::from_secs(interval_secs));
+
+            loop {
+                interval.tick().await;
+
+                debug!(&logger, "Looking for changed files");
+                for (_, entries) in sandbox_storages.lock().await.iter_mut() {
+                    if let Err(err) = entries.check(&logger).await {
+                        // We don't fail background loop, but rather log error instead.
+                        error!(logger, "Check failed: {}", err);
+                    }
+                }
+            }
+        })
+    }
+
+    async fn mount(&self, logger: &Logger) -> Result<()> {
+        fs::create_dir_all(WATCH_MOUNT_POINT_PATH).await?;
+
+        BareMount::new(
+            "tmpfs",
+            WATCH_MOUNT_POINT_PATH,
+            "tmpfs",
+            MsFlags::empty(),
+            "",
+            logger,
+        )
+        .mount()?;
+
+        Ok(())
+    }
+
+    fn cleanup(&mut self) {
+        if let Some(handle) = self.watch_thread.take() {
+            // Stop our background thread
+            handle.abort();
+        }
+
+        let _ = umount(WATCH_MOUNT_POINT_PATH);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::mount::is_mounted;
+    use crate::skip_if_not_root;
+    use std::fs;
+    use std::thread;
+
+    #[tokio::test]
+    async fn watch_entries() {
+        skip_if_not_root!();
+
+        // If there's an error with an entry, let's make sure it is removed, and that the
+        // mount-destination behaves like a standard bind-mount.
+
+        // Create an entries vector with three storage objects: storage, storage1, storage2.
+        // We'll first verify each are evaluated correctly, then increase the first entry's contents
+        // so it fails mount size check (>1MB) (test handling for failure on mount that is a directory).
+        // We'll then similarly cause failure with storage2 (test handling for failure on mount that is
+        // a single file). We'll then verify that storage1 continues to be watchable.
+        let source_dir = tempfile::tempdir().unwrap();
+        let dest_dir = tempfile::tempdir().unwrap();
+
+        let storage = protos::Storage {
+            source: source_dir.path().display().to_string(),
+            mount_point: dest_dir.path().display().to_string(),
+            ..Default::default()
+        };
+        std::fs::File::create(source_dir.path().join("small.txt"))
+            .unwrap()
+            .set_len(10)
+            .unwrap();
+
+        let source_dir1 = tempfile::tempdir().unwrap();
+        let dest_dir1 = tempfile::tempdir().unwrap();
+        let storage1 = protos::Storage {
+            source: source_dir1.path().display().to_string(),
+            mount_point: dest_dir1.path().display().to_string(),
+            ..Default::default()
+        };
+        std::fs::File::create(source_dir1.path().join("large.txt"))
+            .unwrap()
+            .set_len(MAX_SIZE_PER_WATCHABLE_MOUNT)
+            .unwrap();
+
+        // And finally, create a single file mount:
+        let source_dir2 = tempfile::tempdir().unwrap();
+        let dest_dir2 = tempfile::tempdir().unwrap();
+
+        let source_path = source_dir2.path().join("mounted-file");
+        let dest_path = dest_dir2.path().join("mounted-file");
+        let mounted_file = std::fs::File::create(&source_path).unwrap();
+        mounted_file.set_len(MAX_SIZE_PER_WATCHABLE_MOUNT).unwrap();
+
+        let storage2 = protos::Storage {
+            source: source_path.display().to_string(),
+            mount_point: dest_path.display().to_string(),
+            ..Default::default()
+        };
+
+        let logger = slog::Logger::root(slog::Discard, o!());
+
+        let mut entries = SandboxStorages {
+            ..Default::default()
+        };
+
+        entries
+            .add(std::iter::once(storage), &logger)
+            .await
+            .unwrap();
+
+        entries
+            .add(std::iter::once(storage1), &logger)
+            .await
+            .unwrap();
+
+        entries
+            .add(std::iter::once(storage2), &logger)
+            .await
+            .unwrap();
+
+        // Check that there are three entries, and that the
+        // destination (mount point) matches what we expect for
+        // the first:
+        assert!(entries.check(&logger).await.is_ok());
+        assert_eq!(entries.0.len(), 3);
+        assert_eq!(std::fs::read_dir(dest_dir.path()).unwrap().count(), 1);
+
+        // Add a second file which will trip file size check:
+        std::fs::File::create(source_dir.path().join("big.txt"))
+            .unwrap()
+            .set_len(MAX_SIZE_PER_WATCHABLE_MOUNT)
+            .unwrap();
+
+        assert!(entries.check(&logger).await.is_ok());
+
+        // Verify Storage 0 is no longer going to be watched:
+        assert!(!entries.0[0].watch);
+
+        // Verify that the directory has two entries:
+        assert_eq!(std::fs::read_dir(dest_dir.path()).unwrap().count(), 2);
+
+        // Verify that the directory is a bind mount. Add an entry without calling check,
+        // and verify that the destination directory includes these files in the case of
+        // mount that is no longer being watched (storage), but not within the still-being
+        // watched (storage1):
+        fs::write(source_dir.path().join("1.txt"), "updated").unwrap();
+        fs::write(source_dir1.path().join("2.txt"), "updated").unwrap();
+
+        assert_eq!(std::fs::read_dir(source_dir.path()).unwrap().count(), 3);
+        assert_eq!(std::fs::read_dir(dest_dir.path()).unwrap().count(), 3);
+        assert_eq!(std::fs::read_dir(source_dir1.path()).unwrap().count(), 2);
+        assert_eq!(std::fs::read_dir(dest_dir1.path()).unwrap().count(), 1);
+
+        // Verify that storage1 is still working. After running check, we expect that the number
+        // of entries to increment
+        assert!(entries.check(&logger).await.is_ok());
+        assert_eq!(std::fs::read_dir(dest_dir1.path()).unwrap().count(), 2);
+
+        // Break storage2 by increasing the file size
+        mounted_file
+            .set_len(MAX_SIZE_PER_WATCHABLE_MOUNT + 10)
+            .unwrap();
+        assert!(entries.check(&logger).await.is_ok());
+        // Verify Storage 2 is no longer going to be watched:
+        assert!(!entries.0[2].watch);
+
+        // Verify bind mount is working -- let's write to the file and observe output:
+        fs::write(&source_path, "updated").unwrap();
+        assert_eq!(fs::read_to_string(&source_path).unwrap(), "updated");
+    }
+
+    #[tokio::test]
+    async fn watch_directory_too_large() {
+        let source_dir = tempfile::tempdir().unwrap();
+        let dest_dir = tempfile::tempdir().unwrap();
+        let mut entry = Storage::new(protos::Storage {
+            source: source_dir.path().display().to_string(),
+            mount_point: dest_dir.path().display().to_string(),
+            ..Default::default()
+        })
+        .await
+        .unwrap();
+
+        let logger = slog::Logger::root(slog::Discard, o!());
+
+        // Create a file that is too large:
+        std::fs::File::create(source_dir.path().join("big.txt"))
+            .unwrap()
+            .set_len(MAX_SIZE_PER_WATCHABLE_MOUNT + 1)
+            .unwrap();
+        thread::sleep(Duration::from_secs(1));
+        assert!(entry.scan(&logger).await.is_err());
+        fs::remove_file(source_dir.path().join("big.txt")).unwrap();
+
+        std::fs::File::create(source_dir.path().join("big.txt"))
+            .unwrap()
+            .set_len(MAX_SIZE_PER_WATCHABLE_MOUNT - 1)
+            .unwrap();
+        thread::sleep(Duration::from_secs(1));
+        assert!(entry.scan(&logger).await.is_ok());
+
+        std::fs::File::create(source_dir.path().join("too-big.txt"))
+            .unwrap()
+            .set_len(2)
+            .unwrap();
+        thread::sleep(Duration::from_secs(1));
+        assert!(entry.scan(&logger).await.is_err());
+
+        fs::remove_file(source_dir.path().join("big.txt")).unwrap();
+        fs::remove_file(source_dir.path().join("too-big.txt")).unwrap();
+
+        // Up to eight files should be okay:
+        fs::write(source_dir.path().join("1.txt"), "updated").unwrap();
+        fs::write(source_dir.path().join("2.txt"), "updated").unwrap();
+        fs::write(source_dir.path().join("3.txt"), "updated").unwrap();
+        fs::write(source_dir.path().join("4.txt"), "updated").unwrap();
+        fs::write(source_dir.path().join("5.txt"), "updated").unwrap();
+        fs::write(source_dir.path().join("6.txt"), "updated").unwrap();
+        fs::write(source_dir.path().join("7.txt"), "updated").unwrap();
+        fs::write(source_dir.path().join("8.txt"), "updated").unwrap();
+        assert_eq!(entry.scan(&logger).await.unwrap(), 8);
+
+        // Nine files is too many:
+        fs::write(source_dir.path().join("9.txt"), "updated").unwrap();
+        thread::sleep(Duration::from_secs(1));
+        assert!(entry.scan(&logger).await.is_err());
+    }
+
+    #[tokio::test]
+    async fn watch_directory() {
+        // Prepare source directory:
+        // ./tmp/1.txt
+        // ./tmp/A/B/2.txt
+        let source_dir = tempfile::tempdir().unwrap();
+        fs::write(source_dir.path().join("1.txt"), "one").unwrap();
+        fs::create_dir_all(source_dir.path().join("A/B")).unwrap();
+        fs::write(source_dir.path().join("A/B/1.txt"), "two").unwrap();
+
+        let dest_dir = tempfile::tempdir().unwrap();
+
+        let mut entry = Storage::new(protos::Storage {
+            source: source_dir.path().display().to_string(),
+            mount_point: dest_dir.path().display().to_string(),
+            ..Default::default()
+        })
+        .await
+        .unwrap();
+
+        let logger = slog::Logger::root(slog::Discard, o!());
+
+        assert_eq!(entry.scan(&logger).await.unwrap(), 2);
+
+        // Should copy no files since nothing is changed since last check
+        assert_eq!(entry.scan(&logger).await.unwrap(), 0);
+
+        // Should copy 1 file
+        thread::sleep(Duration::from_secs(1));
+        fs::write(source_dir.path().join("A/B/1.txt"), "updated").unwrap();
+        assert_eq!(entry.scan(&logger).await.unwrap(), 1);
+        assert_eq!(
+            fs::read_to_string(dest_dir.path().join("A/B/1.txt")).unwrap(),
+            "updated"
+        );
+
+        // Should copy no new files after copy happened
+        assert_eq!(entry.scan(&logger).await.unwrap(), 0);
+
+        // Update another file
+        fs::write(source_dir.path().join("1.txt"), "updated").unwrap();
+        assert_eq!(entry.scan(&logger).await.unwrap(), 1);
+    }
+
+    #[tokio::test]
+    async fn watch_file() {
+        let source_dir = tempfile::tempdir().unwrap();
+        let source_file = source_dir.path().join("1.txt");
+
+        fs::write(&source_file, "one").unwrap();
+
+        let dest_dir = tempfile::tempdir().unwrap();
+        let dest_file = dest_dir.path().join("1.txt");
+
+        let mut entry = Storage::new(protos::Storage {
+            source: source_file.display().to_string(),
+            mount_point: dest_file.display().to_string(),
+            ..Default::default()
+        })
+        .await
+        .unwrap();
+
+        let logger = slog::Logger::root(slog::Discard, o!());
+
+        assert_eq!(entry.scan(&logger).await.unwrap(), 1);
+
+        thread::sleep(Duration::from_secs(1));
+        fs::write(&source_file, "two").unwrap();
+        assert_eq!(entry.scan(&logger).await.unwrap(), 1);
+        assert_eq!(fs::read_to_string(&dest_file).unwrap(), "two");
+        assert_eq!(entry.scan(&logger).await.unwrap(), 0);
+    }
+
+    #[tokio::test]
+    async fn delete_file() {
+        let source_dir = tempfile::tempdir().unwrap();
+        let source_file = source_dir.path().join("1.txt");
+        fs::write(&source_file, "one").unwrap();
+
+        let dest_dir = tempfile::tempdir().unwrap();
+        let target_file = dest_dir.path().join("1.txt");
+
+        let mut entry = Storage::new(protos::Storage {
+            source: source_dir.path().display().to_string(),
+            mount_point: dest_dir.path().display().to_string(),
+            ..Default::default()
+        })
+        .await
+        .unwrap();
+
+        let logger = slog::Logger::root(slog::Discard, o!());
+
+        assert_eq!(entry.scan(&logger).await.unwrap(), 1);
+        assert_eq!(entry.watched_files.len(), 1);
+
+        assert!(target_file.exists());
+        assert!(entry.watched_files.contains_key(&source_file));
+
+        // Remove source file
+        fs::remove_file(&source_file).unwrap();
+
+        assert_eq!(entry.scan(&logger).await.unwrap(), 0);
+
+        assert_eq!(entry.watched_files.len(), 0);
+        assert!(!target_file.exists());
+    }
+
+    #[tokio::test]
+    async fn make_target_path() {
+        let source_dir = tempfile::tempdir().unwrap();
+        let target_dir = tempfile::tempdir().unwrap();
+
+        let source_dir = source_dir.path();
+        let target_dir = target_dir.path();
+
+        let entry = Storage::new(protos::Storage {
+            source: source_dir.display().to_string(),
+            mount_point: target_dir.display().to_string(),
+            ..Default::default()
+        })
+        .await
+        .unwrap();
+
+        assert_eq!(
+            entry.make_target_path(source_dir.join("1.txt")).unwrap(),
+            target_dir.join("1.txt")
+        );
+
+        assert_eq!(
+            entry
+                .make_target_path(source_dir.join("a/b/2.txt"))
+                .unwrap(),
+            target_dir.join("a/b/2.txt")
+        );
+    }
+
+    #[tokio::test]
+    async fn create_tmpfs() {
+        skip_if_not_root!();
+
+        let logger = slog::Logger::root(slog::Discard, o!());
+        let mut watcher = BindWatcher::default();
+
+        watcher.mount(&logger).await.unwrap();
+        assert!(is_mounted(WATCH_MOUNT_POINT_PATH).unwrap());
+
+        watcher.cleanup();
+        assert!(!is_mounted(WATCH_MOUNT_POINT_PATH).unwrap());
+    }
+
+    #[tokio::test]
+    async fn spawn_thread() {
+        skip_if_not_root!();
+
+        let source_dir = tempfile::tempdir().unwrap();
+        fs::write(source_dir.path().join("1.txt"), "one").unwrap();
+
+        let dest_dir = tempfile::tempdir().unwrap();
+
+        let storage = protos::Storage {
+            source: source_dir.path().display().to_string(),
+            mount_point: dest_dir.path().display().to_string(),
+            ..Default::default()
+        };
+
+        let logger = slog::Logger::root(slog::Discard, o!());
+        let mut watcher = BindWatcher::default();
+
+        watcher
+            .add_container("test".into(), std::iter::once(storage), &logger)
+            .await
+            .unwrap();
+
+        thread::sleep(Duration::from_secs(WATCH_INTERVAL_SECS));
+
+        let out = fs::read_to_string(dest_dir.path().join("1.txt")).unwrap();
+        assert_eq!(out, "one");
+    }
+}

src/agent/vsock-exporter/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "vsock-exporter"
+version = "0.1.0"
+authors = ["James O. D. Hunt <james.o.hunt@intel.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+nix = "0.21.0"
+libc = "0.2.94"
+thiserror = "1.0.24"
+opentelemetry = { version = "0.14.0", features=["serialize"] }
+serde = { version = "1.0.126", features = ["derive"] }
+vsock = "0.2.3"
+bincode = "1.3.3"
+byteorder = "1.4.3"
+slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_info"] }
+async-trait = "0.1.50"

src/agent/vsock-exporter/src/lib.rs

@@ -0,0 +1,196 @@
+// Copyright (c) 2020-2021 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+// The VSOCK Exporter sends trace spans "out" to the forwarder running on the
+// host (which then forwards them on to a trace collector). The data is sent
+// via a VSOCK socket that the forwarder process is listening on. To allow the
+// forwarder to know how much data to each for each trace span the simplest
+// protocol is employed which uses a header packet and the payload (trace
+// span) data. The header packet is a simple count of the number of bytes in the
+// payload, which allows the forwarder to know how many bytes it must read to
+// consume the trace span. The payload is a serialised version of the trace span.
+
+use async_trait::async_trait;
+use byteorder::{ByteOrder, NetworkEndian};
+use opentelemetry::sdk::export::trace::{ExportResult, SpanData, SpanExporter};
+use opentelemetry::sdk::export::ExportError;
+use slog::{error, o, Logger};
+use std::io::{ErrorKind, Write};
+use std::net::Shutdown;
+use std::sync::Mutex;
+use vsock::{SockAddr, VsockStream};
+
+const ANY_CID: &str = "any";
+
+// Must match the value of the variable of the same name in the trace forwarder.
+const HEADER_SIZE_BYTES: u64 = std::mem::size_of::<u64>() as u64;
+
+// By default, the VSOCK exporter should talk "out" to the host where the
+// forwarder is running.
+const DEFAULT_CID: u32 = libc::VMADDR_CID_HOST;
+
+// The VSOCK port the forwarders listens on by default
+const DEFAULT_PORT: u32 = 10240;
+
+#[derive(Debug)]
+pub struct Exporter {
+    port: u32,
+    cid: u32,
+    conn: Mutex<VsockStream>,
+    logger: Logger,
+}
+
+impl Exporter {
+    /// Create a new exporter builder.
+    pub fn builder() -> Builder {
+        Builder::default()
+    }
+}
+
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+    #[error("connection error: {0}")]
+    ConnectionError(String),
+    #[error("serialisation error: {0}")]
+    SerialisationError(#[from] bincode::Error),
+    #[error("I/O error: {0}")]
+    IOError(#[from] std::io::Error),
+}
+
+impl ExportError for Error {
+    fn exporter_name(&self) -> &'static str {
+        "vsock-exporter"
+    }
+}
+
+fn make_io_error(desc: String) -> std::io::Error {
+    std::io::Error::new(ErrorKind::Other, desc)
+}
+
+// Send a trace span to the forwarder running on the host.
+fn write_span(writer: &mut dyn Write, span: &SpanData) -> Result<(), std::io::Error> {
+    let encoded_payload: Vec<u8> =
+        bincode::serialize(&span).map_err(|e| make_io_error(e.to_string()))?;
+
+    let payload_len: u64 = encoded_payload.len() as u64;
+
+    let mut payload_len_as_bytes: [u8; HEADER_SIZE_BYTES as usize] =
+        [0; HEADER_SIZE_BYTES as usize];
+
+    // Encode the header
+    NetworkEndian::write_u64(&mut payload_len_as_bytes, payload_len);
+
+    // Send the header
+    writer
+        .write_all(&payload_len_as_bytes)
+        .map_err(|e| make_io_error(format!("failed to write trace header: {:?}", e)))?;
+
+    writer
+        .write_all(&encoded_payload)
+        .map_err(|e| make_io_error(format!("failed to write trace payload: {:?}", e)))
+}
+
+fn handle_batch(writer: &mut dyn Write, batch: Vec<SpanData>) -> ExportResult {
+    for span_data in batch {
+        write_span(writer, &span_data).map_err(Error::IOError)?;
+    }
+
+    Ok(())
+}
+
+#[async_trait]
+impl SpanExporter for Exporter {
+    async fn export(&mut self, batch: Vec<SpanData>) -> ExportResult {
+        let conn = self.conn.lock();
+
+        match conn {
+            Ok(mut c) => handle_batch(&mut *c, batch),
+            Err(e) => {
+                error!(self.logger, "failed to obtain connection";
+                        "error" => format!("{}", e));
+
+                return Err(Error::ConnectionError(e.to_string()).into());
+            }
+        }
+    }
+
+    fn shutdown(&mut self) {
+        let conn = match self.conn.lock() {
+            Ok(conn) => conn,
+            Err(e) => {
+                error!(self.logger, "failed to obtain connection";
+                        "error" => format!("{}", e));
+                return;
+            }
+        };
+
+        conn.shutdown(Shutdown::Write)
+            .expect("failed to shutdown VSOCK connection");
+    }
+}
+
+#[derive(Debug)]
+pub struct Builder {
+    port: u32,
+    cid: u32,
+    logger: Logger,
+}
+
+impl Default for Builder {
+    fn default() -> Self {
+        let logger = Logger::root(slog::Discard, o!());
+
+        Builder {
+            cid: DEFAULT_CID,
+            port: DEFAULT_PORT,
+            logger,
+        }
+    }
+}
+
+impl Builder {
+    pub fn with_cid(self, cid: u32) -> Self {
+        Builder { cid, ..self }
+    }
+
+    pub fn with_port(self, port: u32) -> Self {
+        Builder { port, ..self }
+    }
+
+    pub fn with_logger(self, logger: &Logger) -> Self {
+        Builder {
+            logger: logger.new(o!()),
+            ..self
+        }
+    }
+
+    pub fn init(self) -> Exporter {
+        let Builder { port, cid, logger } = self;
+
+        let sock_addr = SockAddr::new_vsock(self.cid, self.port);
+
+        let cid_str: String;
+
+        if self.cid == libc::VMADDR_CID_ANY {
+            cid_str = ANY_CID.to_string();
+        } else {
+            cid_str = format!("{}", self.cid);
+        }
+
+        let msg = format!(
+            "failed to connect to VSOCK server (port: {}, cid: {}) - {}",
+            self.port, cid_str, "ensure trace forwarder is running on host"
+        );
+
+        let conn = VsockStream::connect(&sock_addr).expect(&msg);
+
+        Exporter {
+            port,
+            cid,
+            conn: Mutex::new(conn),
+            logger: logger.new(o!("cid" => cid_str, "port" => port)),
+        }
+    }
+}

src/runtime/Makefile

@@ -166,6 +166,7 @@ DEFAULTEXPFEATURES := []
 
 #Default entropy source
 DEFENTROPYSOURCE := /dev/urandom
+DEFVALIDENTROPYSOURCES := [\"/dev/urandom\",\"/dev/random\",\"\"]
 
 DEFDISABLEBLOCK := false
 DEFSHAREDFS_QEMU_VIRTIOFS := virtio-fs
@@ -173,7 +174,7 @@ DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/kata-qemu/virtiofsd
 DEFVALIDVIRTIOFSDAEMONPATHS := [\"$(DEFVIRTIOFSDAEMON)\"]
 # Default DAX mapping cache size in MiB
 #if value is 0, DAX is not enabled
-DEFVIRTIOFSCACHESIZE := 0
+DEFVIRTIOFSCACHESIZE ?= 0
 DEFVIRTIOFSCACHE ?= auto
 # Format example:
 #   [\"-o\", \"arg1=xxx,arg2\", \"-o\", \"hello world\", \"--arg3=yyy\"]
@@ -454,6 +455,7 @@ USER_VARS += DEFFILEMEMBACKEND
 USER_VARS += DEFVALIDFILEMEMBACKENDS
 USER_VARS += DEFMSIZE9P
 USER_VARS += DEFENTROPYSOURCE
+USER_VARS += DEFVALIDENTROPYSOURCES
 USER_VARS += DEFSANDBOXCGROUPONLY
 USER_VARS += DEFBINDMOUNTS
 USER_VARS += FEATURE_SELINUX
@@ -580,7 +582,8 @@ $(MONITOR_OUTPUT): $(SOURCES) $(GENERATED_FILES) $(MAKEFILE_LIST) .git-commit
 	install \
 	show-header \
 	show-summary \
-	show-variables
+	show-variables \
+	vendor
 
 $(TARGET).coverage: $(SOURCES) $(GENERATED_FILES) $(MAKEFILE_LIST)
 	$(QUIET_TEST)go test -o $@ -covermode count
@@ -645,6 +648,14 @@ install-scripts: $(SCRIPTS)
 install-completions:
 	$(QUIET_INST)install --mode 0644 -D  $(BASH_COMPLETIONS) $(DESTDIR)/$(BASH_COMPLETIONSDIR)/$(notdir $(BASH_COMPLETIONS));
 
+handle_vendor:
+	go mod tidy
+	go mod vendor
+	go mod verify
+
+vendor: handle_vendor
+	./hack/tree_status.sh
+
 clean:
 	$(QUIET_CLEAN)rm -f \
 		$(CONFIGS) \

src/runtime/README.md

@@ -1,7 +1,4 @@
-[![Build Status](https://travis-ci.org/kata-containers/kata-containers.svg?branch=master)](https://travis-ci.org/kata-containers/kata-containers)
-[![Build Status](http://jenkins.katacontainers.io/job/kata-containers-runtime-ubuntu-18-04-master/badge/icon)](http://jenkins.katacontainers.io/job/kata-containers-runtime-ubuntu-18-04-master/)
 [![Go Report Card](https://goreportcard.com/badge/github.com/kata-containers/kata-containers)](https://goreportcard.com/report/github.com/kata-containers/kata-containers)
-[![GoDoc](https://godoc.org/github.com/kata-containers/runtime?status.svg)](https://godoc.org/github.com/kata-containers/runtime)
 
 # Runtime
 
@@ -84,7 +81,7 @@ $ kata-runtime check
 
 [![Get it from the Snap Store](https://snapcraft.io/static/images/badges/en/snap-store-black.svg)](https://snapcraft.io/kata-containers)
 
-See the [installation guides](https://github.com/kata-containers/documentation/tree/master/install/README.md)
+See the [installation guides](https://github.com/kata-containers/kata-containers/blob/main/docs/install/README.md)
 available for various operating systems.
 
 ## Quick start for developers

src/runtime/VERSION

@@ -1 +0,0 @@
-2.0.0

src/runtime/VERSION

@@ -0,0 +1 @@
+../../VERSION

src/runtime/arch/amd64-options.mk

@@ -5,7 +5,7 @@
 
 # Intel x86-64 settings
 
-MACHINETYPE := pc
+MACHINETYPE := q35
 KERNELPARAMS :=
 MACHINEACCELERATORS :=
 CPUFEATURES := pmu=off

src/runtime/arch/s390x-options.mk

@@ -11,3 +11,10 @@ MACHINEACCELERATORS :=
 CPUFEATURES :=
 
 QEMUCMD := qemu-system-s390x
+
+# See https://github.com/kata-containers/osbuilder/issues/217
+NEEDS_CC_SETTING = $(shell grep -E "\<(fedora|suse)\>" /etc/os-release 2> /dev/null)
+ifneq (,$(NEEDS_CC_SETTING))
+	CC := gcc
+	export CC
+endif

src/runtime/cli/config/configuration-acrn.toml.in

@@ -150,6 +150,10 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_ACRN@"
 
 #debug_console_enabled = true
 
+# Agent connection dialing timeout value in seconds
+# (default: 30)
+#dial_timeout = 30
+
 [netmon]
 # If enabled, the network monitoring process gets started when the
 # sandbox is created. This allows for the detection of some additional

src/runtime/cli/config/configuration-clh.toml.in

@@ -165,6 +165,10 @@ block_device_driver = "virtio-blk"
 
 #debug_console_enabled = true
 
+# Agent connection dialing timeout value in seconds
+# (default: 30)
+#dial_timeout = 30
+
 [netmon]
 # If enabled, the network monitoring process gets started when the
 # sandbox is created. This allows for the detection of some additional

src/runtime/cli/config/configuration-fc.toml.in

@@ -161,24 +161,24 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_FC@"
 
 # This option changes the default hypervisor and kernel parameters
 # to enable debug output where available.
-# 
+#
 # Default false
 #enable_debug = true
 
 # Disable the customizations done in the runtime when it detects
 # that it is running on top a VMM. This will result in the runtime
 # behaving as it would when running on bare metal.
-# 
+#
 #disable_nesting_checks = true
 
-# This is the msize used for 9p shares. It is the number of bytes 
+# This is the msize used for 9p shares. It is the number of bytes
 # used for 9p packet payload.
 #msize_9p = @DEFMSIZE9P@
 
-# VFIO devices are hotplugged on a bridge by default. 
+# VFIO devices are hotplugged on a bridge by default.
 # Enable hotplugging on root bus. This may be required for devices with
-# a large PCI bar, as this is a current limitation with hotplugging on 
-# a bridge. This value is valid for "pc" machine type.
+# a large PCI bar, as this is a current limitation with hotplugging on
+# a bridge.
 # Default false
 #hotplug_vfio_on_root_bus = true
 
@@ -194,6 +194,11 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_FC@"
 # all practical purposes.
 #entropy_source= "@DEFENTROPYSOURCE@"
 
+# List of valid annotations values for entropy_source
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDENTROPYSOURCES@
+valid_entropy_sources = @DEFVALIDENTROPYSOURCES@
+
 # Path to OCI hook binaries in the *guest rootfs*.
 # This does not affect host-side hooks which must instead be added to
 # the OCI spec passed to the runtime.
@@ -282,6 +287,10 @@ kernel_modules=[]
 
 #debug_console_enabled = true
 
+# Agent connection dialing timeout value in seconds
+# (default: 30)
+#dial_timeout = 30
+
 [netmon]
 # If enabled, the network monitoring process gets started when the
 # sandbox is created. This allows for the detection of some additional

src/runtime/cli/config/configuration-qemu.toml.in

@@ -16,6 +16,14 @@ kernel = "@KERNELPATH@"
 image = "@IMAGEPATH@"
 machine_type = "@MACHINETYPE@"
 
+# Enable confidential guest support.
+# Toggling that setting may trigger different hardware features, ranging
+# from memory encryption to both memory and CPU-state encryption and integrity.
+# The Kata Containers runtime dynamically detects the available feature set and
+# aims at enabling the largest possible one.
+# Default false
+# confidential_guest = true
+
 # List of valid annotation names for the hypervisor
 # Each member of the list is a regular expression, which is the base name
 # of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
@@ -269,7 +277,7 @@ pflashes = []
 # VFIO devices are hotplugged on a bridge by default.
 # Enable hotplugging on root bus. This may be required for devices with
 # a large PCI bar, as this is a current limitation with hotplugging on
-# a bridge. This value is valid for "pc" machine type.
+# a bridge.
 # Default false
 #hotplug_vfio_on_root_bus = true
 
@@ -296,6 +304,11 @@ pflashes = []
 # all practical purposes.
 #entropy_source= "@DEFENTROPYSOURCE@"
 
+# List of valid annotations values for entropy_source
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDENTROPYSOURCES@
+valid_entropy_sources = @DEFVALIDENTROPYSOURCES@
+
 # Path to OCI hook binaries in the *guest rootfs*.
 # This does not affect host-side hooks which must instead be added to
 # the OCI spec passed to the runtime.
@@ -432,6 +445,10 @@ kernel_modules=[]
 
 #debug_console_enabled = true
 
+# Agent connection dialing timeout value in seconds
+# (default: 30)
+#dial_timeout = 30
+
 [netmon]
 # If enabled, the network monitoring process gets started when the
 # sandbox is created. This allows for the detection of some additional
@@ -527,3 +544,30 @@ experimental=@DEFAULTEXPFEATURES@
 # If enabled, user can run pprof tools with shim v2 process through kata-monitor.
 # (default: false)
 # enable_pprof = true
+
+# WARNING: All the options in the following section have not been implemented yet.
+# This section was added as a placeholder. DO NOT USE IT!
+[image]
+# Container image service.
+#
+# Offload the CRI image management service to the Kata agent.
+# (default: false)
+#service_offload = true
+
+# Container image decryption keys provisioning.
+# Applies only if service_offload is true.
+# Keys can be provisioned locally (e.g. through a special command or
+# a local file) or remotely (usually after the guest is remotely attested).
+# The provision setting is a complete URL that lets the Kata agent decide
+# which method to use in order to fetch the keys.
+#
+# Keys can be stored in a local file, in a measured and attested initrd:
+#provision=data:///local/key/file
+#
+# Keys could be fetched through a special command or binary from the
+# initrd (guest) image, e.g. a firmware call:
+#provision=file:///path/to/bin/fetcher/in/guest
+#
+# Keys can be remotely provisioned. The Kata agent fetches them from e.g.
+# a HTTPS URL:
+#provision=https://my-key-broker.foo/tenant/<tenant-id>

src/runtime/cli/console.go

@@ -1,134 +0,0 @@
-// Copyright (c) 2014,2015,2016 Docker, Inc.
-// Copyright (c) 2017 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-package main
-
-import (
-	"fmt"
-	"io"
-	"os"
-	"syscall"
-	"unsafe"
-
-	"golang.org/x/sys/unix"
-)
-
-var ptmxPath = "/dev/ptmx"
-
-// Console represents a pseudo TTY.
-type Console struct {
-	io.ReadWriteCloser
-
-	master    *os.File
-	slavePath string
-}
-
-// isTerminal returns true if fd is a terminal, else false
-func isTerminal(fd uintptr) bool {
-	var termios syscall.Termios
-	_, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, syscall.TCGETS, uintptr(unsafe.Pointer(&termios)))
-	return err == 0
-}
-
-// ConsoleFromFile creates a console from a file
-func ConsoleFromFile(f *os.File) *Console {
-	return &Console{
-		master: f,
-	}
-}
-
-// NewConsole returns an initialized console that can be used within a container by copying bytes
-// from the master side to the slave that is attached as the tty for the container's init process.
-func newConsole() (*Console, error) {
-	master, err := os.OpenFile(ptmxPath, unix.O_RDWR|unix.O_NOCTTY|unix.O_CLOEXEC, 0)
-	if err != nil {
-		return nil, err
-	}
-	if err := saneTerminal(master); err != nil {
-		return nil, err
-	}
-	console, err := ptsname(master)
-	if err != nil {
-		return nil, err
-	}
-	if err := unlockpt(master); err != nil {
-		return nil, err
-	}
-	return &Console{
-		slavePath: console,
-		master:    master,
-	}, nil
-}
-
-// File returns master
-func (c *Console) File() *os.File {
-	return c.master
-}
-
-// Path to slave
-func (c *Console) Path() string {
-	return c.slavePath
-}
-
-// Read from master
-func (c *Console) Read(b []byte) (int, error) {
-	return c.master.Read(b)
-}
-
-// Write to master
-func (c *Console) Write(b []byte) (int, error) {
-	return c.master.Write(b)
-}
-
-// Close master
-func (c *Console) Close() error {
-	if m := c.master; m != nil {
-		return m.Close()
-	}
-	return nil
-}
-
-// unlockpt unlocks the slave pseudoterminal device corresponding to the master pseudoterminal referred to by f.
-// unlockpt should be called before opening the slave side of a pty.
-func unlockpt(f *os.File) error {
-	var u int32
-	if _, _, err := unix.Syscall(unix.SYS_IOCTL, f.Fd(), unix.TIOCSPTLCK, uintptr(unsafe.Pointer(&u))); err != 0 {
-		return err
-	}
-	return nil
-}
-
-// ptsname retrieves the name of the first available pts for the given master.
-func ptsname(f *os.File) (string, error) {
-	var u uint32
-	if _, _, err := unix.Syscall(unix.SYS_IOCTL, f.Fd(), unix.TIOCGPTN, uintptr(unsafe.Pointer(&u))); err != 0 {
-		return "", err
-	}
-	return fmt.Sprintf("/dev/pts/%d", u), nil
-}
-
-// saneTerminal sets the necessary tty_ioctl(4)s to ensure that a pty pair
-// created by us acts normally. In particular, a not-very-well-known default of
-// Linux unix98 ptys is that they have +onlcr by default. While this isn't a
-// problem for terminal emulators, because we relay data from the terminal we
-// also relay that funky line discipline.
-func saneTerminal(terminal *os.File) error {
-	// Go doesn't have a wrapper for any of the termios ioctls.
-	var termios unix.Termios
-
-	if _, _, err := unix.Syscall(unix.SYS_IOCTL, terminal.Fd(), unix.TCGETS, uintptr(unsafe.Pointer(&termios))); err != 0 {
-		return fmt.Errorf("ioctl(tty, tcgets): %s", err.Error())
-	}
-
-	// Set -onlcr so we don't have to deal with \r.
-	termios.Oflag &^= unix.ONLCR
-
-	if _, _, err := unix.Syscall(unix.SYS_IOCTL, terminal.Fd(), unix.TCSETS, uintptr(unsafe.Pointer(&termios))); err != 0 {
-		return fmt.Errorf("ioctl(tty, tcsets): %s", err.Error())
-	}
-
-	return nil
-}

src/runtime/cli/console_test.go

@@ -1,129 +0,0 @@
-// Copyright (c) 2017 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-package main
-
-import (
-	"io/ioutil"
-	"os"
-	"testing"
-
-	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestConsoleFromFile(t *testing.T) {
-	assert := assert.New(t)
-
-	console := ConsoleFromFile(os.Stdout)
-
-	assert.NotNil(console.File(), "console file is nil")
-}
-
-func TestNewConsole(t *testing.T) {
-	if tc.NotValid(ktu.NeedRoot()) {
-		t.Skip(testDisabledAsNonRoot)
-	}
-	assert := assert.New(t)
-
-	console, err := newConsole()
-	assert.NoError(err, "failed to create a new console: %s", err)
-	defer console.Close()
-
-	assert.NotEmpty(console.Path(), "console path is empty")
-
-	assert.NotNil(console.File(), "console file is nil")
-}
-
-func TestIsTerminal(t *testing.T) {
-	if tc.NotValid(ktu.NeedRoot()) {
-		t.Skip(testDisabledAsNonRoot)
-	}
-	assert := assert.New(t)
-
-	var fd uintptr = 4
-	assert.False(isTerminal(fd), "Fd %d is not a terminal", fd)
-
-	console, err := newConsole()
-	assert.NoError(err, "failed to create a new console: %s", err)
-	defer console.Close()
-
-	fd = console.File().Fd()
-	assert.True(isTerminal(fd), "Fd %d is a terminal", fd)
-}
-
-func TestReadWrite(t *testing.T) {
-	assert := assert.New(t)
-
-	// write operation
-	f, err := ioutil.TempFile(os.TempDir(), ".tty")
-	assert.NoError(err, "failed to create a temporal file")
-	defer os.Remove(f.Name())
-
-	console := ConsoleFromFile(f)
-	assert.NotNil(console)
-	defer console.Close()
-
-	msgWrite := "hello"
-	l, err := console.Write([]byte(msgWrite))
-	assert.NoError(err, "failed to write message: %s", msgWrite)
-	assert.Equal(len(msgWrite), l)
-
-	console.master.Sync()
-	console.master.Seek(0, 0)
-
-	// Read operation
-	msgRead := make([]byte, len(msgWrite))
-	l, err = console.Read(msgRead)
-	assert.NoError(err, "failed to read message: %s", msgWrite)
-	assert.Equal(len(msgWrite), l)
-	assert.Equal(msgWrite, string(msgRead))
-}
-
-func TestNewConsoleFail(t *testing.T) {
-	assert := assert.New(t)
-
-	orgPtmxPath := ptmxPath
-	defer func() { ptmxPath = orgPtmxPath }()
-
-	// OpenFile failure
-	ptmxPath = "/this/file/does/not/exist"
-	c, err := newConsole()
-	assert.Error(err)
-	assert.Nil(c)
-
-	// saneTerminal failure
-	f, err := ioutil.TempFile("", "")
-	assert.NoError(err)
-	assert.NoError(f.Close())
-	defer os.Remove(f.Name())
-	ptmxPath = f.Name()
-	c, err = newConsole()
-	assert.Error(err)
-	assert.Nil(c)
-}
-
-func TestConsoleClose(t *testing.T) {
-	assert := assert.New(t)
-
-	// nil master
-	c := &Console{}
-	assert.NoError(c.Close())
-
-	f, err := ioutil.TempFile("", "")
-	assert.NoError(err)
-	defer os.Remove(f.Name())
-
-	c.master = f
-	assert.NoError(c.Close())
-}
-
-func TestConsolePtsnameFail(t *testing.T) {
-	assert := assert.New(t)
-
-	pts, err := ptsname(nil)
-	assert.Error(err)
-	assert.Empty(pts)
-}

src/runtime/cli/kata-check.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/containerd/cgroups"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
@@ -389,13 +390,6 @@ EXAMPLES:
 		if verbose {
 			kataLog.Logger.SetLevel(logrus.InfoLevel)
 		}
-		ctx, err := cliContextToContext(context)
-		if err != nil {
-			return err
-		}
-
-		span, _ := katautils.Trace(ctx, "check")
-		defer span.End()
 
 		if !context.Bool("no-network-checks") && os.Getenv(noNetworkEnvVar) == "" {
 			cmd := RelCmdCheck
@@ -407,8 +401,7 @@ EXAMPLES:
 			if os.Geteuid() == 0 {
 				kataLog.Warn("Not running network checks as super user")
 			} else {
-
-				err = HandleReleaseVersions(cmd, version, context.Bool("include-all-releases"))
+				err := HandleReleaseVersions(cmd, version, context.Bool("include-all-releases"))
 				if err != nil {
 					return err
 				}
@@ -424,7 +417,12 @@ EXAMPLES:
 			return errors.New("check: cannot determine runtime config")
 		}
 
-		err = setCPUtype(runtimeConfig.HypervisorType)
+		// check if cgroup can work use the same logic for creating containers
+		if _, err := vc.V1Constraints(); err != nil && err == cgroups.ErrMountPointNotExist && !runtimeConfig.SandboxCgroupOnly {
+			return fmt.Errorf("Cgroup v2 requires the following configuration: `sandbox_cgroup_only=true`.")
+		}
+
+		err := setCPUtype(runtimeConfig.HypervisorType)
 		if err != nil {
 			return err
 		}
@@ -437,7 +435,6 @@ EXAMPLES:
 		}
 
 		err = hostIsVMContainerCapable(details)
-
 		if err != nil {
 			return err
 		}

src/runtime/cli/kata-env.go

@@ -13,7 +13,6 @@ import (
 	"strings"
 
 	"github.com/BurntSushi/toml"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental"
@@ -266,9 +265,9 @@ func getMemoryInfo() MemoryInfo {
 	}
 
 	return MemoryInfo{
-		Total:     mi.MemTotal,
-		Free:      mi.MemFree,
-		Available: mi.MemAvailable,
+		Total:     *mi.MemTotal,
+		Free:      *mi.MemFree,
+		Available: *mi.MemAvailable,
 	}
 }
 
@@ -448,14 +447,6 @@ var kataEnvCLICommand = cli.Command{
 		},
 	},
 	Action: func(context *cli.Context) error {
-		ctx, err := cliContextToContext(context)
-		if err != nil {
-			return err
-		}
-
-		span, _ := katautils.Trace(ctx, "kata-env")
-		defer span.End()
-
 		return handleSettings(defaultOutputFile, context)
 	},
 }