Merge pull request #1280 from egernst/final-fix-probably-2.0-dev

Final fix probably 2.0 dev

.github/workflows/commit-message-check.yaml

@@ -10,7 +10,7 @@ env:
   error_msg: |+
     See the document below for help on formatting commits for the project.
 
-    https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md#patch-forma
+    https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md#patch-format
 
 jobs:
   commit-message-check:

.github/workflows/main.yaml

@@ -43,7 +43,7 @@ jobs:
            echo "artifact-built=false" >> $GITHUB_ENV
          fi
       - name: store-artifacts
-        if: env.artifact-built == 'true'
+        if: ${{ env.artifact-built }} == 'true'
         uses: actions/upload-artifact@master
         with:
           name: kata-artifacts
@@ -71,7 +71,7 @@ jobs:
            echo "artifact-built=false" >> $GITHUB_ENV
          fi
       - name: store-artifacts
-        if: env.artifact-built == 'true'
+        if: ${{ env.artifact-built }} == 'true'
         uses: actions/upload-artifact@master
         with:
           name: kata-artifacts
@@ -97,7 +97,7 @@ jobs:
            echo "artifact-built=false" >> $GITHUB_ENV
          fi
       - name: store-artifacts
-        if: env.artifact-built == 'true'
+        if: ${{ env.artifact-built }} == 'true'
         uses: actions/upload-artifact@master
         with:
           name: kata-artifacts
@@ -123,7 +123,7 @@ jobs:
            echo "artifact-built=false" >> $GITHUB_ENV
          fi
       - name: store-artifacts
-        if: env.artifact-built == 'true'
+        if: ${{ env.artifact-built }} == 'true'
         uses: actions/upload-artifact@master
         with:
           name: kata-artifacts
@@ -150,7 +150,7 @@ jobs:
            echo "artifact-built=false" >> $GITHUB_ENV
          fi
       - name: store-artifacts
-        if: env.artifact-built == 'true'
+        if: ${{ env.artifact-built }} == 'true'
         uses: actions/upload-artifact@master
         with:
           name: kata-artifacts
@@ -177,7 +177,7 @@ jobs:
            echo "artifact-built=false" >> $GITHUB_ENV
          fi
       - name: store-artifacts
-        if: env.artifact-built == 'true'
+        if: ${{ env.artifact-built }} == 'true'
         uses: actions/upload-artifact@master
         with:
           name: kata-artifacts
@@ -204,7 +204,7 @@ jobs:
            echo "artifact-built=false" >> $GITHUB_ENV
          fi
       - name: store-artifacts
-        if: env.artifact-built == 'true'
+        if: ${{ env.artifact-built }} == 'true'
         uses: actions/upload-artifact@master
         with:
           name: kata-artifacts
@@ -231,7 +231,7 @@ jobs:
            echo "artifact-built=false" >> $GITHUB_ENV
          fi
       - name: store-artifacts
-        if: env.artifact-built == 'true'
+        if: ${{ env.artifact-built }} == 'true'
         uses: actions/upload-artifact@master
         with:
           name: kata-artifacts
@@ -258,7 +258,7 @@ jobs:
            echo "artifact-built=false" >> $GITHUB_ENV
          fi
       - name: store-artifacts
-        if: env.artifact-built == 'true'
+        if: ${{ env.artifact-built }} == 'true'
         uses: actions/upload-artifact@master
         with:
           name: kata-artifacts

.github/workflows/release.yaml

@@ -44,7 +44,7 @@ jobs:
            echo "artifact-built=false" >> $GITHUB_ENV
          fi
       - name: store-artifacts
-        if: env.artifact-built == 'true'
+        if: ${{ env.artifact-built }} == 'true'
         uses: actions/upload-artifact@v2
         with:
           name: kata-artifacts
@@ -72,7 +72,7 @@ jobs:
            echo "artifact-built=false" >> $GITHUB_ENV
          fi
       - name: store-artifacts
-        if: env.artifact-built == 'true'
+        if: ${{ env.artifact-built }} == 'true'
         uses: actions/upload-artifact@v2
         with:
           name: kata-artifacts
@@ -98,7 +98,7 @@ jobs:
            echo "artifact-built=false" >> $GITHUB_ENV
          fi
       - name: store-artifacts
-        if: env.artifact-built == 'true'
+        if: ${{ env.artifact-built }} == 'true'
         uses: actions/upload-artifact@v2
         with:
           name: kata-artifacts
@@ -124,7 +124,7 @@ jobs:
            echo "artifact-built=false" >> $GITHUB_ENV
          fi
       - name: store-artifacts
-        if: env.artifact-built == 'true'
+        if: ${{ env.artifact-built }} == 'true'
         uses: actions/upload-artifact@v2
         with:
           name: kata-artifacts
@@ -150,7 +150,7 @@ jobs:
            echo "artifact-built=false" >> $GITHUB_ENV
          fi
       - name: store-artifacts
-        if: env.artifact-built == 'true'
+        if: ${{ env.artifact-built }} == 'true'
         uses: actions/upload-artifact@v2
         with:
           name: kata-artifacts
@@ -176,7 +176,7 @@ jobs:
            echo "artifact-built=false" >> $GITHUB_ENV
          fi
       - name: store-artifacts
-        if: env.artifact-built == 'true'
+        if: ${{ env.artifact-built }} == 'true'
         uses: actions/upload-artifact@v2
         with:
           name: kata-artifacts
@@ -203,7 +203,7 @@ jobs:
            echo "artifact-built=false" >> $GITHUB_ENV
          fi
       - name: store-artifacts
-        if: env.artifact-built == 'true'
+        if: ${{ env.artifact-built }} == 'true'
         uses: actions/upload-artifact@v2
         with:
           name: kata-artifacts
@@ -229,7 +229,7 @@ jobs:
            echo "artifact-built=false" >> $GITHUB_ENV
          fi
       - name: store-artifacts
-        if: env.artifact-built == 'true'
+        if: ${{ env.artifact-built }} == 'true'
         uses: actions/upload-artifact@v2
         with:
           name: kata-artifacts

.github/workflows/snap-release.yaml

@@ -19,7 +19,7 @@ jobs:
         run: |
           sudo apt-get install -y git git-extras
           kata_url="https://github.com/kata-containers/kata-containers"
-          latest_version=$(git ls-remote --tags ${kata_url} | egrep -o "refs.*" | egrep -o "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+" | sort -V -r -u | head -1)
+          latest_version=$(git ls-remote --tags ${kata_url}  | egrep -o "refs.*" | egrep -v "\-alpha|\-rc|{}" | egrep -o "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+" | sort -V -r | head -1)
           current_version="$(echo ${GITHUB_REF} | cut -d/ -f3)"
           # Check if the current tag is the latest tag
           if echo -e "$latest_version\n$current_version" | sort -C -V; then

.github/workflows/static-checks.yaml

@@ -0,0 +1,68 @@
+on: ["pull_request"]
+name: Static checks
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.13.x, 1.14.x, 1.15.x]
+        os: [ubuntu-18.04]
+    runs-on: ${{ matrix.os }}
+    env:
+      GO111MODULE: off
+      TRAVIS: "true"
+      TRAVIS_BRANCH: ${{ github.base_ref }}
+      TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }}
+      TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }}
+      RUST_BACKTRACE: "1"
+      RUST_AGENT: "yes"
+      target_branch: ${TRAVIS_BRANCH}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+      env:
+        GOPATH: ${{ runner.workspace }}/kata-containers
+    - name: Setup GOPATH
+      run: |
+        gopath_org=$(go env GOPATH)/src/github.com/kata-containers/
+        mkdir -p ${gopath_org}
+        ln -s ${PWD} ${gopath_org}
+        echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}"
+        echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}"
+        echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}"
+        echo "TRAVIS: ${TRAVIS}"
+    - name: Set env
+      run: |
+        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
+        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+        path: ./src/github.com/${{ github.repository }}
+    - name: Setup travis references
+      run: |
+        echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}" 
+        target_branch=${TRAVIS_BRANCH}
+    - name: Setup
+      run: |
+        cd ${GOPATH}/src/github.com/kata-containers/kata-containers && ./ci/setup.sh
+      env:
+        GOPATH: ${{ runner.workspace }}/kata-containers
+    - name: Building rust
+      run: |
+        cd ${GOPATH}/src/github.com/kata-containers/kata-containers && ./ci/install_rust.sh
+        PATH=$PATH:"$HOME/.cargo/bin"
+    - name: Make clippy
+      run: |
+        cd ${GOPATH}/src/github.com/kata-containers/kata-containers/src/agent && rustup target add x86_64-unknown-linux-musl && rustup component add rustfmt && rustup component add clippy && make clippy
+    - name: Static checks
+      run: |
+        cd ${GOPATH}/src/github.com/kata-containers/kata-containers && ./ci/static-checks.sh
+    - name: Build agent
+      run: |
+        cd ${GOPATH}/src/github.com/kata-containers/kata-containers/src/agent && make
+    - name: Run agent unit tests
+      run: |
+        cd ${GOPATH}/src/github.com/kata-containers/kata-containers/src/agent && make check

.gitignore

@@ -3,6 +3,7 @@
 **/*.rej
 **/target
 **/.vscode
+pkg/logging/Cargo.lock
 src/agent/src/version.rs
 src/agent/kata-agent.service
 src/agent/protocols/src/*.rs

.travis.yml

@@ -1,62 +0,0 @@
-# Copyright (c) 2019 Ant Financial
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-dist: bionic
-os: linux
-
-# set cache directories manually, because
-# we are using a non-standard directory struct
-# cargo root is in srs/agent
-#
-# If needed, caches can be cleared
-# by ways documented in
-# https://docs.travis-ci.com/user/caching#clearing-caches
-language: rust
-rust:
-  - 1.44.1
-cache:
-  cargo: true
-  directories:
-    - src/agent/target
-
-before_install:
-  - git remote set-branches --add origin "${TRAVIS_BRANCH}"
-  - git fetch
-  - export RUST_BACKTRACE=1
-  - export target_branch=$TRAVIS_BRANCH
-  - "ci/setup.sh"
-
-# we use install to run check agent
-# so that it is easy to skip for non-amd64 platform
-install:
-  - export PATH=$PATH:"$HOME/.cargo/bin"
-  - export RUST_AGENT=yes
-  - rustup target add x86_64-unknown-linux-musl
-  - sudo ln -sf /usr/bin/g++ /bin/musl-g++
-  - rustup component add rustfmt
-  - make -C ${TRAVIS_BUILD_DIR}/src/agent
-  - make -C ${TRAVIS_BUILD_DIR}/src/agent check
-  - sudo -E PATH=$PATH make -C ${TRAVIS_BUILD_DIR}/src/agent check
-
-before_script:
-  - "ci/install_go.sh"
-  - make -C ${TRAVIS_BUILD_DIR}/src/runtime
-  - make -C ${TRAVIS_BUILD_DIR}/src/runtime test
-  - sudo -E PATH=$PATH GOPATH=$GOPATH make -C ${TRAVIS_BUILD_DIR}/src/runtime test
-
-script:
-  - "ci/static-checks.sh"
-
-jobs:
-  include:
-  - name: x86_64 test
-    os: linux
-  - name: ppc64le test
-    os: linux-ppc64le
-    install: skip
-    script: skip
-  allow_failures:
-    - name: ppc64le test
-  fast_finish: true

README.md

@@ -2,130 +2,143 @@
 
 # Kata Containers
 
-* [Raising issues](#raising-issues)
-* [Kata Containers repositories](#kata-containers-repositories)
-    * [Code Repositories](#code-repositories)
-        * [Kata Containers-developed components](#kata-containers-developed-components)
-            * [Agent](#agent)
-            * [KSM throttler](#ksm-throttler)
-            * [Runtime](#runtime)
-            * [Trace forwarder](#trace-forwarder)
-        * [Additional](#additional)
-            * [Kernel](#kernel)
-    * [CI](#ci)
-    * [Community](#community)
+* [Kata Containers](#kata-containers)
+    * [Introduction](#introduction)
+    * [Getting started](#getting-started)
     * [Documentation](#documentation)
-    * [Packaging](#packaging)
-    * [Test code](#test-code)
-    * [Utilities](#utilities)
-        * [OS builder](#os-builder)
-    * [Web content](#web-content)
+    * [Community](#community)
+    * [Getting help](#getting-help)
+        * [Raising issues](#raising-issues)
+            * [Kata Containers 1.x versions](#kata-containers-1x-versions)
+    * [Developers](#developers)
+        * [Components](#components)
+            * [Kata Containers 1.x components](#kata-containers-1x-components)
+        * [Common repositories](#common-repositories)
+        * [Packaging and releases](#packaging-and-releases)
 
 ---
 
 Welcome to Kata Containers!
 
-The purpose of this repository is to act as a "top level" site for the project. Specifically it is used:
+This repository is the home of the Kata Containers code for the 2.0 and newer
+releases.
 
-- To provide a list of the various *other* [Kata Containers repositories](#kata-containers-repositories),
-  along with a brief explanation of their purpose.
+If you want to learn about Kata Containers, visit the main
+[Kata Containers website](https://katacontainers.io).
 
-- To provide a general area for [Raising Issues](#raising-issues).
+For further details on the older (first generation) Kata Containers 1.x
+versions, see the
+[Kata Containers 1.x components](#kata-containers-1x-components)
+section.
 
-## Raising issues
+## Introduction
 
-This repository is used for [raising
-issues](https://github.com/kata-containers/kata-containers/issues/new):
+Kata Containers is an open source project and community working to build a
+standard implementation of lightweight Virtual Machines (VMs) that feel and
+perform like containers, but provide the workload isolation and security
+advantages of VMs.
 
-- That might affect multiple code repositories.
+## Getting started
 
-- Where the raiser is unsure which repositories are affected.
+See the [installation documentation](docs/install).
+
+## Documentation
+
+See the [official documentation](docs)
+(including [installation guides](docs/install),
+[the developer guide](docs/Developer-Guide.md),
+[design documents](docs/design) and more).
+
+## Community
+
+To learn more about the project, its community and governance, see the
+[community repository](https://github.com/kata-containers/community). This is
+the first place to go if you wish to contribute to the project.
+
+## Getting help
+
+See the [community](#community) section for ways to contact us.
+
+### Raising issues
+
+Please raise an issue
+[in this repository](https://github.com/kata-containers/kata-containers/issues).
 
 > **Note:**
-> 
-> - If an issue affects only a single component, it should be raised in that
->   components repository.
+> If you are reporting a security issue, please follow the [vulnerability reporting process](https://github.com/kata-containers/community#vulnerability-handling)
 
-## Kata Containers repositories
+#### Kata Containers 1.x versions
 
-### CI
+For older Kata Containers 1.x releases, please raise an issue in the
+[Kata Containers 1.x component repository](#kata-containers-1x-components)
+that seems most appropriate.
 
-The [CI](https://github.com/kata-containers/ci) repository stores the Continuous
-Integration (CI) system configuration information.
+If in doubt, raise an issue
+[in the Kata Containers 1.x runtime repository](https://github.com/kata-containers/runtime/issues).
 
-### Community
+## Developers
 
-The [Community](https://github.com/kata-containers/community) repository is
-the first place to go if you want to use or contribute to the project.
+### Components
 
-### Code Repositories
+| Component | Type | Description |
+|-|-|-|
+| [agent-ctl](tools/agent-ctl) | utility | Tool that provides low-level access for testing the agent. |
+| [agent](src/agent) | core | Management process running inside the virtual machine / POD that sets up the container environment. |
+| [documentation](docs) | documentation | Documentation common to all components (such as design and install documentation). |
+| [osbuilder](tools/osbuilder) | infrastructure | Tool to create "mini O/S" rootfs and initrd images for the hypervisor. |
+| [packaging](tools/packaging) | infrastructure | Scripts and metadata for producing packaged binaries<br/>(components, hypervisors, kernel and rootfs). |
+| [runtime](src/runtime) | core | Main component run by a container manager and providing a containerd shimv2 runtime implementation. |
+| [trace-forwarder](src/trace-forwarder) | utility | Agent tracing helper. |
 
-#### Kata Containers-developed components
+#### Kata Containers 1.x components
 
-##### Agent
+For the first generation of Kata Containers (1.x versions), each component was
+kept in a separate repository.
 
-The [`kata-agent`](src/agent/README.md) runs inside the
-virtual machine and sets up the container environment.
+For information on the Kata Containers 1.x releases, see the
+[Kata Containers 1.x releases page](https://github.com/kata-containers/runtime/releases).
 
-##### KSM throttler
+For further information on particular Kata Containers 1.x components, see the
+individual component repositories:
 
-The [`kata-ksm-throttler`](https://github.com/kata-containers/ksm-throttler)
-is an optional utility that monitors containers and deduplicates memory to
-maximize container density on a host.
+| Component | Type | Description |
+|-|-|-|
+| [agent](https://github.com/kata-containers/agent) | core | See [components](#components). |
+| [documentation](https://github.com/kata-containers/documentation) | documentation | |
+| [KSM throttler](https://github.com/kata-containers/ksm-throttler) | optional core | Daemon that monitors containers and deduplicates memory to maximize container density on the host. |
+| [osbuilder](https://github.com/kata-containers/osbuilder) | infrastructure | See [components](#components). |
+| [packaging](https://github.com/kata-containers/packaging) | infrastructure | See [components](#components). |
+| [proxy](https://github.com/kata-containers/proxy) | core | Multiplexes communications between the shims, agent and runtime. |
+| [runtime](https://github.com/kata-containers/runtime) | core | See [components](#components). |
+| [shim](https://github.com/kata-containers/shim) | core | Handles standard I/O and signals on behalf of the container process. |
 
-##### Runtime
+> **Note:**
+>
+> - There are more components for the original Kata Containers 1.x implementation.
+> - The current implementation simplifies the design significantly:
+>   compare the [current](docs/design/architecture.md) and
+>   [previous generation](https://github.com/kata-containers/documentation/blob/master/design/architecture.md)
+>   designs.
 
-The [`kata-runtime`](src/runtime/README.md) is usually
-invoked by a container manager and provides high-level verbs to manage
-containers.
+### Common repositories
 
-##### Trace forwarder
+The following repositories are used by both the current and first generation Kata Containers implementations:
 
-The [`kata-trace-forwarder`](src/trace-forwarder) is a component only used
-when tracing the [agent](#agent) process.
+| Component | Description | Current | First generation | Notes |
+|-|-|-|-|-|
+| CI | Continuous Integration configuration files and scripts. | [Kata 2.x](https://github.com/kata-containers/ci/tree/2.0-dev) | [Kata 1.x](https://github.com/kata-containers/ci/tree/master) | |
+| kernel | The Linux kernel used by the hypervisor to boot the guest image. | [Kata 2.x][kernel] | [Kata 1.x][kernel] | Patches are stored in the packaging component. |
+| tests | Test code. | [Kata 2.x](https://github.com/kata-containers/tests/tree/2.0-dev) | [Kata 1.x](https://github.com/kata-containers/tests/tree/master) | Excludes unit tests which live with the main code. |
+| www.katacontainers.io | Contains the source for the [main web site](https://www.katacontainers.io). | [Kata 2.x][github-katacontainers.io] | [Kata 1.x][github-katacontainers.io] | | |
 
-#### Additional
+### Packaging and releases
 
-##### Kernel
+Kata Containers is now
+[available natively for most distributions](docs/install/README.md#packaged-installation-methods).
+However, packaging scripts and metadata are still used to generate snap and GitHub releases. See
+the [components](#components) section for further details.
 
-The hypervisor uses a [Linux\* kernel](https://github.com/kata-containers/linux) to boot the guest image.
+---
 
-### Documentation
-
-The [docs](docs/README.md) directory holds documentation common to all code components.
-
-### Packaging
-
-We use the [packaging](tools/packaging/README.md) to create packages for the [system
-components](#kata-containers-developed-components) including
-[rootfs](#os-builder) and [kernel](#kernel) images.
-
-### Test code
-
-The [tests](https://github.com/kata-containers/tests) repository hosts all
-test code except the unit testing code (which is kept in the same repository
-as the component it tests).
-
-### Utilities
-
-#### OS builder
-
-The [osbuilder](tools/osbuilder/README.md) tool can create
-a rootfs and a "mini O/S" image. This image is used by the hypervisor to setup
-the environment before switching to the workload.
-
-#### `kata-agent-ctl`
-
-[`kata-agent-ctl`](tools/agent-ctl) is a low-level test tool for
-interacting with the agent.
-
-### Web content
-
-The
-[www.katacontainers.io](https://github.com/kata-containers/www.katacontainers.io)
-repository contains all sources for the https://www.katacontainers.io site.
-
-## Credits
-
-Kata Containers uses [packagecloud](https://packagecloud.io) for package
-hosting.
+[kernel]: https://www.kernel.org
+[github-katacontainers.io]: https://github.com/kata-containers/www.katacontainers.io

VERSION

@@ -1 +1 @@
-2.0.1
+2.1-alpha0

docs/Developer-Guide.md

@@ -104,7 +104,7 @@ The build will create the following:
 You can check if your system is capable of creating a Kata Container by running the following:
 
 ```
-$ sudo kata-runtime kata-check
+$ sudo kata-runtime check
 ```
 
 If your system is *not* able to run Kata Containers, the previous command will error out and explain why.
@@ -354,9 +354,12 @@ You MUST choose one of `alpine`, `centos`, `clearlinux`, `euleros`, and `fedora`
 >
 > - Check the [compatibility matrix](../tools/osbuilder/README.md#platform-distro-compatibility-matrix) before creating rootfs.
 
-Optionally, add your custom agent binary to the rootfs with the following:
+Optionally, add your custom agent binary to the rootfs with the following, `LIBC` default is `musl`, if `ARCH` is `ppc64le`, should set the `LIBC=gnu` and `ARCH=powerpc64le`:
 ```
-$ sudo install -o root -g root -m 0550 -T ../../agent/kata-agent ${ROOTFS_DIR}/sbin/init
+$ export ARCH=$(shell uname -m)
+$ [ ${ARCH} == "ppc64le" ] && export LIBC=gnu || export LIBC=musl
+$ [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
+$ sudo install -o root -g root -m 0550 -T ../../../src/agent/target/$(ARCH)-unknown-linux-$(LIBC)/release/kata-agent ${ROOTFS_DIR}/sbin/init
 ```
 
 ### Build an initrd image

docs/Documentation-Requirements.md

@@ -25,7 +25,7 @@ All documents must:
 - Have a `.md` file extension.
 - Include a TOC (table of contents) at the top of the document with links to
   all heading sections. We recommend using the
-  [`kata-check-markdown`](https://github.com/kata-containers/tests/tree/master/cmd/check-markdown)
+  [`check-markdown`](https://github.com/kata-containers/tests/tree/master/cmd/check-markdown)
   tool to generate the TOC.
 - Be linked to from another document in the same repository.
 

docs/README.md

@@ -40,6 +40,7 @@ See the [howto documentation](how-to).
 * [Intel QAT with Kata](./use-cases/using-Intel-QAT-and-kata.md)
 * [VPP with Kata](./use-cases/using-vpp-and-kata.md)
 * [SPDK vhost-user with Kata](./use-cases/using-SPDK-vhostuser-and-kata.md)
+* [Intel SGX with Kata](./use-cases/using-Intel-SGX-and-kata.md)
 
 ## Developer Guide
 

docs/Upgrading.md

@@ -48,10 +48,10 @@ Alternatively, if you are using Kata Containers version 1.12.0 or newer, you
 can check for newer releases using the command line:
 
 ```bash
-$ kata-runtime kata-check --check-version-only
+$ kata-runtime check --check-version-only
 ```
 
-There are various other related options. Run `kata-runtime kata-check --help`
+There are various other related options. Run `kata-runtime check --help`
 for further details.
 
 # Configuration changes

docs/design/architecture.md

@@ -58,7 +58,7 @@ to go through the VSOCK interface exported by QEMU.
 
 The container workload, that is, the actual OCI bundle rootfs, is exported from the
 host to the virtual machine.  In the case where a block-based graph driver is
-configured, `virtio-scsi` will be used. In all other cases a 9pfs VIRTIO mount point
+configured, `virtio-scsi` will be used. In all other cases a `virtio-fs` VIRTIO mount point
 will be used. `kata-agent` uses this mount point as the root filesystem for the
 container processes.
 

docs/design/kata-api-design.md

@@ -3,7 +3,6 @@ To fulfill the [Kata design requirements](kata-design-requirements.md), and base
 -  Sandbox based top API
 -  Storage and network hotplug API
 -  Plugin frameworks for external proprietary Kata runtime extensions
--  Built-in shim and proxy types and capabilities
 
 ## Sandbox Based API
 ### Sandbox Management API
@@ -57,7 +56,7 @@ To fulfill the [Kata design requirements](kata-design-requirements.md), and base
 |Name|Description|
 |---|---|
 |`sandbox.GetOOMEvent()`| Monitor the OOM events that occur in the sandbox..|
-|`sandbox.UpdateRuntimeMetrics()`| Update the shim/`hypervisor`'s metrics of the running sandbox.|
+|`sandbox.UpdateRuntimeMetrics()`| Update the `shim/hypervisor` metrics of the running sandbox.|
 |`sandbox.GetAgentMetrics()`| Get metrics of the agent and the guest in the running sandbox.|
 
 ## Plugin framework for external proprietary Kata runtime extensions
@@ -99,32 +98,3 @@ Built-in implementations include:
 
 ### Sandbox Connection Plugin Workflow
 ![Sandbox Connection Plugin Workflow](https://raw.githubusercontent.com/bergwolf/raw-contents/master/kata/Sandbox-Connection.png "Sandbox Connection Plugin Workflow")
-
-## Built-in Shim and Proxy Types and Capabilities
-### Built-in shim/proxy sandbox configurations
--  Supported shim configurations:
-
-|Name|Description|
-|---|---|
-|`noopshim`|Do not start any shim process.|
-|`ccshim`| Start the cc-shim binary.|
-|`katashim`| Start the `kata-shim` binary.|
-|`katashimbuiltin`|No standalone shim process but shim functionality APIs are exported.|
--  Supported proxy configurations:
-
-|Name|Description|
-|---|---|
-|`noopProxy`| a dummy proxy implementation of the proxy interface, only used for testing purpose.|
-|`noProxy`|generic implementation for any case where no actual proxy is needed.|
-|`ccProxy`|run `ccProxy` to proxy between runtime and agent.|
-|`kataProxy`|run `kata-proxy` to translate Yamux connections between runtime and Kata agent. |
-|`kataProxyBuiltin`| no standalone proxy process and connect to Kata agent with internal Yamux translation.|
-
-### Built-in Shim Capability
-Built-in shim capability is implemented by removing standalone shim process, and
-supporting the shim related APIs.
-
-### Built-in Proxy Capability
-Built-in proxy capability is achieved by removing standalone proxy process, and
-connecting to Kata agent with a custom gRPC dialer that is internal Yamux translation.
-The behavior is enabled when proxy is configured as `kataProxyBuiltin`.

docs/design/virtualization.md

@@ -22,10 +22,10 @@ the multiple hypervisors and virtual machine monitors that Kata supports.
 ## Mapping container concepts to virtual machine technologies
 
 A typical deployment of Kata Containers will be in Kubernetes by way of a Container Runtime Interface (CRI) implementation. On every node,
-Kubelet will interact with a CRI implementor (such as containerd or CRI-O), which will in turn interface with Kata Containers (an OCI based runtime).
+Kubelet will interact with a CRI implementer (such as containerd or CRI-O), which will in turn interface with Kata Containers (an OCI based runtime).
 
 The CRI API, as defined at the [Kubernetes CRI-API repo](https://github.com/kubernetes/cri-api/), implies a few constructs being supported by the
-CRI implementation, and ultimately in Kata Containers. In order to support the full [API](https://github.com/kubernetes/cri-api/blob/a6f63f369f6d50e9d0886f2eda63d585fbd1ab6a/pkg/apis/runtime/v1alpha2/api.proto#L34-L110) with the CRI-implementor, Kata must provide the following constructs:
+CRI implementation, and ultimately in Kata Containers. In order to support the full [API](https://github.com/kubernetes/cri-api/blob/a6f63f369f6d50e9d0886f2eda63d585fbd1ab6a/pkg/apis/runtime/v1alpha2/api.proto#L34-L110) with the CRI-implementer, Kata must provide the following constructs:
 
 ![API to construct](./arch-images/api-to-construct.png)
 
@@ -41,14 +41,9 @@ Each hypervisor or VMM varies on how or if it handles each of these.
 
 ## Kata Containers Hypervisor and VMM support
 
-Kata Containers is designed to support multiple virtual machine monitors (VMMs) and hypervisors.
-Kata Containers supports:
- - [ACRN hypervisor](https://projectacrn.org/)
- - [Cloud Hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor)/[KVM](https://www.linux-kvm.org/page/Main_Page)
- - [Firecracker](https://github.com/firecracker-microvm/firecracker)/KVM
- - [QEMU](http://www.qemu-project.org/)/KVM
+Kata Containers [supports multiple hypervisors](../hypervisors.md).
 
-Which configuration to use will depend on the end user's requirements. Details of each solution and a summary are provided below.
+Details of each solution and a summary are provided below.
 
 ### QEMU/KVM
 
@@ -62,7 +57,7 @@ be changed by editing the runtime [`configuration`](./architecture.md/#configura
 Devices and features used:
 - virtio VSOCK or virtio serial
 - virtio block or virtio SCSI
-- virtio net
+- [virtio net](https://www.redhat.com/en/virtio-networking-series)
 - virtio fs or virtio 9p (recommend: virtio fs)
 - VFIO
 - hotplug
@@ -105,25 +100,34 @@ Devices used:
 
 ### Cloud Hypervisor/KVM
 
-Cloud Hypervisor, based on [rust-VMM](https://github.com/rust-vmm), is designed to have a lighter footprint and attack surface. For Kata Containers,
-relative to Firecracker, the Cloud Hypervisor configuration provides better compatibility at the expense of exposing additional devices: file system
-sharing and direct device assignment.  As of the 1.10 release of Kata Containers, Cloud Hypervisor does not support device hotplug, and as a result
-does not support updating container resources after boot, or utilizing block based volumes. While Cloud Hypervisor does support VFIO, Kata is still adding
-this support. As of 1.10, Kata does not support block based volumes or direct device assignment. See [Cloud Hypervisor device support documentation](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/device_model.md)
-for more details on Cloud Hypervisor.
+[Cloud Hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor), based
+on [rust-vmm](https://github.com/rust-vmm), is designed to have a
+lighter footprint and smaller attack surface for running modern cloud
+workloads. Kata Containers with Cloud
+Hypervisor provides mostly complete compatibility with Kubernetes
+comparable to the QEMU configuration. As of the 1.12 and 2.0.0 release
+of Kata Containers, the Cloud Hypervisor configuration supports both CPU
+and memory resize, device hotplug (disk and VFIO), file-system sharing through virtio-fs,
+block-based volumes, booting from VM images backed by pmem device, and
+fine-grained seccomp filters for each VMM threads (e.g. all virtio
+device worker threads). Please check [this GitHub Project](https://github.com/orgs/kata-containers/projects/21)
+for details of ongoing integration efforts.
 
-Devices used:
-- virtio VSOCK
+Devices and features used:
+- virtio VSOCK or virtio serial
 - virtio block
 - virtio net
 - virtio fs
+- virtio pmem
+- VFIO
+- hotplug
+- seccomp filters
+- [HTTP OpenAPI](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/vmm/src/api/openapi/cloud-hypervisor.yaml)
 
 ### Summary
 
 | Solution | release introduced | brief summary |
 |-|-|-|
-| QEMU | 1.0 | upstream QEMU, with support for hotplug and filesystem sharing |
-| NEMU | 1.4 | Deprecated, removed as of 1.10 release. Slimmed down fork of QEMU, with experimental support of virtio-fs |
+| Cloud Hypervisor | 1.10 | upstream Cloud Hypervisor with rich feature support, e.g. hotplug, VFIO and FS sharing|
 | Firecracker | 1.5 | upstream Firecracker, rust-VMM based, no VFIO, no FS sharing, no memory/CPU hotplug |
-| QEMU-virtio-fs | 1.7 | upstream QEMU with support for virtio-fs. Will be removed once virtio-fs lands in upstream QEMU |
-| Cloud Hypervisor | 1.10 |  rust-VMM based, includes VFIO and FS sharing through virtio-fs, no hotplug |
+| QEMU | 1.0 | upstream QEMU, with support for hotplug and filesystem sharing |

docs/how-to/how-to-load-kernel-modules-with-kata.md

@@ -56,8 +56,9 @@ There are some limitations with this approach:
 
 As was mentioned above, not all containers need the same modules, therefore using
 the configuration file for specifying the list of kernel modules per [POD][3] can
-be a pain. Unlike the configuration file, annotations provide a way to specify
-custom configurations per POD.
+be a pain.
+Unlike the configuration file, [annotations](how-to-set-sandbox-config-kata.md)
+provide a way to specify custom configurations per POD.
 
 The list of kernel modules and parameters can be set using the annotation
 `io.katacontainers.config.agent.kernel_modules` as a semicolon separated
@@ -101,7 +102,7 @@ spec:
     tty: true
 ```
 
-> **Note**: To pass annotations to Kata containers, [`CRI` must to be configured correctly](how-to-set-sandbox-config-kata.md#cri-configuration)
+> **Note**: To pass annotations to Kata containers, [CRI-O must be configured correctly](how-to-set-sandbox-config-kata.md#cri-o-configuration)
 
 [1]: ../../src/runtime
 [2]: ../../src/agent

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -3,6 +3,11 @@
 Kata Containers gives users freedom to customize at per-pod level, by setting
 a wide range of Kata specific annotations in the pod specification.
 
+Some annotations may be [restricted](#restricted-annotations) by the
+configuration file for security reasons, notably annotations that could lead the
+runtime to execute programs on the host. Such annotations are marked with _(R)_ in
+the tables below.
+
 # Kata Configuration Annotations
 There are several kinds of Kata configurations and they are listed below.
 
@@ -26,6 +31,7 @@ There are several kinds of Kata configurations and they are listed below.
 | Key | Value Type | Comments |
 |-------| ----- | ----- |
 | `io.katacontainers.config.agent.enable_tracing` | `boolean` | enable tracing for the agent |
+| `io.katacontainers.config.agent.container_pipe_size` | uint32 | specify the size of the std(in/out) pipes created for containers |
 | `io.katacontainers.config.agent.kernel_modules` | string | the list of kernel modules and their parameters that will be loaded in the guest kernel. Semicolon separated list of kernel modules and their parameters. These modules will be loaded in the guest kernel using `modprobe`(8). E.g., `e1000e InterruptThrottleRate=3000,3000,3000 EEE=1; i915 enable_ppgtt=0` |
 | `io.katacontainers.config.agent.trace_mode` | string | the trace mode for the agent |
 | `io.katacontainers.config.agent.trace_type` | string | the trace type for the agent |
@@ -38,17 +44,24 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.block_device_cache_noflush` | `boolean` | Denotes whether flush requests for the device are ignored |
 | `io.katacontainers.config.hypervisor.block_device_cache_set` | `boolean` | cache-related options will be set to block devices or not |
 | `io.katacontainers.config.hypervisor.block_device_driver` | string | the driver to be used for block device, valid values are `virtio-blk`, `virtio-scsi`, `nvdimm`|
+| `io.katacontainers.config.hypervisor.cpu_features` | `string` | Comma-separated list of CPU features to pass to the CPU (QEMU) |
+| `io.katacontainers.config.hypervisor.ctlpath` (R) | `string` | Path to the `acrnctl` binary for the ACRN hypervisor |
 | `io.katacontainers.config.hypervisor.default_max_vcpus` | uint32| the maximum number of vCPUs allocated for the VM by the hypervisor |
 | `io.katacontainers.config.hypervisor.default_memory` | uint32| the memory assigned for a VM by the hypervisor in `MiB` |
 | `io.katacontainers.config.hypervisor.default_vcpus` | uint32| the default vCPUs assigned for a VM by the hypervisor |
 | `io.katacontainers.config.hypervisor.disable_block_device_use` | `boolean` | disallow a block device from being used |
+| `io.katacontainers.config.hypervisor.disable_image_nvdimm` | `boolean` | specify if a `nvdimm` device should be used as rootfs for the guest (QEMU) |
 | `io.katacontainers.config.hypervisor.disable_vhost_net` | `boolean` | specify if `vhost-net` is not available on the host |
 | `io.katacontainers.config.hypervisor.enable_hugepages` | `boolean` | if the memory should be `pre-allocated` from huge pages |
+| `io.katacontainers.config.hypervisor.enable_iommu_platform` | `boolean` | enable `iommu` on CCW devices (QEMU s390x) |
+| `io.katacontainers.config.hypervisor.enable_iommu` | `boolean` | enable `iommu` on Q35 (QEMU x86_64) |
 | `io.katacontainers.config.hypervisor.enable_iothreads` | `boolean`| enable IO to be processed in a separate thread. Supported currently for virtio-`scsi` driver |
 | `io.katacontainers.config.hypervisor.enable_mem_prealloc` | `boolean` | the memory space used for `nvdimm` device by the hypervisor |
 | `io.katacontainers.config.hypervisor.enable_swap` | `boolean` | enable swap of VM memory |
+| `io.katacontainers.config.hypervisor.enable_vhost_user_store` | `boolean` | enable vhost-user storage device (QEMU) |
+| `io.katacontainers.config.hypervisor.enable_virtio_mem` | `boolean` | enable virtio-mem (QEMU) |
 | `io.katacontainers.config.hypervisor.entropy_source` | string| the path to a host source of entropy (`/dev/random`, `/dev/urandom` or real hardware RNG device) |
-| `io.katacontainers.config.hypervisor.file_mem_backend` | string | file based memory backend root directory |
+| `io.katacontainers.config.hypervisor.file_mem_backend` (R) | string | file based memory backend root directory |
 | `io.katacontainers.config.hypervisor.firmware_hash` | string | container firmware SHA-512 hash value |
 | `io.katacontainers.config.hypervisor.firmware` | string | the guest firmware that will run the container VM |
 | `io.katacontainers.config.hypervisor.guest_hook_path` | string | the path within the VM that will be used for drop in hooks |
@@ -59,7 +72,7 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.initrd_hash` | string | container guest initrd SHA-512 hash value |
 | `io.katacontainers.config.hypervisor.initrd` | string | the guest initrd image that will run in the container VM |
 | `io.katacontainers.config.hypervisor.jailer_hash` | string | container jailer SHA-512 hash value |
-| `io.katacontainers.config.hypervisor.jailer_path` | string | the jailer that will constrain the container VM |
+| `io.katacontainers.config.hypervisor.jailer_path` (R) | string | the jailer that will constrain the container VM |
 | `io.katacontainers.config.hypervisor.kernel_hash` | string | container kernel image SHA-512 hash value |
 | `io.katacontainers.config.hypervisor.kernel_params` | string | additional guest kernel parameters |
 | `io.katacontainers.config.hypervisor.kernel` | string | the kernel used to boot the container VM |
@@ -69,14 +82,16 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.memory_slots` | uint32| the memory slots assigned to the VM by the hypervisor |
 | `io.katacontainers.config.hypervisor.msize_9p` | uint32 | the `msize` for 9p shares |
 | `io.katacontainers.config.hypervisor.path` | string | the hypervisor that will run the container VM |
+| `io.katacontainers.config.hypervisor.pcie_root_port` | specify the number of PCIe Root Port devices. The PCIe Root Port device is used to hot-plug a PCIe device (QEMU) |
 | `io.katacontainers.config.hypervisor.shared_fs` | string | the shared file system type, either `virtio-9p` or `virtio-fs` |
 | `io.katacontainers.config.hypervisor.use_vsock` | `boolean` | specify use of `vsock` for agent communication |
+| `io.katacontainers.config.hypervisor.vhost_user_store_path` (R) | `string` | specify the directory path where vhost-user devices related folders, sockets and device nodes should be (QEMU) |
 | `io.katacontainers.config.hypervisor.virtio_fs_cache_size` | uint32 | virtio-fs DAX cache size in `MiB` |
 | `io.katacontainers.config.hypervisor.virtio_fs_cache` | string | the cache mode for virtio-fs, valid values are `always`, `auto` and `none` |
 | `io.katacontainers.config.hypervisor.virtio_fs_daemon` | string | virtio-fs `vhost-user` daemon path |
 | `io.katacontainers.config.hypervisor.virtio_fs_extra_args` | string | extra options passed to `virtiofs` daemon |
 
-# CRI Configuration
+# CRI-O Configuration
 
 In case of CRI-O, all annotations specified in the pod spec are passed down to Kata.
 
@@ -101,7 +116,7 @@ $ cat /etc/containerd/config
 
 ```
 
-Additional documentation on the above configuration can be found in the 
+Additional documentation on the above configuration can be found in the
 [containerd docs](https://github.com/containerd/cri/blob/8d5a8355d07783ba2f8f451209f6bdcc7c412346/docs/config.md).
 
 # Example - Using annotations
@@ -159,3 +174,31 @@ spec:
     stdin: true
     tty: true
 ```
+
+# Restricted annotations
+
+Some annotations are _restricted_, meaning that the configuration file specifies
+the acceptable values. Currently, only hypervisor annotations are restricted,
+for security reason, with the intent to control which binaries the Kata
+Containers runtime will launch on your behalf.
+
+The configuration file validates the annotation _name_ as well as the annotation
+_value_.
+
+The acceptable annotation names are defined by the `enable_annotations` entry in
+the configuration file.
+
+For restricted annotations, an additional configuration entry provides a list of
+acceptable values. Since most restricted annotations are intended to control
+which binaries the runtime can execute, the valid value is generally provided by
+a shell pattern, as defined by `glob(3)`. The table below provides the name of
+the configuration entry:
+
+| Key | Config file entry | Comments |
+|-------| ----- | ----- |
+| `ctlpath`  | `valid_ctlpaths` | Valid paths for `acrnctl` binary |
+| `file_mem_backend`  | `valid_file_mem_backends` | Valid locations for the file-based memory backend root directory |
+| `jailer_path`  | `valid_jailer_paths`| Valid paths for the jailer constraining the container VM (Firecracker) |
+| `path`  | `valid_hypervisor_paths` | Valid hypervisors to run the container VM |
+| `vhost_user_store_path`  | `valid_vhost_user_store_paths` | Valid paths for vhost-user related files|
+| `virtio_fs_daemon`  | `valid_virtio_fs_daemon_paths` | Valid paths for the `virtiofsd` daemon |

docs/how-to/what-is-vm-templating-and-how-do-I-use-it.md

@@ -46,6 +46,7 @@ overridden by `/etc/kata-containers/configuration.toml` if provided) such that:
   - `enable_template = true`
   - `initrd =` is set
   - `image =` option is commented out or removed
+  - `shared_fs` should not be `virtio-fs`
 
 Then you can create a VM templating for later usage by calling
 ```

docs/hypervisors.md

@@ -0,0 +1,68 @@
+# Hypervisors
+
+* [Hypervisors](#hypervisors)
+    * [Introduction](#introduction)
+    * [Types](#types)
+    * [Determine currently configured hypervisor](#determine-currently-configured-hypervisor)
+    * [Choose a Hypervisor](#choose-a-hypervisor)
+
+## Introduction
+
+Kata Containers supports multiple hypervisors. This document provides a very
+high level overview of the available hypervisors, giving suggestions as to
+which hypervisors you may wish to investigate further.
+
+> **Note:**
+>
+> This document is not prescriptive or authoritative:
+>
+> - It is up to you to decide which hypervisors may be most appropriate for
+>   your use-case.
+> - Refer to the official documentation for each hypervisor for further details.
+
+## Types
+
+Since each hypervisor offers different features and options, Kata Containers
+provides a separate
+[configuration file](/src/runtime/README.md#configuration)
+for each. The configuration files contain comments explaining which options
+are available, their default values and how each setting can be used.
+
+> **Note:**
+>
+> The simplest way to switch between hypervisors is to create a symbolic link
+> to the appropriate hypervisor-specific configuration file.
+
+| Hypervisor | Written in | Architectures | Type | Configuration file |
+|-|-|-|-|-|
+[ACRN] | C | `x86_64` | Type 1 (bare metal) | `configuration-acrn.toml` |
+[Cloud Hypervisor] | rust | `aarch64`, `x86_64` | Type 2 ([KVM]) | `configuration-clh.toml` |
+[Firecracker] | rust | `aarch64`, `x86_64` | Type 2 ([KVM]) | `configuration-fc.toml` |
+[QEMU] | C | all | Type 2 ([KVM]) | `configuration-qemu.toml` |
+
+## Determine currently configured hypervisor
+
+```bash
+$ kata-runtime kata-env | awk -v RS= '/\[Hypervisor\]/' | grep Path
+```
+
+## Choose a Hypervisor
+
+The table below provides a brief summary of some of the differences between
+the hypervisors:
+
+
+| Hypervisor | Summary | Features | Limitations | Container Creation speed | Memory density | Use cases | Comment |
+|-|-|-|-|-|-|-|-|
+[ACRN] | Safety critical and real-time workloads | | | excellent | excellent | Embedded and IOT systems | For advanced users |
+[Cloud Hypervisor] | Low latency, small memory footprint, small attack surface | Minimal | | excellent | excellent | High performance modern cloud workloads | |
+[Firecracker] | Very slimline | Extremely minimal | Doesn't support all device types | excellent | excellent | Serverless / FaaS | |
+[QEMU] | Lots of features | Lots | | good | good | Good option for most users | | All users |
+
+For further details, see the [Virtualization in Kata Containers](design/virtualization.md) document and the official documentation for each hypervisor.
+
+[ACRN]: https://projectacrn.org
+[Cloud Hypervisor]: https://github.com/cloud-hypervisor/cloud-hypervisor
+[Firecracker]: https://github.com/firecracker-microvm/firecracker
+[KVM]: https://en.wikipedia.org/wiki/Kernel-based_Virtual_Machine
+[QEMU]: http://www.qemu-project.org

docs/install/container-manager/containerd/containerd-install.md

@@ -18,7 +18,7 @@
 >
 > - If you decide to proceed and install a Kata Containers release, you can
 >   still check for the latest version of Kata Containers by running
->   `kata-runtime kata-check --only-list-releases`.
+>   `kata-runtime check --only-list-releases`.
 >
 > - These instructions will not work for Fedora 31 and higher since those
 >   distribution versions only support cgroups version 2 by default. However,

docs/install/gce-installation-guide.md

@@ -6,7 +6,7 @@
 * [Install Kata](#install-kata)
 * [Create a Kata-enabled Image](#create-a-kata-enabled-image)
 
-Kata Containers on Google Compute Engine (GCE) makes use of [nested virtualization](https://cloud.google.com/compute/docs/instances/enable-nested-virtualization-vm-instances). Most of the installation procedure is identical to that for Kata on your preferred distribution, but enabling nested virtualization currently requires extra steps on GCE. This guide walks you through creating an image and instance with nested virtualization enabled. Note that `kata-runtime kata-check` checks for nested virtualization, but does not fail if support is not found.
+Kata Containers on Google Compute Engine (GCE) makes use of [nested virtualization](https://cloud.google.com/compute/docs/instances/enable-nested-virtualization-vm-instances). Most of the installation procedure is identical to that for Kata on your preferred distribution, but enabling nested virtualization currently requires extra steps on GCE. This guide walks you through creating an image and instance with nested virtualization enabled. Note that `kata-runtime check` checks for nested virtualization, but does not fail if support is not found.
 
 As a pre-requisite this guide assumes an installed and configured instance of the [Google Cloud SDK](https://cloud.google.com/sdk/downloads). For a zero-configuration option, all of the commands below were been tested under [Google Cloud Shell](https://cloud.google.com/shell/) (as of Jun 2018). Verify your `gcloud` installation and configuration:
 

docs/install/minikube-installation-guide.md

@@ -54,7 +54,7 @@ to enable nested virtualization can be found on the
 [KVM Nested Guests page](https://www.linux-kvm.org/page/Nested_Guests)
 
 Alternatively, and for other architectures, the Kata Containers built in
-[`kata-check`](../../src/runtime/README.md#hardware-requirements)
+[`check`](../../src/runtime/README.md#hardware-requirements)
 command can be used *inside Minikube* once Kata has been installed, to check for compatibility.
 
 ## Setting up Minikube

docs/install/snap-installation-guide.md

@@ -1,13 +1,123 @@
-# Install Kata Containers from `snapcraft.io`
+# Kata Containers snap package
+
+* [Install Kata Containers](#install-kata-containers)
+* [Configure Kata Containers](#configure-kata-containers)
+* [Integration with non-compatible shim v2 Container Engines](#integration-with-non-compatible-shim-v2-container-engines)
+    * [Integration with Docker](#integration-with-docker)
+    * [Integration with Podman](#integration-with-podman)
+* [Integration with shim v2 Container Engines](#integration-with-shim-v2-container-engines)
+* [Remove Kata Containers snap package](#remove-kata-containers-snap-package)
+
+
+## Install Kata Containers
 
 Kata Containers can be installed in any Linux distribution that supports
 [snapd](https://docs.snapcraft.io/installing-snapd).
 
-Run the following command to install Kata Containers:
+> NOTE: From Kata Containers 2.x, only the [Containerd Runtime V2 (Shim API)](https://github.com/containerd/containerd/tree/master/runtime/v2)
+> is supported, note that some container engines (`docker`, `podman`, etc) may not
+> be able to run Kata Containers 2.x.
 
-   ```bash
-   $ sudo snap install kata-containers --classic
-   ```
+Kata Containers 1.x is released through the *stable* channel while Kata Containers
+2.x is available in the *candidate* channel.
 
-For further information on integrating and configuring the `snap` Kata Containers install,
-refer to the [Kata Containers packaging `snap` documentation](https://github.com/kata-containers/packaging/blob/master/snap/README.md#configure-kata-containers).
+Run the following command to install **Kata Containers 1.x**:
+
+```sh
+$ sudo snap install kata-containers --classic
+```
+
+Run the following command to install **Kata Containers 2.x**:
+
+```sh
+$ sudo snap install kata-containers --candidate --classic
+```
+
+## Configure Kata Containers
+
+By default Kata Containers snap image is mounted at `/snap/kata-containers` as a
+read-only file system, therefore default configuration file can not be edited.
+Fortunately Kata Containers supports loading a configuration file from another
+path than the default.
+
+```sh
+$ sudo mkdir -p /etc/kata-containers
+$ sudo cp /snap/kata-containers/current/usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers/
+$ $EDITOR /etc/kata-containers/configuration.toml
+```
+
+## Integration with non-compatible shim v2 Container Engines
+
+At the time of writing this document, `docker` and `podman` **do not support Kata
+Containers 2.x, therefore Kata Containers 1.x must be used instead.**
+
+The path to the runtime provided by the Kata Containers 1.x snap package is
+`/snap/bin/kata-containers.runtime`, it should be used to run Kata Containers 1.x.
+
+### Integration with Docker
+
+`/etc/docker/daemon.json` is the configuration file for `docker`, use the
+following configuration to add a new runtime (`kata`) to `docker`.
+
+```json
+{
+  "runtimes": {
+    "kata": {
+      "path": "/snap/bin/kata-containers.runtime"
+    }
+  }
+}
+```
+
+Once the above configuration has been applied, use the
+following commands to restart `docker` and run Kata Containers 1.x.
+
+```sh
+$ sudo systemctl restart docker
+$ docker run -ti --runtime kata busybox sh
+```
+
+### Integration with Podman
+
+`/usr/share/containers/containers.conf` is the configuration file for `podman`,
+add the following configuration in the `[engine.runtimes]` section.
+
+```toml
+kata = [
+   "/snap/bin/kata-containers.runtime"
+]
+```
+
+Once the above configuration has been applied, use the following command to run
+Kata Containers 1.x with `podman`
+
+```sh
+$ sudo podman run -ti --runtime kata docker.io/library/busybox sh
+```
+
+## Integration with shim v2 Container Engines
+
+The Container engine daemon (`cri-o`, `containerd`, etc) needs to be able to find the
+`containerd-shim-kata-v2` binary to allow Kata Containers to be created.
+Run the following command to create a symbolic link to the shim v2 binary.
+
+```sh
+$ sudo ln -sf /snap/kata-containers/current/usr/bin/containerd-shim-kata-v2 /usr/local/bin/containerd-shim-kata-v2
+```
+
+Once the symbolic link has been created and the engine daemon configured, `io.containerd.kata.v2`
+can be used as runtime.
+
+Read the following documents to know how to run Kata Containers 2.x with `containerd`.
+
+* [How to use Kata Containers and Containerd](https://github.com/kata-containers/kata-containers/blob/2.0-dev/docs/how-to/containerd-kata.md)
+* [Install Kata Containers with containerd](https://github.com/kata-containers/kata-containers/blob/2.0-dev/docs/install/container-manager/containerd/containerd-install.md)
+
+
+## Remove Kata Containers snap package
+
+Run the following command to remove the Kata Containers snap:
+
+```sh
+$ sudo snap remove kata-containers
+```

docs/use-cases/using-Intel-SGX-and-kata.md

@@ -0,0 +1,112 @@
+# Kata Containers with SGX
+
+- [Check if SGX is enabled](#check-if-sgx-is-enabled)
+- [Install Host kernel with SGX support](#install-host-kernel-with-sgx-support)
+- [Install Guest kernel with SGX support](#install-guest-kernel-with-sgx-support)
+- [Run Kata Containers with SGX enabled](#run-kata-containers-with-sgx-enabled)
+
+Intel® Software Guard Extensions (SGX) is a set of instructions that increases the security
+of applications code and data, giving them more protections from disclosure or modification.
+
+> **Note:** At the time of writing this document, SGX patches have not landed on the Linux kernel
+> project, so specific versions for guest and host kernels must be installed to enable SGX.
+
+## Check if SGX is enabled
+
+Run the following command to check if your host supports SGX.
+
+```sh
+$ grep -o sgx /proc/cpuinfo
+```
+
+Continue to the following section if the output of the above command is empty,
+otherwise continue to section [Install Guest kernel with SGX support](#install-guest-kernel-with-sgx-support)
+
+## Install Host kernel with SGX support
+
+The following commands were tested on Fedora 32, they might work on other distros too.
+
+```sh
+$ git clone --depth=1 https://github.com/intel/kvm-sgx
+$ pushd kvm-sgx
+$ cp /boot/config-$(uname -r) .config
+$ yes "" | make oldconfig
+$ # In the following step, enable: INTEL_SGX and INTEL_SGX_VIRTUALIZATION
+$ make menuconfig
+$ make -j$(($(nproc)-1)) bzImage
+$ make -j$(($(nproc)-1)) modules
+$ sudo make modules_install
+$ sudo make install
+$ popd
+$ sudo reboot
+```
+
+> **Notes:**
+> * Run: `mokutil --sb-state` to check whether secure boot is enabled, if so, you will need to sign the kernel.
+> * You'll lose SGX support when a new distro kernel is installed and the system rebooted.
+
+Once you have restarted your system with the new brand Linux Kernel with SGX support, run
+the following command to make sure it's enabled. If the output is empty, go to the BIOS
+setup and enable SGX manually.
+
+```sh
+$ grep -o sgx /proc/cpuinfo
+```
+
+## Install Guest kernel with SGX support
+
+Install the guest kernel in the Kata Containers directory, this way it can be used to run
+Kata Containers.
+
+```sh
+$ curl -LOk https://github.com/devimc/kvm-sgx/releases/download/v0.0.1/kata-virtiofs-sgx.tar.gz
+$ sudo tar -xf kata-virtiofs-sgx.tar.gz -C /usr/share/kata-containers/
+$ sudo sed -i 's|kernel =|kernel = "/usr/share/kata-containers/vmlinux-virtiofs-sgx.container"|g' \
+  /usr/share/defaults/kata-containers/configuration.toml
+```
+
+## Run Kata Containers with SGX enabled
+
+Before running a Kata Container make sure that your version of `crio` or `containerd`
+supports annotations.
+For `containerd` check in `/etc/containerd/config.toml` that the list of `pod_annotations` passed
+to the `sandbox` are: `["io.katacontainers.*", "sgx.intel.com/epc"]`.
+
+> `sgx.yaml`
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+  name: sgx
+  annotations:
+    sgx.intel.com/epc: "32Mi"
+spec:
+  terminationGracePeriodSeconds: 0
+  runtimeClassName: kata
+  containers:
+  - name: c1
+    image: busybox
+    command:
+        - sh
+    stdin: true
+    tty: true
+    volumeMounts:
+    - mountPath: /dev/sgx/
+      name: test-volume
+  volumes:
+  - name: test-volume
+    hostPath:
+      path: /dev/sgx/
+      type: Directory
+```
+
+```sh
+$ kubectl apply -f sgx.yaml
+$ kubectl exec -ti sgx ls /dev/sgx/
+enclave    provision
+```
+
+The output of the latest command shouldn't be empty, otherwise check
+your system environment to make sure SGX is fully supported.
+
+[1]: github.com/cloud-hypervisor/cloud-hypervisor/

pkg/logging/src/lib.rs

@@ -93,9 +93,7 @@ impl HashSerializer {
         // Take care to only add the first instance of a key. This matters for loggers (but not
         // Records) since a child loggers have parents and the loggers are serialised child first
         // meaning the *newest* fields are serialised first.
-        if !self.fields.contains_key(&key) {
-            self.fields.insert(key, value);
-        }
+        self.fields.entry(key).or_insert(value);
     }
 
     fn remove_field(&mut self, key: &str) {

snap/snapcraft.yaml

@@ -305,4 +305,8 @@ parts:
 
 apps:
   runtime:
+    command: usr/bin/kata-runtime
+  shim:
     command: usr/bin/containerd-shim-kata-v2
+  collect-data:
+    command: usr/bin/kata-collect-data.sh

src/agent/Cargo.lock

@@ -143,13 +143,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
-name = "cgroups"
-version = "0.1.1-alpha.0"
-source = "git+https://github.com/kata-containers/cgroups-rs?branch=stable-0.1.1#8717524f2c95aacd30768b6f0f7d7f2fddef5cac"
+name = "cgroups-rs"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "02274214de2526e48355facdd16c9d774bba2cf74d135ffb9876a60b4d613464"
 dependencies = [
  "libc",
  "log",
  "nix 0.18.0",
+ "procinfo",
  "regex",
 ]
 
@@ -248,6 +250,16 @@ version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
 
+[[package]]
+name = "epoll"
+version = "4.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "20df693c700404f7e19d4d6fae6b15215d2913c27955d2b9d6f2c0f537511cd0"
+dependencies = [
+ "bitflags",
+ "libc",
+]
+
 [[package]]
 name = "errno"
 version = "0.2.6"
@@ -372,9 +384,10 @@ name = "kata-agent"
 version = "0.1.0"
 dependencies = [
  "anyhow",
- "cgroups",
+ "cgroups-rs",
  "lazy_static",
  "libc",
+ "log",
  "logging",
  "netlink",
  "nix 0.17.0",
@@ -392,6 +405,7 @@ dependencies = [
  "signal-hook",
  "slog",
  "slog-scope",
+ "slog-stdlog",
  "tempfile",
  "ttrpc",
 ]
@@ -545,6 +559,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "nom"
+version = "2.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf51a729ecf40266a2368ad335a5fdde43471f545a967109cd62146ecf8b66ff"
+
 [[package]]
 name = "num-integer"
 version = "0.1.43"
@@ -681,6 +701,18 @@ dependencies = [
  "libflate",
 ]
 
+[[package]]
+name = "procinfo"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ab1427f3d2635891f842892dda177883dca0639e05fe66796a62c9d2f23b49c"
+dependencies = [
+ "byteorder",
+ "libc",
+ "nom",
+ "rustc_version",
+]
+
 [[package]]
 name = "prometheus"
 version = "0.9.0"
@@ -911,14 +943,24 @@ version = "0.1.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232"
 
+[[package]]
+name = "rustc_version"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
+dependencies = [
+ "semver",
+]
+
 [[package]]
 name = "rustjail"
 version = "0.1.0"
 dependencies = [
  "anyhow",
  "caps",
- "cgroups",
+ "cgroups-rs",
  "dirs",
+ "epoll",
  "lazy_static",
  "libc",
  "nix 0.17.0",
@@ -960,6 +1002,21 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
+[[package]]
+name = "semver"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
+dependencies = [
+ "semver-parser",
+]
+
+[[package]]
+name = "semver-parser"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
+
 [[package]]
 name = "serde"
 version = "1.0.117"
@@ -1077,6 +1134,17 @@ dependencies = [
  "slog",
 ]
 
+[[package]]
+name = "slog-stdlog"
+version = "4.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8228ab7302adbf4fcb37e66f3cda78003feb521e7fd9e3847ec117a7784d0f5a"
+dependencies = [
+ "log",
+ "slog",
+ "slog-scope",
+]
+
 [[package]]
 name = "smallvec"
 version = "1.4.2"

src/agent/Cargo.toml

@@ -21,18 +21,24 @@ signal-hook = "0.1.9"
 scan_fmt = "0.2.3"
 scopeguard = "1.0.0"
 regex = "1"
+
 # slog:
 # - Dynamic keys required to allow HashMap keys to be slog::Serialized.
 # - The 'max_*' features allow changing the log level at runtime
 #   (by stopping the compiler from removing log calls).
 slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_info"] }
 slog-scope = "4.1.2"
+
+# Redirect ttrpc log calls
+slog-stdlog = "4.0.0"
+log = "0.4.11"
+
 # for testing
 tempfile = "3.1.0"
 prometheus = { version = "0.9.0", features = ["process"] }
 procfs = "0.7.9"
 anyhow = "1.0.32"
-cgroups = { git = "https://github.com/kata-containers/cgroups-rs", branch = "stable-0.1.1"}
+cgroups = { package = "cgroups-rs", version = "0.2.0" }
 
 [workspace]
 members = [
@@ -41,3 +47,6 @@ members = [
     "protocols",
     "rustjail",
 ]
+
+[profile.release]
+lto = true

src/agent/Makefile

@@ -116,6 +116,12 @@ optimize: $(SOURCES) | show-summary show-header
 show-header:
 	@printf "%s - version %s (commit %s)\n\n" "$(TARGET)" "$(VERSION)" "$(COMMIT_MSG)"
 
+clippy: $(GENERATED_CODE)
+	cargo clippy --all-targets --all-features --release \
+		-- \
+		-Aclippy::redundant_allocation \
+		-D warnings
+
 $(GENERATED_FILES): %: %.in
 	@sed $(foreach r,$(GENERATED_REPLACEMENTS),-e 's|@$r@|$($r)|g') "$<" > "$@"
 

src/agent/README.md

@@ -39,11 +39,22 @@ After that, we drafted the initial code here, and any contributions are welcome.
 ## Getting Started
 
 ### Build from Source
-The rust-agent need to be built with rust newer than 1.37, and static linked with `musl`.
+The rust-agent needs to be built statically and linked with `musl`
+
+> **Note:** skip this step for ppc64le, the build scripts explicitly use gnu for ppc64le.
+
 ```bash
-rustup target add x86_64-unknown-linux-musl
-sudo ln -s /usr/bin/g++ /bin/musl-g++  
-cargo build --target x86_64-unknown-linux-musl --release
+$ arch=$(uname -m)
+$ rustup target add "${arch}-unknown-linux-musl"
+$ sudo ln -s /usr/bin/g++ /bin/musl-g++
+```
+
+Download the source files in the Kata containers repository and build the agent:
+```bash
+$ GOPATH="${GOPATH:-$HOME/go}"
+$ dir="$GOPATH/src/github.com/kata-containers"
+$ git -C ${dir} clone --depth 1 https://github.com/kata-containers/kata-containers
+$ make -C ${dir}/kata-containers/src/agent
 ```
 
 ## Run Kata CI with rust-agent

src/agent/VERSION

@@ -1 +0,0 @@
-../../VERSION

src/agent/VERSION

@@ -0,0 +1 @@
+2.0.0

src/agent/kata-agent.service.in

@@ -20,3 +20,5 @@ LimitNOFILE=infinity
 # the runtime handles shutting down the VM.
 ExecStop=/bin/sync ; /usr/bin/systemctl --force poweroff
 FailureAction=poweroff
+# Discourage OOM-killer from touching the agent
+OOMScoreAdjust=-997

src/agent/oci/src/serialize.rs

@@ -4,7 +4,6 @@
 //
 
 use serde::{Deserialize, Serialize};
-use serde_json;
 
 use std::error;
 use std::fmt::{Display, Formatter, Result as FmtResult};

src/agent/protocols/hack/update-generated-proto.sh

@@ -47,7 +47,7 @@ show_usage() {
 }
 
 generate_go_sources() {
-    local cmd="protoc -I$GOPATH/src/github.com/kata-containers/agent/vendor/github.com/gogo/protobuf:$GOPATH/src/github.com/kata-containers/agent/vendor:$GOPATH/src/github.com/gogo/protobuf:$GOPATH/src/github.com/gogo/googleapis:$GOPATH/src:$GOPATH/src/github.com/kata-containers/kata-containers/src/agent/protocols/protos \
+    local cmd="protoc -I$GOPATH/src:$GOPATH/src/github.com/kata-containers/kata-containers/src/agent/protocols/protos \
 --gogottrpc_out=plugins=ttrpc+fieldpath,\
 import_path=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc,\
 \
@@ -80,12 +80,6 @@ fi;
 which protoc
 [ $? -eq 0 ] || die "Please install protoc from github.com/protocolbuffers/protobuf"
 
-which protoc-gen-rust
-[ $? -eq 0 ] || die "Please install protobuf-codegen from github.com/pingcap/grpc-rs"
-
-which ttrpc_rust_plugin
-[ $? -eq 0 ] || die "Please install ttrpc_rust_plugin from https://github.com/containerd/ttrpc-rust"
-
 which protoc-gen-gogottrpc
 [ $? -eq 0 ] || die "Please install protoc-gen-gogottrpc from https://github.com/containerd/ttrpc"
 

src/agent/protocols/protos/oci.proto

@@ -12,7 +12,6 @@ option go_package = "github.com/kata-containers/kata-containers/src/runtime/virt
 package grpc;
 
 import "gogo/protobuf/gogoproto/gogo.proto";
-import "google/protobuf/wrappers.proto";
 
 option (gogoproto.equal_all) = true;
 option (gogoproto.populate_all) = true;

src/agent/protocols/src/lib.rs

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 #![allow(bare_trait_objects)]
+#![allow(clippy::redundant_field_names)]
 
 pub mod agent;
 pub mod agent_ttrpc;
@@ -11,11 +12,3 @@ pub mod health;
 pub mod health_ttrpc;
 pub mod oci;
 pub mod types;
-
-#[cfg(test)]
-mod tests {
-    #[test]
-    fn it_works() {
-        assert_eq!(2 + 2, 4);
-    }
-}

src/agent/rustjail/Cargo.toml

@@ -24,8 +24,9 @@ regex = "1.1"
 path-absolutize = "1.2.0"
 dirs = "3.0.1"
 anyhow = "1.0.32"
-cgroups = { git = "https://github.com/kata-containers/cgroups-rs", branch = "stable-0.1.1"}
+cgroups = { package = "cgroups-rs", version = "0.2.0" }
 tempfile = "3.1.0"
+epoll = "4.3.1"
 
 [dev-dependencies]
 serial_test = "0.5.0"

src/agent/rustjail/src/capabilities.rs

@@ -6,8 +6,6 @@
 // looks like we can use caps to manipulate capabilities
 // conveniently, use caps to do it directly.. maybe
 
-use lazy_static;
-
 use crate::log_child;
 use crate::sync::write_count;
 use anyhow::{anyhow, Result};

src/agent/rustjail/src/cgroups/fs/mod.rs

@@ -21,7 +21,6 @@ use cgroups::{
 use crate::cgroups::Manager as CgroupManager;
 use crate::container::DEFAULT_DEVICES;
 use anyhow::{anyhow, Context, Result};
-use lazy_static;
 use libc::{self, pid_t};
 use nix::errno::Errno;
 use oci::{
@@ -45,28 +44,6 @@ macro_rules! sl {
     };
 }
 
-pub fn load_or_create<'a>(h: Box<&'a dyn cgroups::Hierarchy>, path: &str) -> Cgroup<'a> {
-    let valid_path = path.trim_start_matches("/").to_string();
-    let cg = load(h.clone(), &valid_path);
-    if cg.is_none() {
-        info!(sl!(), "create new cgroup: {}", &valid_path);
-        cgroups::Cgroup::new(h, valid_path.as_str())
-    } else {
-        cg.unwrap()
-    }
-}
-
-pub fn load<'a>(h: Box<&'a dyn cgroups::Hierarchy>, path: &str) -> Option<Cgroup<'a>> {
-    let valid_path = path.trim_start_matches("/").to_string();
-    let cg = cgroups::Cgroup::load(h, valid_path.as_str());
-    let cpu_controller: &CpuController = cg.controller_of().unwrap();
-    if cpu_controller.exists() {
-        Some(cg)
-    } else {
-        None
-    }
-}
-
 macro_rules! get_controller_or_return_singular_none {
     ($cg:ident) => {
         match $cg.controller_of() {
@@ -80,8 +57,10 @@ macro_rules! get_controller_or_return_singular_none {
 pub struct Manager {
     pub paths: HashMap<String, String>,
     pub mounts: HashMap<String, String>,
-    // pub rels: HashMap<String, String>,
     pub cpath: String,
+    #[serde(skip)]
+    cgroup: cgroups::Cgroup,
+    relative_paths: HashMap<String, String>,
 }
 
 // set_resource is used to set reources by cgroup controller.
@@ -96,17 +75,11 @@ macro_rules! set_resource {
 
 impl CgroupManager for Manager {
     fn apply(&self, pid: pid_t) -> Result<()> {
-        let h = cgroups::hierarchies::auto();
-        let h = Box::new(&*h);
-        let cg = load_or_create(h, &self.cpath);
-        cg.add_task(CgroupPid::from(pid as u64))?;
+        self.cgroup.add_task(CgroupPid::from(pid as u64))?;
         Ok(())
     }
 
     fn set(&self, r: &LinuxResources, update: bool) -> Result<()> {
-        let h = cgroups::hierarchies::auto();
-        let h = Box::new(&*h);
-        let cg = load_or_create(h, &self.cpath);
         info!(
             sl!(),
             "cgroup manager set resources for container. Resources input {:?}", r
@@ -116,53 +89,49 @@ impl CgroupManager for Manager {
 
         // set cpuset and cpu reources
         if let Some(cpu) = &r.cpu {
-            set_cpu_resources(&cg, cpu)?;
+            set_cpu_resources(&self.cgroup, cpu)?;
         }
 
         // set memory resources
         if let Some(memory) = &r.memory {
-            set_memory_resources(&cg, memory, update)?;
+            set_memory_resources(&self.cgroup, memory, update)?;
         }
 
         // set pids resources
         if let Some(pids_resources) = &r.pids {
-            set_pids_resources(&cg, pids_resources)?;
+            set_pids_resources(&self.cgroup, pids_resources)?;
         }
 
         // set block_io resources
         if let Some(blkio) = &r.block_io {
-            set_block_io_resources(&cg, blkio, res)?;
+            set_block_io_resources(&self.cgroup, blkio, res)?;
         }
 
         // set hugepages resources
-        if r.hugepage_limits.len() > 0 {
-            set_hugepages_resources(&cg, &r.hugepage_limits, res)?;
+        if !r.hugepage_limits.is_empty() {
+            set_hugepages_resources(&self.cgroup, &r.hugepage_limits, res)?;
         }
 
         // set network resources
         if let Some(network) = &r.network {
-            set_network_resources(&cg, network, res)?;
+            set_network_resources(&self.cgroup, network, res)?;
         }
 
         // set devices resources
-        set_devices_resources(&cg, &r.devices, res)?;
+        set_devices_resources(&self.cgroup, &r.devices, res)?;
         info!(sl!(), "resources after processed {:?}", res);
 
         // apply resources
-        cg.apply(res)?;
+        self.cgroup.apply(res)?;
 
         Ok(())
     }
 
     fn get_stats(&self) -> Result<CgroupStats> {
-        let h = cgroups::hierarchies::auto();
-        let h = Box::new(&*h);
-        let cg = load_or_create(h, &self.cpath);
-
         // CpuStats
-        let cpu_usage = get_cpuacct_stats(&cg);
+        let cpu_usage = get_cpuacct_stats(&self.cgroup);
 
-        let throttling_data = get_cpu_stats(&cg);
+        let throttling_data = get_cpu_stats(&self.cgroup);
 
         let cpu_stats = SingularPtrField::some(CpuStats {
             cpu_usage,
@@ -172,17 +141,17 @@ impl CgroupManager for Manager {
         });
 
         // Memorystats
-        let memory_stats = get_memory_stats(&cg);
+        let memory_stats = get_memory_stats(&self.cgroup);
 
         // PidsStats
-        let pids_stats = get_pids_stats(&cg);
+        let pids_stats = get_pids_stats(&self.cgroup);
 
         // BlkioStats
         // note that virtiofs has no blkio stats
-        let blkio_stats = get_blkio_stats(&cg);
+        let blkio_stats = get_blkio_stats(&self.cgroup);
 
         // HugetlbStats
-        let hugetlb_stats = get_hugetlb_stats(&cg);
+        let hugetlb_stats = get_hugetlb_stats(&self.cgroup);
 
         Ok(CgroupStats {
             cpu_stats,
@@ -196,10 +165,7 @@ impl CgroupManager for Manager {
     }
 
     fn freeze(&self, state: FreezerState) -> Result<()> {
-        let h = cgroups::hierarchies::auto();
-        let h = Box::new(&*h);
-        let cg = load_or_create(h, &self.cpath);
-        let freezer_controller: &FreezerController = cg.controller_of().unwrap();
+        let freezer_controller: &FreezerController = self.cgroup.controller_of().unwrap();
         match state {
             FreezerState::Thawed => {
                 freezer_controller.thaw()?;
@@ -216,20 +182,12 @@ impl CgroupManager for Manager {
     }
 
     fn destroy(&mut self) -> Result<()> {
-        let h = cgroups::hierarchies::auto();
-        let h = Box::new(&*h);
-        let cg = load(h, &self.cpath);
-        if cg.is_some() {
-            cg.unwrap().delete();
-        }
+        let _ = self.cgroup.delete();
         Ok(())
     }
 
     fn get_pids(&self) -> Result<Vec<pid_t>> {
-        let h = cgroups::hierarchies::auto();
-        let h = Box::new(&*h);
-        let cg = load_or_create(h, &self.cpath);
-        let mem_controller: &MemController = cg.controller_of().unwrap();
+        let mem_controller: &MemController = self.cgroup.controller_of().unwrap();
         let pids = mem_controller.tasks();
         let result = pids.iter().map(|x| x.pid as i32).collect::<Vec<i32>>();
 
@@ -248,7 +206,7 @@ fn set_network_resources(
     // description can be found at https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v1/net_cls.html
     let class_id = network.class_id.unwrap_or(0) as u64;
     if class_id != 0 {
-        res.network.class_id = class_id;
+        res.network.class_id = Some(class_id);
     }
 
     // set network priorities
@@ -261,14 +219,13 @@ fn set_network_resources(
         });
     }
 
-    res.network.update_values = true;
     res.network.priorities = priorities;
     Ok(())
 }
 
 fn set_devices_resources(
     _cg: &cgroups::Cgroup,
-    device_resources: &Vec<LinuxDeviceCgroup>,
+    device_resources: &[LinuxDeviceCgroup],
     res: &mut cgroups::Resources,
 ) -> Result<()> {
     info!(sl!(), "cgroup manager set devices");
@@ -292,7 +249,6 @@ fn set_devices_resources(
         }
     }
 
-    res.devices.update_values = true;
     res.devices.devices = devices;
 
     Ok(())
@@ -300,11 +256,10 @@ fn set_devices_resources(
 
 fn set_hugepages_resources(
     _cg: &cgroups::Cgroup,
-    hugepage_limits: &Vec<LinuxHugepageLimit>,
+    hugepage_limits: &[LinuxHugepageLimit],
     res: &mut cgroups::Resources,
 ) -> Result<()> {
     info!(sl!(), "cgroup manager set hugepage");
-    res.hugepages.update_values = true;
     let mut limits = vec![];
 
     for l in hugepage_limits.iter() {
@@ -325,7 +280,6 @@ fn set_block_io_resources(
     res: &mut cgroups::Resources,
 ) -> Result<()> {
     info!(sl!(), "cgroup manager set block io");
-    res.blkio.update_values = true;
 
     if cg.v2() {
         res.blkio.weight = convert_blk_io_to_v2_value(blkio.weight);
@@ -374,7 +328,9 @@ fn set_cpu_resources(cg: &cgroups::Cgroup, cpu: &LinuxCPU) -> Result<()> {
     let cpuset_controller: &CpuSetController = cg.controller_of().unwrap();
 
     if !cpu.cpus.is_empty() {
-        cpuset_controller.set_cpus(&cpu.cpus)?;
+        if let Err(e) = cpuset_controller.set_cpus(&cpu.cpus) {
+            warn!(sl!(), "write cpuset failed: {:?}", e);
+        }
     }
 
     if !cpu.mems.is_empty() {
@@ -462,7 +418,7 @@ fn set_pids_resources(cg: &cgroups::Cgroup, pids: &LinuxPids) -> Result<()> {
 }
 
 fn build_blk_io_device_throttle_resource(
-    input: &Vec<oci::LinuxThrottleDevice>,
+    input: &[oci::LinuxThrottleDevice],
 ) -> Vec<BlkIoDeviceThrottleResource> {
     let mut blk_io_device_throttle_resources = vec![];
     for d in input.iter() {
@@ -690,7 +646,7 @@ fn get_memory_stats(cg: &cgroups::Cgroup) -> SingularPtrField<MemoryStats> {
 
     // use_hierarchy
     let value = memory.use_hierarchy;
-    let use_hierarchy = if value == 1 { true } else { false };
+    let use_hierarchy = value == 1;
 
     // gte memory datas
     let usage = SingularPtrField::some(MemoryData {
@@ -744,13 +700,12 @@ fn get_pids_stats(cg: &cgroups::Cgroup) -> SingularPtrField<PidsStats> {
     let current = pid_controller.get_pid_current().unwrap_or(0);
     let max = pid_controller.get_pid_max();
 
-    let limit = if max.is_err() {
-        0
-    } else {
-        match max.unwrap() {
+    let limit = match max {
+        Err(_) => 0,
+        Ok(max) => match max {
             MaxValue::Value(v) => v,
             MaxValue::Max => 0,
-        }
+        },
     } as u64;
 
     SingularPtrField::some(PidsStats {
@@ -793,9 +748,9 @@ https://github.com/opencontainers/runc/blob/a5847db387ae28c0ca4ebe4beee1a76900c8
     Total 0
 */
 
-fn get_blkio_stat_blkiodata(blkiodata: &Vec<BlkIoData>) -> RepeatedField<BlkioStatsEntry> {
+fn get_blkio_stat_blkiodata(blkiodata: &[BlkIoData]) -> RepeatedField<BlkioStatsEntry> {
     let mut m = RepeatedField::new();
-    if blkiodata.len() == 0 {
+    if blkiodata.is_empty() {
         return m;
     }
 
@@ -815,10 +770,10 @@ fn get_blkio_stat_blkiodata(blkiodata: &Vec<BlkIoData>) -> RepeatedField<BlkioSt
     m
 }
 
-fn get_blkio_stat_ioservice(services: &Vec<IoService>) -> RepeatedField<BlkioStatsEntry> {
+fn get_blkio_stat_ioservice(services: &[IoService]) -> RepeatedField<BlkioStatsEntry> {
     let mut m = RepeatedField::new();
 
-    if services.len() == 0 {
+    if services.is_empty() {
         return m;
     }
 
@@ -839,7 +794,7 @@ fn build_blkio_stats_entry(major: i16, minor: i16, op: &str, value: u64) -> Blki
         major: major as u64,
         minor: minor as u64,
         op: op.to_string(),
-        value: value,
+        value,
         unknown_fields: UnknownFields::default(),
         cached_size: CachedSize::default(),
     }
@@ -880,7 +835,7 @@ fn get_blkio_stats(cg: &cgroups::Cgroup) -> SingularPtrField<BlkioStats> {
     let mut m = BlkioStats::new();
     let io_serviced_recursive = blkio.io_serviced_recursive;
 
-    if io_serviced_recursive.len() == 0 {
+    if io_serviced_recursive.is_empty() {
         // fall back to generic stats
         // blkio.throttle.io_service_bytes,
         // maybe io_service_bytes_recursive?
@@ -935,8 +890,8 @@ fn get_hugetlb_stats(cg: &cgroups::Cgroup) -> HashMap<String, HugetlbStats> {
     h
 }
 
-pub const PATHS: &'static str = "/proc/self/cgroup";
-pub const MOUNTS: &'static str = "/proc/self/mountinfo";
+pub const PATHS: &str = "/proc/self/cgroup";
+pub const MOUNTS: &str = "/proc/self/mountinfo";
 
 pub fn get_paths() -> Result<HashMap<String, String>> {
     let mut m = HashMap::new();
@@ -991,9 +946,19 @@ pub fn get_mounts() -> Result<HashMap<String, String>> {
     Ok(m)
 }
 
+fn new_cgroup(
+    h: Box<dyn cgroups::Hierarchy>,
+    path: &str,
+    relative_paths: HashMap<String, String>,
+) -> Cgroup {
+    let valid_path = path.trim_start_matches('/').to_string();
+    cgroups::Cgroup::new_with_relative_paths(h, valid_path.as_str(), relative_paths)
+}
+
 impl Manager {
     pub fn new(cpath: &str) -> Result<Self> {
         let mut m = HashMap::new();
+        let mut relative_paths = HashMap::new();
 
         let paths = get_paths()?;
         let mounts = get_mounts()?;
@@ -1012,6 +977,7 @@ impl Manager {
             };
 
             m.insert(key.to_string(), p);
+            relative_paths.insert(key.to_string(), value.to_string());
         }
 
         Ok(Self {
@@ -1019,29 +985,27 @@ impl Manager {
             mounts,
             // rels: paths,
             cpath: cpath.to_string(),
+            cgroup: new_cgroup(cgroups::hierarchies::auto(), cpath, relative_paths.clone()),
+            relative_paths,
         })
     }
 
-    pub fn update_cpuset_path(&self, cpuset_cpus: &str) -> Result<()> {
-        if cpuset_cpus == "" {
+    pub fn update_cpuset_path(&self, guest_cpuset: &str, container_cpuset: &str) -> Result<()> {
+        if guest_cpuset == "" {
             return Ok(());
         }
-        info!(sl!(), "update_cpuset_path to: {}", cpuset_cpus);
+        info!(sl!(), "update_cpuset_path to: {}", guest_cpuset);
 
         let h = cgroups::hierarchies::auto();
-        let h = Box::new(&*h);
-        let root_cg = load_or_create(h, "");
+        let root_cg = h.root_control_group();
 
         let root_cpuset_controller: &CpuSetController = root_cg.controller_of().unwrap();
         let path = root_cpuset_controller.path();
         let root_path = Path::new(path);
         info!(sl!(), "root cpuset path: {:?}", &path);
 
-        let h = cgroups::hierarchies::auto();
-        let h = Box::new(&*h);
-        let cg = load_or_create(h, &self.cpath);
-        let cpuset_controller: &CpuSetController = cg.controller_of().unwrap();
-        let path = cpuset_controller.path();
+        let container_cpuset_controller: &CpuSetController = self.cgroup.controller_of().unwrap();
+        let path = container_cpuset_controller.path();
         let container_path = Path::new(path);
         info!(sl!(), "container cpuset path: {:?}", &path);
 
@@ -1050,30 +1014,40 @@ impl Manager {
             if ancestor == root_path {
                 break;
             }
-            if ancestor != container_path {
-                paths.push(ancestor);
-            }
+            paths.push(ancestor);
         }
-        info!(sl!(), "paths to update cpuset: {:?}", &paths);
+        info!(sl!(), "parent paths to update cpuset: {:?}", &paths);
 
         let mut i = paths.len();
         loop {
             if i == 0 {
                 break;
             }
-            i = i - 1;
-            let h = cgroups::hierarchies::auto();
-            let h = Box::new(&*h);
+            i -= 1;
 
             // remove cgroup root from path
             let r_path = &paths[i]
                 .to_str()
                 .unwrap()
                 .trim_start_matches(root_path.to_str().unwrap());
-            info!(sl!(), "updating cpuset for path {:?}", &r_path);
-            let cg = load_or_create(h, &r_path);
+            info!(sl!(), "updating cpuset for parent path {:?}", &r_path);
+            let cg = new_cgroup(
+                cgroups::hierarchies::auto(),
+                &r_path,
+                self.relative_paths.clone(),
+            );
             let cpuset_controller: &CpuSetController = cg.controller_of().unwrap();
-            cpuset_controller.set_cpus(cpuset_cpus)?;
+            cpuset_controller.set_cpus(guest_cpuset)?;
+        }
+
+        if !container_cpuset.is_empty() {
+            info!(
+                sl!(),
+                "updating cpuset for container path: {:?} cpuset: {}",
+                &container_path,
+                container_cpuset
+            );
+            container_cpuset_controller.set_cpus(container_cpuset)?;
         }
 
         Ok(())

src/agent/rustjail/src/cgroups/mock.rs

@@ -0,0 +1,74 @@
+// Copyright (c) 2020 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use protobuf::{CachedSize, SingularPtrField, UnknownFields};
+
+use crate::cgroups::Manager as CgroupManager;
+use crate::protocols::agent::{BlkioStats, CgroupStats, CpuStats, MemoryStats, PidsStats};
+use anyhow::Result;
+use cgroups::freezer::FreezerState;
+use libc::{self, pid_t};
+use oci::LinuxResources;
+use std::collections::HashMap;
+use std::string::String;
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct Manager {
+    pub paths: HashMap<String, String>,
+    pub mounts: HashMap<String, String>,
+    pub cpath: String,
+}
+
+impl CgroupManager for Manager {
+    fn apply(&self, _: pid_t) -> Result<()> {
+        Ok(())
+    }
+
+    fn set(&self, _: &LinuxResources, _: bool) -> Result<()> {
+        Ok(())
+    }
+
+    fn get_stats(&self) -> Result<CgroupStats> {
+        Ok(CgroupStats {
+            cpu_stats: SingularPtrField::some(CpuStats::default()),
+            memory_stats: SingularPtrField::some(MemoryStats::new()),
+            pids_stats: SingularPtrField::some(PidsStats::new()),
+            blkio_stats: SingularPtrField::some(BlkioStats::new()),
+            hugetlb_stats: HashMap::new(),
+            unknown_fields: UnknownFields::default(),
+            cached_size: CachedSize::default(),
+        })
+    }
+
+    fn freeze(&self, _: FreezerState) -> Result<()> {
+        Ok(())
+    }
+
+    fn destroy(&mut self) -> Result<()> {
+        Ok(())
+    }
+
+    fn get_pids(&self) -> Result<Vec<pid_t>> {
+        Ok(Vec::new())
+    }
+}
+
+impl Manager {
+    pub fn new(cpath: &str) -> Result<Self> {
+        Ok(Self {
+            paths: HashMap::new(),
+            mounts: HashMap::new(),
+            cpath: cpath.to_string(),
+        })
+    }
+
+    pub fn update_cpuset_path(&self, _: &str, _: &str) -> Result<()> {
+        Ok(())
+    }
+
+    pub fn get_cg_path(&self, _: &str) -> Option<String> {
+        Some("".to_string())
+    }
+}

src/agent/rustjail/src/cgroups/mod.rs

@@ -10,6 +10,7 @@ use protocols::agent::CgroupStats;
 use cgroups::freezer::FreezerState;
 
 pub mod fs;
+pub mod mock;
 pub mod notifier;
 pub mod systemd;
 

src/agent/rustjail/src/cgroups/notifier.rs

@@ -41,7 +41,7 @@ fn get_value_from_cgroup(path: &PathBuf, key: &str) -> Result<i64> {
     );
 
     for line in content.lines() {
-        let arr: Vec<&str> = line.split(" ").collect();
+        let arr: Vec<&str> = line.split(' ').collect();
         if arr.len() == 2 && arr[0] == key {
             let r = arr[1].parse::<i64>()?;
             return Ok(r);

src/agent/rustjail/src/container.rs

@@ -4,12 +4,9 @@
 //
 
 use anyhow::{anyhow, Context, Result};
-use dirs;
-use lazy_static;
 use libc::pid_t;
 use oci::{Hook, Linux, LinuxNamespace, LinuxResources, POSIXRlimit, Spec};
 use oci::{LinuxDevice, LinuxIDMapping};
-use serde_json;
 use std::clone::Clone;
 use std::ffi::{CStr, CString};
 use std::fmt;
@@ -23,7 +20,10 @@ use std::time::SystemTime;
 use cgroups::freezer::FreezerState;
 
 use crate::capabilities::{self, CAPSMAP};
+#[cfg(not(test))]
 use crate::cgroups::fs::Manager as FsManager;
+#[cfg(test)]
+use crate::cgroups::mock::Manager as FsManager;
 use crate::cgroups::Manager;
 use crate::log_child;
 use crate::process::Process;
@@ -41,9 +41,10 @@ use nix::pty;
 use nix::sched::{self, CloneFlags};
 use nix::sys::signal::{self, Signal};
 use nix::sys::stat::{self, Mode};
-use nix::unistd::{self, ForkResult, Gid, Pid, Uid};
+use nix::unistd::{self, fork, ForkResult, Gid, Pid, Uid};
+use std::os::unix::fs::MetadataExt;
+use std::os::unix::io::AsRawFd;
 
-use libc;
 use protobuf::SingularPtrField;
 
 use oci::State as OCIState;
@@ -54,9 +55,9 @@ use std::os::unix::io::FromRawFd;
 
 use slog::{info, o, Logger};
 
-const STATE_FILENAME: &'static str = "state.json";
-const EXEC_FIFO_FILENAME: &'static str = "exec.fifo";
-const VER_MARKER: &'static str = "1.2.5";
+const STATE_FILENAME: &str = "state.json";
+const EXEC_FIFO_FILENAME: &str = "exec.fifo";
+const VER_MARKER: &str = "1.2.5";
 const PID_NS_PATH: &str = "/proc/self/ns/pid";
 
 const INIT: &str = "INIT";
@@ -66,6 +67,7 @@ const CWFD_FD: &str = "CWFD_FD";
 const CLOG_FD: &str = "CLOG_FD";
 const FIFO_FD: &str = "FIFO_FD";
 const HOME_ENV_KEY: &str = "HOME";
+const PIDNS_FD: &str = "PIDNS_FD";
 
 #[derive(PartialEq, Clone, Copy)]
 pub enum Status {
@@ -321,10 +323,13 @@ pub fn init_child() {
     let cwfd = std::env::var(CWFD_FD).unwrap().parse::<i32>().unwrap();
     let cfd_log = std::env::var(CLOG_FD).unwrap().parse::<i32>().unwrap();
 
-    let _ = do_init_child(cwfd).map_err(|e| {
-        log_child!(cfd_log, "child exit: {:?}", e);
-        let _ = write_sync(cwfd, SYNC_FAILED, format!("{:?}", e).as_str());
-    });
+    match do_init_child(cwfd) {
+        Ok(_) => log_child!(cfd_log, "temporary parent process exit successfully"),
+        Err(e) => {
+            log_child!(cfd_log, "temporary parent process exit:child exit: {:?}", e);
+            let _ = write_sync(cwfd, SYNC_FAILED, format!("{:?}", e).as_str());
+        }
+    }
 }
 
 fn do_init_child(cwfd: RawFd) -> Result<()> {
@@ -339,6 +344,38 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     let crfd = std::env::var(CRFD_FD)?.parse::<i32>().unwrap();
     let cfd_log = std::env::var(CLOG_FD)?.parse::<i32>().unwrap();
 
+    // get the pidns fd from parent, if parent had passed the pidns fd,
+    // then get it and join in this pidns; otherwise, create a new pidns
+    // by unshare from the parent pidns.
+    match std::env::var(PIDNS_FD) {
+        Ok(fd) => {
+            let pidns_fd = fd.parse::<i32>().context("get parent pidns fd")?;
+            sched::setns(pidns_fd, CloneFlags::CLONE_NEWPID).context("failed to join pidns")?;
+            let _ = unistd::close(pidns_fd);
+        }
+        Err(_e) => sched::unshare(CloneFlags::CLONE_NEWPID)?,
+    }
+
+    match fork() {
+        Ok(ForkResult::Parent { child, .. }) => {
+            log_child!(
+                cfd_log,
+                "Continuing execution in temporary process, new child has pid: {:?}",
+                child
+            );
+            let _ = write_sync(cwfd, SYNC_DATA, format!("{}", pid_t::from(child)).as_str());
+            // parent return
+            return Ok(());
+        }
+        Ok(ForkResult::Child) => (),
+        Err(e) => {
+            return Err(anyhow!(format!(
+                "failed to fork temporary process: {:?}",
+                e
+            )));
+        }
+    }
+
     log_child!(cfd_log, "child process start run");
     let buf = read_sync(crfd)?;
     let spec_str = std::str::from_utf8(&buf)?;
@@ -549,9 +586,15 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     let uid = Uid::from_raw(guser.uid);
     let gid = Gid::from_raw(guser.gid);
 
+    // only change stdio devices owner when user
+    // isn't root.
+    if guser.uid != 0 {
+        set_stdio_permissions(guser.uid)?;
+    }
+
     setid(uid, gid)?;
 
-    if guser.additional_gids.len() > 0 {
+    if !guser.additional_gids.is_empty() {
         setgroups(guser.additional_gids.as_slice()).map_err(|e| {
             let _ = write_sync(
                 cwfd,
@@ -595,7 +638,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
 
     // setup the envs
     for e in env.iter() {
-        let v: Vec<&str> = e.splitn(2, "=").collect();
+        let v: Vec<&str> = e.splitn(2, '=').collect();
         if v.len() != 2 {
             continue;
         }
@@ -643,6 +686,43 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     do_exec(&args);
 }
 
+// set_stdio_permissions fixes the permissions of PID 1's STDIO
+// within the container to the specified user.
+// The ownership needs to match because it is created outside of
+// the container and needs to be localized.
+fn set_stdio_permissions(uid: libc::uid_t) -> Result<()> {
+    let meta = fs::metadata("/dev/null")?;
+    let fds = [
+        std::io::stdin().as_raw_fd(),
+        std::io::stdout().as_raw_fd(),
+        std::io::stderr().as_raw_fd(),
+    ];
+
+    for fd in &fds {
+        let stat = stat::fstat(*fd)?;
+        // Skip chown of /dev/null if it was used as one of the STDIO fds.
+        if stat.st_rdev == meta.rdev() {
+            continue;
+        }
+
+        // According to the POSIX specification, -1 is used to indicate that owner and group
+        // are not to be changed.  Since uid_t and gid_t are unsigned types, we have to wrap
+        // around to get -1.
+        let gid = (0 as libc::gid_t).wrapping_sub(1);
+
+        // We only change the uid owner (as it is possible for the mount to
+        // prefer a different gid, and there's no reason for us to change it).
+        // The reason why we don't just leave the default uid=X mount setup is
+        // that users expect to be able to actually use their console. Without
+        // this code, you couldn't effectively run as a non-root user inside a
+        // container and also have a console set up.
+        let res = unsafe { libc::fchown(*fd, uid, gid) };
+        Errno::result(res).map_err(|e| anyhow!(e).context("set stdio permissions failed"))?;
+    }
+
+    Ok(())
+}
+
 impl BaseContainer for LinuxContainer {
     fn id(&self) -> String {
         self.id.clone()
@@ -653,11 +733,15 @@ impl BaseContainer for LinuxContainer {
     }
 
     fn state(&self) -> Result<State> {
-        Err(anyhow!("not suppoerted"))
+        Err(anyhow!("not supported"))
     }
 
     fn oci_state(&self) -> Result<OCIState> {
-        let oci = self.config.spec.as_ref().unwrap();
+        let oci = match self.config.spec.as_ref() {
+            Some(s) => s,
+            None => return Err(anyhow!("Unable to get OCI state: spec not found")),
+        };
+
         let status = self.status();
         let pid = if status != Status::STOPPED {
             self.init_process_pid
@@ -665,9 +749,17 @@ impl BaseContainer for LinuxContainer {
             0
         };
 
-        let root = oci.root.as_ref().unwrap().path.as_str();
+        let root = match oci.root.as_ref() {
+            Some(s) => s.path.as_str(),
+            None => return Err(anyhow!("Unable to get root path: oci.root is none")),
+        };
+
         let path = fs::canonicalize(root)?;
-        let bundle = path.parent().unwrap().to_str().unwrap().to_string();
+        let bundle = match path.parent() {
+            Some(s) => s.to_str().unwrap().to_string(),
+            None => return Err(anyhow!("could not get root parent: root path {:?}", path)),
+        };
+
         Ok(OCIState {
             version: oci.version.clone(),
             id: self.id(),
@@ -731,7 +823,7 @@ impl BaseContainer for LinuxContainer {
         info!(logger, "enter container.start!");
         let mut fifofd: RawFd = -1;
         if p.init {
-            if let Ok(_) = stat::stat(fifo_file.as_str()) {
+            if stat::stat(fifo_file.as_str()).is_ok() {
                 return Err(anyhow!("exec fifo exists"));
             }
             unistd::mkfifo(fifo_file.as_str(), Mode::from_bits(0o622).unwrap())?;
@@ -762,30 +854,8 @@ impl BaseContainer for LinuxContainer {
             .map_err(|e| warn!(logger, "fcntl pfd log FD_CLOEXEC {:?}", e));
 
         let child_logger = logger.new(o!("action" => "child process log"));
-        let log_handler = thread::spawn(move || {
-            let log_file = unsafe { std::fs::File::from_raw_fd(pfd_log) };
-            let mut reader = BufReader::new(log_file);
+        let log_handler = setup_child_logger(pfd_log, child_logger)?;
 
-            loop {
-                let mut line = String::new();
-                match reader.read_line(&mut line) {
-                    Err(e) => {
-                        info!(child_logger, "read child process log error: {:?}", e);
-                        break;
-                    }
-                    Ok(count) => {
-                        if count == 0 {
-                            info!(child_logger, "read child process log end",);
-                            break;
-                        }
-
-                        info!(child_logger, "{}", line);
-                    }
-                }
-            }
-        });
-
-        info!(logger, "exec fifo opened!");
         let (prfd, cwfd) = unistd::pipe().context("failed to create pipe")?;
         let (crfd, pwfd) = unistd::pipe().context("failed to create pipe")?;
 
@@ -824,32 +894,11 @@ impl BaseContainer for LinuxContainer {
             child_stderr = unsafe { std::process::Stdio::from_raw_fd(stderr) };
         }
 
-        let old_pid_ns =
-            fcntl::open(PID_NS_PATH, OFlag::O_CLOEXEC, Mode::empty()).map_err(|e| {
-                error!(
-                    logger,
-                    "cannot open pid ns path: {} with error: {:?}", PID_NS_PATH, e
-                );
-                e
-            })?;
-
-        //restore the parent's process's pid namespace.
-        defer!({
-            let _ = sched::setns(old_pid_ns, CloneFlags::CLONE_NEWPID)
-                .map_err(|e| warn!(logger, "settns CLONE_NEWPID {:?}", e));
-            let _ = unistd::close(old_pid_ns)
-                .map_err(|e| warn!(logger, "close old pid namespace {:?}", e));
-        });
-
         let pidns = get_pid_namespace(&self.logger, linux)?;
 
-        if pidns.is_some() {
-            sched::setns(pidns.unwrap(), CloneFlags::CLONE_NEWPID)
-                .context("failed to join pidns")?;
-            unistd::close(pidns.unwrap())?;
-        } else {
-            sched::unshare(CloneFlags::CLONE_NEWPID)?;
-        }
+        defer!(if let Some(pid) = pidns {
+            let _ = unistd::close(pid);
+        });
 
         let exec_path = std::env::current_exe()?;
         let mut child = std::process::Command::new(exec_path);
@@ -868,13 +917,31 @@ impl BaseContainer for LinuxContainer {
             child = child.env(FIFO_FD, format!("{}", fifofd));
         }
 
+        if pidns.is_some() {
+            child = child.env(PIDNS_FD, format!("{}", pidns.unwrap()));
+        }
+
         let child = child.spawn()?;
 
         unistd::close(crfd)?;
         unistd::close(cwfd)?;
         unistd::close(cfd_log)?;
 
-        p.pid = child.id() as i32;
+        // get container process's pid
+        let pid_buf = read_sync(prfd)?;
+        let pid_str = std::str::from_utf8(&pid_buf).context("get pid string")?;
+        let pid = match pid_str.parse::<i32>() {
+            Ok(i) => i,
+            Err(e) => {
+                return Err(anyhow!(format!(
+                    "failed to get container process's pid: {:?}",
+                    e
+                )));
+            }
+        };
+
+        p.pid = pid;
+
         if p.init {
             self.init_process_pid = p.pid;
         }
@@ -931,7 +998,7 @@ impl BaseContainer for LinuxContainer {
             .join()
             .map_err(|e| warn!(logger, "joining log handler {:?}", e));
         info!(logger, "create process completed");
-        return Ok(());
+        Ok(())
     }
 
     fn run(&mut self, p: Process) -> Result<()> {
@@ -963,7 +1030,7 @@ impl BaseContainer for LinuxContainer {
         }
 
         self.status.transition(Status::STOPPED);
-        nix::mount::umount2(
+        mount::umount2(
             spec.root.as_ref().unwrap().path.as_str(),
             MntFlags::MNT_DETACH,
         )?;
@@ -1073,7 +1140,7 @@ fn get_pid_namespace(logger: &Logger, linux: &Linux) -> Result<Option<RawFd>> {
             }
 
             let fd =
-                fcntl::open(ns.path.as_str(), OFlag::O_CLOEXEC, Mode::empty()).map_err(|e| {
+                fcntl::open(ns.path.as_str(), OFlag::O_RDONLY, Mode::empty()).map_err(|e| {
                     error!(
                         logger,
                         "cannot open type: {} path: {}",
@@ -1110,6 +1177,34 @@ fn get_namespaces(linux: &Linux) -> Vec<LinuxNamespace> {
         .collect()
 }
 
+pub fn setup_child_logger(fd: RawFd, child_logger: Logger) -> Result<std::thread::JoinHandle<()>> {
+    let builder = thread::Builder::new();
+    builder
+        .spawn(move || {
+            let log_file = unsafe { std::fs::File::from_raw_fd(fd) };
+            let mut reader = BufReader::new(log_file);
+
+            loop {
+                let mut line = String::new();
+                match reader.read_line(&mut line) {
+                    Err(e) => {
+                        info!(child_logger, "read child process log error: {:?}", e);
+                        break;
+                    }
+                    Ok(count) => {
+                        if count == 0 {
+                            info!(child_logger, "read child process log end",);
+                            break;
+                        }
+
+                        info!(child_logger, "{}", line);
+                    }
+                }
+            }
+        })
+        .map_err(|e| anyhow!(e).context("failed to create thread"))
+}
+
 fn join_namespaces(
     logger: &Logger,
     spec: &Spec,
@@ -1164,11 +1259,9 @@ fn join_namespaces(
     }
 
     // apply cgroups
-    if p.init {
-        if res.is_some() {
-            info!(logger, "apply cgroups!");
-            cm.set(res.unwrap(), false)?;
-        }
+    if p.init && res.is_some() {
+        info!(logger, "apply cgroups!");
+        cm.set(res.unwrap(), false)?;
     }
 
     if res.is_some() {
@@ -1464,7 +1557,7 @@ fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
                 }
             }
 
-            return Ok(());
+            Ok(())
         }
 
         ForkResult::Child => {
@@ -1567,13 +1660,11 @@ fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
                             error
                         }
                     }
+                } else if let Ok(s) = rx.recv() {
+                    s
                 } else {
-                    if let Ok(s) = rx.recv() {
-                        s
-                    } else {
-                        let _ = signal::kill(Pid::from_raw(pid), Some(Signal::SIGKILL));
-                        -libc::EPIPE
-                    }
+                    let _ = signal::kill(Pid::from_raw(pid), Some(Signal::SIGKILL));
+                    -libc::EPIPE
                 }
             };
 
@@ -1591,6 +1682,18 @@ fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::process::Process;
+    use crate::skip_if_not_root;
+    use std::fs;
+    use std::os::unix::fs::MetadataExt;
+    use std::os::unix::io::AsRawFd;
+    use tempfile::tempdir;
+
+    macro_rules! sl {
+        () => {
+            slog_scope::logger()
+        };
+    }
 
     #[test]
     fn test_status_transtition() {
@@ -1609,4 +1712,318 @@ mod tests {
             assert_eq!(pre_status, status.pre_status());
         }
     }
+
+    #[test]
+    fn test_set_stdio_permissions() {
+        skip_if_not_root!();
+
+        let meta = fs::metadata("/dev/stdin").unwrap();
+        let old_uid = meta.uid();
+
+        let uid = 1000;
+        set_stdio_permissions(uid).unwrap();
+
+        let meta = fs::metadata("/dev/stdin").unwrap();
+        assert_eq!(meta.uid(), uid);
+
+        let meta = fs::metadata("/dev/stdout").unwrap();
+        assert_eq!(meta.uid(), uid);
+
+        let meta = fs::metadata("/dev/stderr").unwrap();
+        assert_eq!(meta.uid(), uid);
+
+        // restore the uid
+        set_stdio_permissions(old_uid).unwrap();
+    }
+
+    #[test]
+    fn test_status_fmt() {
+        assert_eq!("\"created\"", format!("{:?}", Status::CREATED));
+        assert_eq!("\"running\"", format!("{:?}", Status::RUNNING));
+        assert_eq!("\"paused\"", format!("{:?}", Status::PAUSED));
+        assert_eq!("\"stopped\"", format!("{:?}", Status::STOPPED));
+    }
+
+    #[test]
+    fn test_namespaces() {
+        lazy_static::initialize(&NAMESPACES);
+        assert_eq!(NAMESPACES.len(), 7);
+
+        let ns = NAMESPACES.get("user");
+        assert!(ns.is_some());
+
+        let ns = NAMESPACES.get("ipc");
+        assert!(ns.is_some());
+
+        let ns = NAMESPACES.get("pid");
+        assert!(ns.is_some());
+
+        let ns = NAMESPACES.get("network");
+        assert!(ns.is_some());
+
+        let ns = NAMESPACES.get("mount");
+        assert!(ns.is_some());
+
+        let ns = NAMESPACES.get("uts");
+        assert!(ns.is_some());
+
+        let ns = NAMESPACES.get("cgroup");
+        assert!(ns.is_some());
+    }
+
+    #[test]
+    fn test_typetoname() {
+        lazy_static::initialize(&TYPETONAME);
+        assert_eq!(TYPETONAME.len(), 7);
+
+        let ns = TYPETONAME.get("user");
+        assert!(ns.is_some());
+
+        let ns = TYPETONAME.get("ipc");
+        assert!(ns.is_some());
+
+        let ns = TYPETONAME.get("pid");
+        assert!(ns.is_some());
+
+        let ns = TYPETONAME.get("network");
+        assert!(ns.is_some());
+
+        let ns = TYPETONAME.get("mount");
+        assert!(ns.is_some());
+
+        let ns = TYPETONAME.get("uts");
+        assert!(ns.is_some());
+
+        let ns = TYPETONAME.get("cgroup");
+        assert!(ns.is_some());
+    }
+
+    fn create_dummy_opts() -> CreateOpts {
+        let mut root = oci::Root::default();
+        root.path = "/tmp".to_string();
+
+        let linux = Linux::default();
+        let mut spec = Spec::default();
+        spec.root = Some(root).into();
+        spec.linux = Some(linux).into();
+
+        CreateOpts {
+            cgroup_name: "".to_string(),
+            use_systemd_cgroup: false,
+            no_pivot_root: false,
+            no_new_keyring: false,
+            spec: Some(spec),
+            rootless_euid: false,
+            rootless_cgroup: false,
+        }
+    }
+
+    fn new_linux_container<U, F: FnOnce(LinuxContainer) -> Result<U, anyhow::Error>>(
+        op: F,
+    ) -> Result<U, anyhow::Error> {
+        // Create a temporal directory
+        tempdir()
+            .map_err(|e| anyhow!(e).context("tempdir failed"))
+            .and_then(|p: tempfile::TempDir| {
+                // Create a new container
+                LinuxContainer::new(
+                    "some_id",
+                    &p.path().join("rootfs").to_str().unwrap(),
+                    create_dummy_opts(),
+                    &slog_scope::logger(),
+                )
+                .and_then(op)
+            })
+    }
+
+    #[test]
+    fn test_linuxcontainer_pause_bad_status() {
+        let ret = new_linux_container(|mut c: LinuxContainer| {
+            // Change state to pause, c.pause() should fail
+            c.status.transition(Status::PAUSED);
+            c.pause().map_err(|e| anyhow!(e))
+        });
+
+        assert!(ret.is_err(), "Expecting error, Got {:?}", ret);
+        assert!(format!("{:?}", ret).contains("failed to pause container"))
+    }
+
+    #[test]
+    fn test_linuxcontainer_pause_cgroupmgr_is_none() {
+        let ret = new_linux_container(|mut c: LinuxContainer| {
+            c.cgroup_manager = None;
+            c.pause().map_err(|e| anyhow!(e))
+        });
+
+        assert!(ret.is_err(), "Expecting error, Got {:?}", ret);
+    }
+
+    #[test]
+    fn test_linuxcontainer_pause() {
+        let ret = new_linux_container(|mut c: LinuxContainer| {
+            c.cgroup_manager = FsManager::new("").ok();
+            c.pause().map_err(|e| anyhow!(e))
+        });
+
+        assert!(ret.is_ok(), "Expecting Ok, Got {:?}", ret);
+    }
+
+    #[test]
+    fn test_linuxcontainer_resume_bad_status() {
+        let ret = new_linux_container(|mut c: LinuxContainer| {
+            // Change state to created, c.resume() should fail
+            c.status.transition(Status::CREATED);
+            c.resume().map_err(|e| anyhow!(e))
+        });
+
+        assert!(ret.is_err(), "Expecting error, Got {:?}", ret);
+        assert!(format!("{:?}", ret).contains("not paused"))
+    }
+
+    #[test]
+    fn test_linuxcontainer_resume_cgroupmgr_is_none() {
+        let ret = new_linux_container(|mut c: LinuxContainer| {
+            c.status.transition(Status::PAUSED);
+            c.cgroup_manager = None;
+            c.resume().map_err(|e| anyhow!(e))
+        });
+
+        assert!(ret.is_err(), "Expecting error, Got {:?}", ret);
+    }
+
+    #[test]
+    fn test_linuxcontainer_resume() {
+        let ret = new_linux_container(|mut c: LinuxContainer| {
+            c.cgroup_manager = FsManager::new("").ok();
+            // Change status to paused, this way we can resume it
+            c.status.transition(Status::PAUSED);
+            c.resume().map_err(|e| anyhow!(e))
+        });
+
+        assert!(ret.is_ok(), "Expecting Ok, Got {:?}", ret);
+    }
+
+    #[test]
+    fn test_linuxcontainer_state() {
+        let ret = new_linux_container(|c: LinuxContainer| c.state());
+        assert!(ret.is_err(), "Expecting Err, Got {:?}", ret);
+        assert!(
+            format!("{:?}", ret).contains("not supported"),
+            "Got: {:?}",
+            ret
+        )
+    }
+
+    #[test]
+    fn test_linuxcontainer_oci_state_no_root_parent() {
+        let ret = new_linux_container(|mut c: LinuxContainer| {
+            c.config.spec.as_mut().unwrap().root.as_mut().unwrap().path = "/".to_string();
+            c.oci_state()
+        });
+        assert!(ret.is_err(), "Expecting Err, Got {:?}", ret);
+        assert!(
+            format!("{:?}", ret).contains("could not get root parent"),
+            "Got: {:?}",
+            ret
+        )
+    }
+
+    #[test]
+    fn test_linuxcontainer_oci_state() {
+        let ret = new_linux_container(|c: LinuxContainer| c.oci_state());
+        assert!(ret.is_ok(), "Expecting Ok, Got {:?}", ret);
+    }
+
+    #[test]
+    fn test_linuxcontainer_config() {
+        let ret = new_linux_container(|c: LinuxContainer| Ok(c));
+        assert!(ret.is_ok(), "Expecting ok, Got {:?}", ret);
+        assert!(
+            ret.as_ref().unwrap().config().is_ok(),
+            "Expecting ok, Got {:?}",
+            ret
+        );
+    }
+
+    #[test]
+    fn test_linuxcontainer_processes() {
+        let ret = new_linux_container(|c: LinuxContainer| c.processes());
+        assert!(ret.is_ok(), "Expecting Ok, Got {:?}", ret);
+    }
+
+    #[test]
+    fn test_linuxcontainer_get_process_not_found() {
+        let _ = new_linux_container(|mut c: LinuxContainer| {
+            let p = c.get_process("123");
+            assert!(p.is_err(), "Expecting Err, Got {:?}", p);
+            Ok(())
+        });
+    }
+
+    #[test]
+    fn test_linuxcontainer_get_process() {
+        let _ = new_linux_container(|mut c: LinuxContainer| {
+            c.processes.insert(
+                1,
+                Process::new(&sl!(), &oci::Process::default(), "123", true, 1).unwrap(),
+            );
+            let p = c.get_process("123");
+            assert!(p.is_ok(), "Expecting Ok, Got {:?}", p);
+            Ok(())
+        });
+    }
+
+    #[test]
+    fn test_linuxcontainer_stats() {
+        let ret = new_linux_container(|c: LinuxContainer| c.stats());
+        assert!(ret.is_ok(), "Expecting Ok, Got {:?}", ret);
+    }
+
+    #[test]
+    fn test_linuxcontainer_set() {
+        let ret =
+            new_linux_container(|mut c: LinuxContainer| c.set(oci::LinuxResources::default()));
+        assert!(ret.is_ok(), "Expecting Ok, Got {:?}", ret);
+    }
+
+    #[test]
+    fn test_linuxcontainer_start() {
+        let ret = new_linux_container(|mut c: LinuxContainer| {
+            c.start(Process::new(&sl!(), &oci::Process::default(), "123", true, 1).unwrap())
+        });
+        assert!(ret.is_err(), "Expecting Err, Got {:?}", ret);
+    }
+
+    #[test]
+    fn test_linuxcontainer_run() {
+        let ret = new_linux_container(|mut c: LinuxContainer| {
+            c.run(Process::new(&sl!(), &oci::Process::default(), "123", true, 1).unwrap())
+        });
+        assert!(ret.is_err(), "Expecting Err, Got {:?}", ret);
+    }
+
+    #[test]
+    fn test_linuxcontainer_destroy() {
+        let ret = new_linux_container(|mut c: LinuxContainer| c.destroy());
+        assert!(ret.is_ok(), "Expecting Ok, Got {:?}", ret);
+    }
+
+    #[test]
+    fn test_linuxcontainer_signal() {
+        let ret =
+            new_linux_container(|c: LinuxContainer| c.signal(nix::sys::signal::SIGCONT, true));
+        assert!(ret.is_ok(), "Expecting Ok, Got {:?}", ret);
+    }
+
+    #[test]
+    fn test_linuxcontainer_exec() {
+        let ret = new_linux_container(|mut c: LinuxContainer| c.exec());
+        assert!(ret.is_err(), "Expecting Err, Got {:?}", ret);
+    }
+
+    #[test]
+    fn test_linuxcontainer_do_init_child() {
+        let ret = do_init_child(std::io::stdin().as_raw_fd());
+        assert!(ret.is_err(), "Expecting Err, Got {:?}", ret);
+    }
 }

src/agent/rustjail/src/lib.rs

@@ -41,6 +41,7 @@ pub mod cgroups;
 pub mod container;
 pub mod mount;
 pub mod process;
+pub mod reaper;
 pub mod specconv;
 pub mod sync;
 pub mod validator;

src/agent/rustjail/src/mount.rs

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use anyhow::{anyhow, bail, Context, Error, Result};
+use anyhow::{anyhow, bail, Context, Result};
 use libc::uid_t;
 use nix::errno::Errno;
 use nix::fcntl::{self, OFlag};
@@ -22,13 +22,11 @@ use std::os::unix::io::RawFd;
 use std::path::{Path, PathBuf};
 
 use path_absolutize::*;
-use scan_fmt;
 use std::fs::File;
 use std::io::{BufRead, BufReader};
 
 use crate::container::DEFAULT_DEVICES;
 use crate::sync::write_count;
-use lazy_static;
 use std::string::ToString;
 
 use crate::log_child;
@@ -50,7 +48,7 @@ pub struct Info {
     vfs_opts: String,
 }
 
-const MOUNTINFOFORMAT: &'static str = "{d} {d} {d}:{d} {} {} {} {}";
+const MOUNTINFOFORMAT: &str = "{d} {d} {d}:{d} {} {} {} {}";
 const PROC_PATH: &str = "/proc";
 
 // since libc didn't defined this const for musl, thus redefined it here.
@@ -114,7 +112,12 @@ lazy_static! {
 
 #[inline(always)]
 #[allow(unused_variables)]
-fn mount<P1: ?Sized + NixPath, P2: ?Sized + NixPath, P3: ?Sized + NixPath, P4: ?Sized + NixPath>(
+pub fn mount<
+    P1: ?Sized + NixPath,
+    P2: ?Sized + NixPath,
+    P3: ?Sized + NixPath,
+    P4: ?Sized + NixPath,
+>(
     source: Option<&P1>,
     target: &P2,
     fstype: Option<&P3>,
@@ -129,7 +132,7 @@ fn mount<P1: ?Sized + NixPath, P2: ?Sized + NixPath, P3: ?Sized + NixPath, P4: ?
 
 #[inline(always)]
 #[allow(unused_variables)]
-fn umount2<P: ?Sized + NixPath>(
+pub fn umount2<P: ?Sized + NixPath>(
     target: &P,
     flags: MntFlags,
 ) -> std::result::Result<(), nix::Error> {
@@ -153,7 +156,7 @@ pub fn init_rootfs(
     let linux = &spec
         .linux
         .as_ref()
-        .ok_or::<Error>(anyhow!("Could not get linux configuration from spec"))?;
+        .ok_or_else(|| anyhow!("Could not get linux configuration from spec"))?;
 
     let mut flags = MsFlags::MS_REC;
     match PROPAGATION.get(&linux.rootfs_propagation.as_str()) {
@@ -164,14 +167,14 @@ pub fn init_rootfs(
     let root = spec
         .root
         .as_ref()
-        .ok_or(anyhow!("Could not get rootfs path from spec"))
+        .ok_or_else(|| anyhow!("Could not get rootfs path from spec"))
         .and_then(|r| {
             fs::canonicalize(r.path.as_str()).context("Could not canonicalize rootfs path")
         })?;
 
     let rootfs = (*root)
         .to_str()
-        .ok_or(anyhow!("Could not convert rootfs path to string"))?;
+        .ok_or_else(|| anyhow!("Could not convert rootfs path to string"))?;
 
     mount(None::<&str>, "/", None::<&str>, flags, None::<&str>)?;
 
@@ -188,7 +191,7 @@ pub fn init_rootfs(
     let mut bind_mount_dev = false;
     for m in &spec.mounts {
         let (mut flags, data) = parse_mount(&m);
-        if !m.destination.starts_with("/") || m.destination.contains("..") {
+        if !m.destination.starts_with('/') || m.destination.contains("..") {
             return Err(anyhow!(
                 "the mount destination {} is invalid",
                 m.destination
@@ -282,9 +285,9 @@ fn check_proc_mount(m: &Mount) -> Result<()> {
         // only allow a mount on-top of proc if it's source is "proc"
         unsafe {
             let mut stats = MaybeUninit::<libc::statfs>::uninit();
-            if let Ok(_) = m
-                .source
+            if m.source
                 .with_nix_path(|path| libc::statfs(path.as_ptr(), stats.as_mut_ptr()))
+                .is_ok()
             {
                 if stats.assume_init().f_type == PROC_SUPER_MAGIC {
                     return Ok(());
@@ -307,7 +310,7 @@ fn check_proc_mount(m: &Mount) -> Result<()> {
         )));
     }
 
-    return Ok(());
+    Ok(())
 }
 
 fn mount_cgroups_v2(cfd_log: RawFd, m: &Mount, rootfs: &str, flags: MsFlags) -> Result<()> {
@@ -595,15 +598,14 @@ pub fn ms_move_root(rootfs: &str) -> Result<bool> {
     let abs_root_buf = root_path.absolutize()?;
     let abs_root = abs_root_buf
         .to_str()
-        .ok_or::<Error>(anyhow!("failed to parse {} to absolute path", rootfs))?;
+        .ok_or_else(|| anyhow!("failed to parse {} to absolute path", rootfs))?;
 
     for info in mount_infos.iter() {
         let mount_point = Path::new(&info.mount_point);
         let abs_mount_buf = mount_point.absolutize()?;
-        let abs_mount_point = abs_mount_buf.to_str().ok_or::<Error>(anyhow!(
-            "failed to parse {} to absolute path",
-            info.mount_point
-        ))?;
+        let abs_mount_point = abs_mount_buf
+            .to_str()
+            .ok_or_else(|| anyhow!("failed to parse {} to absolute path", info.mount_point))?;
         let abs_mount_point_string = String::from(abs_mount_point);
 
         // Umount every syfs and proc file systems, except those under the container rootfs
@@ -764,7 +766,7 @@ fn mount_from(
     Ok(())
 }
 
-static SYMLINKS: &'static [(&'static str, &'static str)] = &[
+static SYMLINKS: &[(&str, &str)] = &[
     ("/proc/self/fd", "dev/fd"),
     ("/proc/self/fd/0", "dev/stdin"),
     ("/proc/self/fd/1", "dev/stdout"),
@@ -897,7 +899,7 @@ pub fn finish_rootfs(cfd_log: RawFd, spec: &Spec) -> Result<()> {
 }
 
 fn mask_path(path: &str) -> Result<()> {
-    if !path.starts_with("/") || path.contains("..") {
+    if !path.starts_with('/') || path.contains("..") {
         return Err(nix::Error::Sys(Errno::EINVAL).into());
     }
 
@@ -926,7 +928,7 @@ fn mask_path(path: &str) -> Result<()> {
 }
 
 fn readonly_path(path: &str) -> Result<()> {
-    if !path.starts_with("/") || path.contains("..") {
+    if !path.starts_with('/') || path.contains("..") {
         return Err(nix::Error::Sys(Errno::EINVAL).into());
     }
 

src/agent/rustjail/src/process.rs

@@ -14,6 +14,7 @@ use nix::sys::wait::{self, WaitStatus};
 use nix::unistd::{self, Pid};
 use nix::Result;
 
+use crate::reaper::Epoller;
 use oci::Process as OCIProcess;
 use slog::Logger;
 
@@ -40,6 +41,7 @@ pub struct Process {
     pub exit_watchers: Vec<Sender<i32>>,
     pub oci: OCIProcess,
     pub logger: Logger,
+    pub epoller: Option<Epoller>,
 }
 
 pub trait ProcessOperations {
@@ -91,6 +93,7 @@ impl Process {
             exit_watchers: Vec::new(),
             oci: ocip.clone(),
             logger: logger.clone(),
+            epoller: None,
         };
 
         info!(logger, "before create console socket!");
@@ -112,6 +115,29 @@ impl Process {
         }
         Ok(p)
     }
+
+    pub fn close_epoller(&mut self) {
+        if let Some(epoller) = self.epoller.take() {
+            epoller.close();
+        }
+    }
+
+    pub fn create_epoller(&mut self) -> anyhow::Result<()> {
+        match self.term_master {
+            Some(term_master) => {
+                // add epoller to process
+                let epoller = Epoller::new(&self.logger, term_master)?;
+                self.epoller = Some(epoller)
+            }
+            None => {
+                info!(
+                    self.logger,
+                    "try to add epoller to a process without a term master fd"
+                );
+            }
+        }
+        Ok(())
+    }
 }
 
 fn create_extended_pipe(flags: OFlag, pipe_size: i32) -> Result<(RawFd, RawFd)> {

src/agent/rustjail/src/reaper.rs

@@ -0,0 +1,150 @@
+// Copyright (c) 2020 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use nix::fcntl::OFlag;
+use slog::Logger;
+
+use nix::unistd;
+use std::os::unix::io::RawFd;
+
+use anyhow::Result;
+
+const MAX_EVENTS: usize = 2;
+
+#[derive(Debug, Clone)]
+pub struct Epoller {
+    logger: Logger,
+    epoll_fd: RawFd,
+    // rfd and wfd are a pipe's files two ends, this pipe is
+    // used to sync between the readStdio and the process exits.
+    // once the process exits, it will close one end to notify
+    // the readStdio that the process has exited and it should not
+    // wait on the process's terminal which has been inherited
+    // by it's children and hasn't exited.
+    rfd: RawFd,
+    wfd: RawFd,
+}
+
+impl Epoller {
+    pub fn new(logger: &Logger, fd: RawFd) -> Result<Epoller> {
+        let epoll_fd = epoll::create(true)?;
+        let (rfd, wfd) = unistd::pipe2(OFlag::O_CLOEXEC)?;
+
+        let mut epoller = Self {
+            logger: logger.clone(),
+            epoll_fd,
+            rfd,
+            wfd,
+        };
+
+        epoller.add(rfd)?;
+        epoller.add(fd)?;
+
+        Ok(epoller)
+    }
+
+    pub fn close_wfd(&self) {
+        let _ = unistd::close(self.wfd);
+    }
+
+    pub fn close(&self) {
+        let _ = unistd::close(self.rfd);
+        let _ = unistd::close(self.wfd);
+        let _ = unistd::close(self.epoll_fd);
+    }
+
+    fn add(&mut self, fd: RawFd) -> Result<()> {
+        info!(self.logger, "Epoller add fd {}", fd);
+        // add creates an epoll which is used to monitor the process's pty's master and
+        // one end of its exit notify pipe. Those files will be registered with level-triggered
+        // notification.
+        epoll::ctl(
+            self.epoll_fd,
+            epoll::ControlOptions::EPOLL_CTL_ADD,
+            fd,
+            epoll::Event::new(
+                epoll::Events::EPOLLHUP
+                    | epoll::Events::EPOLLIN
+                    | epoll::Events::EPOLLERR
+                    | epoll::Events::EPOLLRDHUP,
+                fd as u64,
+            ),
+        )?;
+
+        Ok(())
+    }
+
+    // There will be three cases on the epoller once it poll:
+    // a: only pty's master get an event(other than self.rfd);
+    // b: only the pipe get an event(self.rfd);
+    // c: both of pty and pipe have event occur;
+    // for case a, it means there is output in process's terminal and what needed to do is
+    // just read the terminal and send them out; for case b, it means the process has exited
+    // and there is no data in the terminal, thus just return the "EOF" to end the io;
+    // for case c, it means the process has exited but there is some data in the terminal which
+    // hasn't been send out, thus it should send those data out first and then send "EOF" last to
+    // end the io.
+    pub fn poll(&self) -> Result<RawFd> {
+        let mut rfd = self.rfd;
+        let mut epoll_events = vec![epoll::Event::new(epoll::Events::empty(), 0); MAX_EVENTS];
+
+        loop {
+            let event_count = match epoll::wait(self.epoll_fd, -1, epoll_events.as_mut_slice()) {
+                Ok(ec) => ec,
+                Err(e) => {
+                    info!(self.logger, "loop wait err {:?}", e);
+                    // EINTR: The call was interrupted by a signal handler before either
+                    // any of the requested events occurred or the timeout expired
+                    if e.kind() == std::io::ErrorKind::Interrupted {
+                        continue;
+                    }
+                    return Err(e.into());
+                }
+            };
+
+            for event in epoll_events.iter().take(event_count) {
+                let fd = event.data as i32;
+                // fd has been assigned with one end of process's exited pipe by default, and
+                // here to check is there any event occur on process's terminal, if "yes", it
+                // should be dealt first, otherwise, it means the process has exited and there
+                // is nothing left in the process's terminal needed to be read.
+                if fd != rfd {
+                    rfd = fd;
+                    break;
+                }
+            }
+            break;
+        }
+
+        Ok(rfd)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Epoller;
+    use nix::fcntl::OFlag;
+    use nix::unistd;
+    use std::thread;
+
+    #[test]
+    fn test_epoller_poll() {
+        let logger = slog::Logger::root(slog::Discard, o!());
+        let (rfd, wfd) = unistd::pipe2(OFlag::O_CLOEXEC).unwrap();
+        let epoller = Epoller::new(&logger, rfd).unwrap();
+
+        let child = thread::spawn(move || {
+            let _ = unistd::write(wfd, "temporary file's content".as_bytes());
+        });
+
+        // wait write to finish
+        let _ = child.join();
+
+        let fd = epoller.poll().unwrap();
+        assert_eq!(fd, rfd, "Should get rfd");
+
+        epoller.close();
+    }
+}

src/agent/rustjail/src/sync.rs

@@ -96,14 +96,14 @@ pub fn read_sync(fd: RawFd) -> Result<Vec<u8>> {
     let buf_array: [u8; MSG_SIZE] = [buf[0], buf[1], buf[2], buf[3]];
     let msg: i32 = i32::from_be_bytes(buf_array);
     match msg {
-        SYNC_SUCCESS => return Ok(Vec::new()),
+        SYNC_SUCCESS => Ok(Vec::new()),
         SYNC_DATA => {
             let buf = read_count(fd, MSG_SIZE)?;
             let buf_array: [u8; MSG_SIZE] = [buf[0], buf[1], buf[2], buf[3]];
             let msg_length: i32 = i32::from_be_bytes(buf_array);
             let data_buf = read_count(fd, msg_length as usize)?;
 
-            return Ok(data_buf);
+            Ok(data_buf)
         }
         SYNC_FAILED => {
             let mut error_buf = vec![];
@@ -127,9 +127,9 @@ pub fn read_sync(fd: RawFd) -> Result<Vec<u8>> {
                 }
             };
 
-            return Err(anyhow!(error_str));
+            Err(anyhow!(error_str))
         }
-        _ => return Err(anyhow!("error in receive sync message")),
+        _ => Err(anyhow!("error in receive sync message")),
     }
 }
 

src/agent/rustjail/src/validator.rs

@@ -4,14 +4,21 @@
 //
 
 use crate::container::Config;
-use anyhow::{anyhow, Context, Result};
-use lazy_static;
+use anyhow::{anyhow, Context, Error, Result};
 use nix::errno::Errno;
-use oci::{LinuxIDMapping, LinuxNamespace, Spec};
+use oci::{Linux, LinuxIDMapping, LinuxNamespace, Spec};
 use std::collections::HashMap;
 use std::path::{Component, PathBuf};
 
-fn contain_namespace(nses: &Vec<LinuxNamespace>, key: &str) -> bool {
+fn einval() -> Error {
+    anyhow!(nix::Error::from_errno(Errno::EINVAL))
+}
+
+fn get_linux(oci: &Spec) -> Result<&Linux> {
+    oci.linux.as_ref().ok_or_else(einval)
+}
+
+fn contain_namespace(nses: &[LinuxNamespace], key: &str) -> bool {
     for ns in nses {
         if ns.r#type.as_str() == key {
             return true;
@@ -21,30 +28,28 @@ fn contain_namespace(nses: &Vec<LinuxNamespace>, key: &str) -> bool {
     false
 }
 
-fn get_namespace_path(nses: &Vec<LinuxNamespace>, key: &str) -> Result<String> {
+fn get_namespace_path(nses: &[LinuxNamespace], key: &str) -> Result<String> {
     for ns in nses {
         if ns.r#type.as_str() == key {
             return Ok(ns.path.clone());
         }
     }
 
-    Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)))
+    Err(einval())
 }
 
 fn rootfs(root: &str) -> Result<()> {
     let path = PathBuf::from(root);
     // not absolute path or not exists
     if !path.exists() || !path.is_absolute() {
-        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
+        return Err(einval());
     }
 
     // symbolic link? ..?
     let mut stack: Vec<String> = Vec::new();
     for c in path.components() {
-        if stack.is_empty() {
-            if c == Component::RootDir || c == Component::ParentDir {
-                continue;
-            }
+        if stack.is_empty() && (c == Component::RootDir || c == Component::ParentDir) {
+            continue;
         }
 
         if c == Component::ParentDir {
@@ -55,7 +60,7 @@ fn rootfs(root: &str) -> Result<()> {
         if let Some(v) = c.as_os_str().to_str() {
             stack.push(v.to_string());
         } else {
-            return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
+            return Err(einval());
         }
     }
 
@@ -67,7 +72,7 @@ fn rootfs(root: &str) -> Result<()> {
     let canon = path.canonicalize().context("canonicalize")?;
     if cleaned != canon {
         // There is symbolic in path
-        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
+        return Err(einval());
     }
 
     Ok(())
@@ -78,32 +83,27 @@ fn network(_oci: &Spec) -> Result<()> {
 }
 
 fn hostname(oci: &Spec) -> Result<()> {
-    if oci.hostname.is_empty() || oci.hostname == "".to_string() {
+    if oci.hostname.is_empty() || oci.hostname == "" {
         return Ok(());
     }
 
-    let linux = oci
-        .linux
-        .as_ref()
-        .ok_or(anyhow!(nix::Error::from_errno(Errno::EINVAL)))?;
+    let linux = get_linux(oci)?;
     if !contain_namespace(&linux.namespaces, "uts") {
-        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
+        return Err(einval());
     }
 
     Ok(())
 }
 
 fn security(oci: &Spec) -> Result<()> {
-    let linux = oci
-        .linux
-        .as_ref()
-        .ok_or(anyhow!(nix::Error::from_errno(Errno::EINVAL)))?;
+    let linux = get_linux(oci)?;
+
     if linux.masked_paths.is_empty() && linux.readonly_paths.is_empty() {
         return Ok(());
     }
 
     if !contain_namespace(&linux.namespaces, "mount") {
-        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
+        return Err(einval());
     }
 
     // don't care about selinux at present
@@ -111,21 +111,19 @@ fn security(oci: &Spec) -> Result<()> {
     Ok(())
 }
 
-fn idmapping(maps: &Vec<LinuxIDMapping>) -> Result<()> {
+fn idmapping(maps: &[LinuxIDMapping]) -> Result<()> {
     for map in maps {
         if map.size > 0 {
             return Ok(());
         }
     }
 
-    Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)))
+    Err(einval())
 }
 
 fn usernamespace(oci: &Spec) -> Result<()> {
-    let linux = oci
-        .linux
-        .as_ref()
-        .ok_or(anyhow!(nix::Error::from_errno(Errno::EINVAL)))?;
+    let linux = get_linux(oci)?;
+
     if contain_namespace(&linux.namespaces, "user") {
         let user_ns = PathBuf::from("/proc/self/ns/user");
         if !user_ns.exists() {
@@ -137,8 +135,8 @@ fn usernamespace(oci: &Spec) -> Result<()> {
         idmapping(&linux.gid_mappings).context("idmapping gid")?;
     } else {
         // no user namespace but idmap
-        if linux.uid_mappings.len() != 0 || linux.gid_mappings.len() != 0 {
-            return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
+        if !linux.uid_mappings.is_empty() || !linux.gid_mappings.is_empty() {
+            return Err(einval());
         }
     }
 
@@ -146,10 +144,8 @@ fn usernamespace(oci: &Spec) -> Result<()> {
 }
 
 fn cgroupnamespace(oci: &Spec) -> Result<()> {
-    let linux = oci
-        .linux
-        .as_ref()
-        .ok_or(anyhow!(nix::Error::from_errno(Errno::EINVAL)))?;
+    let linux = get_linux(oci)?;
+
     if contain_namespace(&linux.namespaces, "cgroup") {
         let path = PathBuf::from("/proc/self/ns/cgroup");
         if !path.exists() {
@@ -193,23 +189,21 @@ fn check_host_ns(path: &str) -> Result<()> {
         .read_link()
         .context(format!("read link {:?}", cpath))?;
     if real_cpath == real_hpath {
-        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
+        return Err(einval());
     }
 
     Ok(())
 }
 
 fn sysctl(oci: &Spec) -> Result<()> {
-    let linux = oci
-        .linux
-        .as_ref()
-        .ok_or(anyhow!(nix::Error::from_errno(Errno::EINVAL)))?;
+    let linux = get_linux(oci)?;
+
     for (key, _) in linux.sysctl.iter() {
         if SYSCTLS.contains_key(key.as_str()) || key.starts_with("fs.mqueue.") {
             if contain_namespace(&linux.namespaces, "ipc") {
                 continue;
             } else {
-                return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
+                return Err(einval());
             }
         }
 
@@ -224,33 +218,31 @@ fn sysctl(oci: &Spec) -> Result<()> {
             }
 
             if key == "kernel.hostname" {
-                return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
+                return Err(einval());
             }
         }
 
-        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
+        return Err(einval());
     }
     Ok(())
 }
 
 fn rootless_euid_mapping(oci: &Spec) -> Result<()> {
-    let linux = oci
-        .linux
-        .as_ref()
-        .ok_or(anyhow!(nix::Error::from_errno(Errno::EINVAL)))?;
+    let linux = get_linux(oci)?;
+
     if !contain_namespace(&linux.namespaces, "user") {
-        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
+        return Err(einval());
     }
 
-    if linux.uid_mappings.len() == 0 || linux.gid_mappings.len() == 0 {
+    if linux.uid_mappings.is_empty() || linux.gid_mappings.is_empty() {
         // rootless containers requires at least one UID/GID mapping
-        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
+        return Err(einval());
     }
 
     Ok(())
 }
 
-fn has_idmapping(maps: &Vec<LinuxIDMapping>, id: u32) -> bool {
+fn has_idmapping(maps: &[LinuxIDMapping], id: u32) -> bool {
     for map in maps {
         if id >= map.container_id && id < map.container_id + map.size {
             return true;
@@ -260,10 +252,7 @@ fn has_idmapping(maps: &Vec<LinuxIDMapping>, id: u32) -> bool {
 }
 
 fn rootless_euid_mount(oci: &Spec) -> Result<()> {
-    let linux = oci
-        .linux
-        .as_ref()
-        .ok_or(anyhow!(nix::Error::from_errno(Errno::EINVAL)))?;
+    let linux = get_linux(oci)?;
 
     for mnt in oci.mounts.iter() {
         for opt in mnt.options.iter() {
@@ -271,7 +260,7 @@ fn rootless_euid_mount(oci: &Spec) -> Result<()> {
                 let fields: Vec<&str> = opt.split('=').collect();
 
                 if fields.len() != 2 {
-                    return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
+                    return Err(einval());
                 }
 
                 let id = fields[1]
@@ -279,16 +268,12 @@ fn rootless_euid_mount(oci: &Spec) -> Result<()> {
                     .parse::<u32>()
                     .context(format!("parse field {}", &fields[1]))?;
 
-                if opt.starts_with("uid=") {
-                    if !has_idmapping(&linux.uid_mappings, id) {
-                        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
-                    }
+                if opt.starts_with("uid=") && !has_idmapping(&linux.uid_mappings, id) {
+                    return Err(einval());
                 }
 
-                if opt.starts_with("gid=") {
-                    if !has_idmapping(&linux.gid_mappings, id) {
-                        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
-                    }
+                if opt.starts_with("gid=") && !has_idmapping(&linux.gid_mappings, id) {
+                    return Err(einval());
                 }
             }
         }
@@ -304,18 +289,15 @@ fn rootless_euid(oci: &Spec) -> Result<()> {
 
 pub fn validate(conf: &Config) -> Result<()> {
     lazy_static::initialize(&SYSCTLS);
-    let oci = conf
-        .spec
-        .as_ref()
-        .ok_or(anyhow!(nix::Error::from_errno(Errno::EINVAL)))?;
+    let oci = conf.spec.as_ref().ok_or_else(einval)?;
 
     if oci.linux.is_none() {
-        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
+        return Err(einval());
     }
 
     let root = match oci.root.as_ref() {
         Some(v) => v.path.as_str(),
-        None => return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL))),
+        None => return Err(einval()),
     };
 
     rootfs(root).context("rootfs")?;
@@ -332,3 +314,274 @@ pub fn validate(conf: &Config) -> Result<()> {
 
     Ok(())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use oci::Mount;
+
+    #[test]
+    fn test_namespace() {
+        let namespaces = [
+            LinuxNamespace {
+                r#type: "net".to_owned(),
+                path: "/sys/cgroups/net".to_owned(),
+            },
+            LinuxNamespace {
+                r#type: "uts".to_owned(),
+                path: "/sys/cgroups/uts".to_owned(),
+            },
+        ];
+
+        assert_eq!(contain_namespace(&namespaces, "net"), true);
+        assert_eq!(contain_namespace(&namespaces, "uts"), true);
+
+        assert_eq!(contain_namespace(&namespaces, ""), false);
+        assert_eq!(contain_namespace(&namespaces, "Net"), false);
+        assert_eq!(contain_namespace(&namespaces, "ipc"), false);
+
+        assert_eq!(
+            get_namespace_path(&namespaces, "net").unwrap(),
+            "/sys/cgroups/net"
+        );
+        assert_eq!(
+            get_namespace_path(&namespaces, "uts").unwrap(),
+            "/sys/cgroups/uts"
+        );
+
+        get_namespace_path(&namespaces, "").unwrap_err();
+        get_namespace_path(&namespaces, "Uts").unwrap_err();
+        get_namespace_path(&namespaces, "ipc").unwrap_err();
+    }
+
+    #[test]
+    fn test_rootfs() {
+        rootfs("/_no_exit_fs_xxxxxxxxxxx").unwrap_err();
+        rootfs("sys").unwrap_err();
+        rootfs("/proc/self/root").unwrap_err();
+        rootfs("/proc/self/root/sys").unwrap_err();
+
+        rootfs("/proc/self").unwrap_err();
+        rootfs("/./proc/self").unwrap_err();
+        rootfs("/proc/././self").unwrap_err();
+        rootfs("/proc/.././self").unwrap_err();
+
+        rootfs("/proc/uptime").unwrap();
+        rootfs("/../proc/uptime").unwrap();
+        rootfs("/../../proc/uptime").unwrap();
+        rootfs("/proc/../proc/uptime").unwrap();
+        rootfs("/proc/../../proc/uptime").unwrap();
+    }
+
+    #[test]
+    fn test_hostname() {
+        let mut spec = Spec::default();
+
+        hostname(&spec).unwrap();
+
+        spec.hostname = "a.test.com".to_owned();
+        hostname(&spec).unwrap_err();
+
+        let mut linux = Linux::default();
+        linux.namespaces = vec![
+            LinuxNamespace {
+                r#type: "net".to_owned(),
+                path: "/sys/cgroups/net".to_owned(),
+            },
+            LinuxNamespace {
+                r#type: "uts".to_owned(),
+                path: "/sys/cgroups/uts".to_owned(),
+            },
+        ];
+        spec.linux = Some(linux);
+        hostname(&spec).unwrap();
+    }
+
+    #[test]
+    fn test_security() {
+        let mut spec = Spec::default();
+
+        let linux = Linux::default();
+        spec.linux = Some(linux);
+        security(&spec).unwrap();
+
+        let mut linux = Linux::default();
+        linux.masked_paths.push("/test".to_owned());
+        linux.namespaces = vec![
+            LinuxNamespace {
+                r#type: "net".to_owned(),
+                path: "/sys/cgroups/net".to_owned(),
+            },
+            LinuxNamespace {
+                r#type: "uts".to_owned(),
+                path: "/sys/cgroups/uts".to_owned(),
+            },
+        ];
+        spec.linux = Some(linux);
+        security(&spec).unwrap_err();
+
+        let mut linux = Linux::default();
+        linux.masked_paths.push("/test".to_owned());
+        linux.namespaces = vec![
+            LinuxNamespace {
+                r#type: "net".to_owned(),
+                path: "/sys/cgroups/net".to_owned(),
+            },
+            LinuxNamespace {
+                r#type: "mount".to_owned(),
+                path: "/sys/cgroups/mount".to_owned(),
+            },
+        ];
+        spec.linux = Some(linux);
+        security(&spec).unwrap();
+    }
+
+    #[test]
+    fn test_usernamespace() {
+        let mut spec = Spec::default();
+        usernamespace(&spec).unwrap_err();
+
+        let linux = Linux::default();
+        spec.linux = Some(linux);
+        usernamespace(&spec).unwrap();
+
+        let mut linux = Linux::default();
+        linux.uid_mappings = vec![LinuxIDMapping {
+            container_id: 0,
+            host_id: 1000,
+            size: 0,
+        }];
+        spec.linux = Some(linux);
+        usernamespace(&spec).unwrap_err();
+
+        let mut linux = Linux::default();
+        linux.uid_mappings = vec![LinuxIDMapping {
+            container_id: 0,
+            host_id: 1000,
+            size: 100,
+        }];
+        spec.linux = Some(linux);
+        usernamespace(&spec).unwrap_err();
+    }
+
+    #[test]
+    fn test_rootless_euid() {
+        let mut spec = Spec::default();
+
+        // Test case: without linux
+        rootless_euid_mapping(&spec).unwrap_err();
+        rootless_euid_mount(&spec).unwrap_err();
+
+        // Test case: without user namespace
+        let linux = Linux::default();
+        spec.linux = Some(linux);
+        rootless_euid_mapping(&spec).unwrap_err();
+
+        // Test case: without user namespace
+        let linux = spec.linux.as_mut().unwrap();
+        linux.namespaces = vec![
+            LinuxNamespace {
+                r#type: "net".to_owned(),
+                path: "/sys/cgroups/net".to_owned(),
+            },
+            LinuxNamespace {
+                r#type: "uts".to_owned(),
+                path: "/sys/cgroups/uts".to_owned(),
+            },
+        ];
+        rootless_euid_mapping(&spec).unwrap_err();
+
+        let linux = spec.linux.as_mut().unwrap();
+        linux.namespaces = vec![
+            LinuxNamespace {
+                r#type: "net".to_owned(),
+                path: "/sys/cgroups/net".to_owned(),
+            },
+            LinuxNamespace {
+                r#type: "user".to_owned(),
+                path: "/sys/cgroups/user".to_owned(),
+            },
+        ];
+        linux.uid_mappings = vec![LinuxIDMapping {
+            container_id: 0,
+            host_id: 1000,
+            size: 1000,
+        }];
+        linux.gid_mappings = vec![LinuxIDMapping {
+            container_id: 0,
+            host_id: 1000,
+            size: 1000,
+        }];
+        rootless_euid_mapping(&spec).unwrap();
+
+        spec.mounts.push(Mount {
+            destination: "/app".to_owned(),
+            r#type: "tmpfs".to_owned(),
+            source: "".to_owned(),
+            options: vec!["uid=10000".to_owned()],
+        });
+        rootless_euid_mount(&spec).unwrap_err();
+
+        spec.mounts = vec![
+            (Mount {
+                destination: "/app".to_owned(),
+                r#type: "tmpfs".to_owned(),
+                source: "".to_owned(),
+                options: vec!["uid=500".to_owned(), "gid=500".to_owned()],
+            }),
+        ];
+        rootless_euid(&spec).unwrap();
+    }
+
+    #[test]
+    fn test_check_host_ns() {
+        check_host_ns("/proc/self/ns/net").unwrap_err();
+        check_host_ns("/proc/sys/net/ipv4/tcp_sack").unwrap();
+    }
+
+    #[test]
+    fn test_sysctl() {
+        let mut spec = Spec::default();
+
+        let mut linux = Linux::default();
+        linux.namespaces = vec![LinuxNamespace {
+            r#type: "net".to_owned(),
+            path: "/sys/cgroups/net".to_owned(),
+        }];
+        linux
+            .sysctl
+            .insert("kernel.domainname".to_owned(), "test.com".to_owned());
+        spec.linux = Some(linux);
+        sysctl(&spec).unwrap_err();
+
+        spec.linux
+            .as_mut()
+            .unwrap()
+            .namespaces
+            .push(LinuxNamespace {
+                r#type: "uts".to_owned(),
+                path: "/sys/cgroups/uts".to_owned(),
+            });
+        sysctl(&spec).unwrap();
+    }
+
+    #[test]
+    fn test_validate() {
+        let spec = Spec::default();
+        let mut config = Config {
+            cgroup_name: "container1".to_owned(),
+            use_systemd_cgroup: false,
+            no_pivot_root: true,
+            no_new_keyring: true,
+            rootless_euid: false,
+            rootless_cgroup: false,
+            spec: Some(spec),
+        };
+
+        validate(&config).unwrap_err();
+
+        let linux = Linux::default();
+        config.spec.as_mut().unwrap().linux = Some(linux);
+        validate(&config).unwrap_err();
+    }
+}

src/agent/src/config.rs

@@ -21,7 +21,10 @@ const DEFAULT_HOTPLUG_TIMEOUT: time::Duration = time::Duration::from_secs(3);
 const DEFAULT_CONTAINER_PIPE_SIZE: i32 = 0;
 const VSOCK_ADDR: &str = "vsock://-1";
 const VSOCK_PORT: u16 = 1024;
+
+// Environment variables used for development and testing
 const SERVER_ADDR_ENV_VAR: &str = "KATA_AGENT_SERVER_ADDR";
+const LOG_LEVEL_ENV_VAR: &str = "KATA_AGENT_LOG_LEVEL";
 
 // FIXME: unused
 const TRACE_MODE_FLAG: &str = "agent.trace";
@@ -139,12 +142,18 @@ impl agentConfig {
             self.server_addr = addr;
         }
 
+        if let Ok(addr) = env::var(LOG_LEVEL_ENV_VAR) {
+            if let Ok(level) = logrus_to_slog_level(&addr) {
+                self.log_level = level;
+            }
+        }
+
         Ok(())
     }
 }
 
 fn get_vsock_port(p: &str) -> Result<i32> {
-    let fields: Vec<&str> = p.split("=").collect();
+    let fields: Vec<&str> = p.split('=').collect();
     if fields.len() != 2 {
         return Err(anyhow!("invalid port parameter"));
     }
@@ -180,7 +189,7 @@ fn logrus_to_slog_level(logrus_level: &str) -> Result<slog::Level> {
 }
 
 fn get_log_level(param: &str) -> Result<slog::Level> {
-    let fields: Vec<&str> = param.split("=").collect();
+    let fields: Vec<&str> = param.split('=').collect();
 
     if fields.len() != 2 {
         return Err(anyhow!("invalid log level parameter"));
@@ -194,7 +203,7 @@ fn get_log_level(param: &str) -> Result<slog::Level> {
 }
 
 fn get_hotplug_timeout(param: &str) -> Result<time::Duration> {
-    let fields: Vec<&str> = param.split("=").collect();
+    let fields: Vec<&str> = param.split('=').collect();
 
     if fields.len() != 2 {
         return Err(anyhow!("invalid hotplug timeout parameter"));
@@ -214,7 +223,7 @@ fn get_hotplug_timeout(param: &str) -> Result<time::Duration> {
 }
 
 fn get_bool_value(param: &str) -> Result<bool> {
-    let fields: Vec<&str> = param.split("=").collect();
+    let fields: Vec<&str> = param.split('=').collect();
 
     if fields.len() != 2 {
         return Ok(false);
@@ -225,18 +234,12 @@ fn get_bool_value(param: &str) -> Result<bool> {
     // first try to parse as bool value
     v.parse::<bool>().or_else(|_err1| {
         // then try to parse as integer value
-        v.parse::<u64>().or_else(|_err2| Ok(0)).and_then(|v| {
-            // only `0` returns false, otherwise returns true
-            Ok(match v {
-                0 => false,
-                _ => true,
-            })
-        })
+        v.parse::<u64>().or(Ok(0)).map(|v| !matches!(v, 0))
     })
 }
 
 fn get_container_pipe_size(param: &str) -> Result<i32> {
-    let fields: Vec<&str> = param.split("=").collect();
+    let fields: Vec<&str> = param.split('=').collect();
 
     if fields.len() != 2 {
         return Err(anyhow!("invalid container pipe size parameter"));
@@ -321,297 +324,495 @@ mod tests {
 
     #[test]
     fn test_parse_cmdline() {
+        const TEST_SERVER_ADDR: &str = "vsock://-1:1024";
+
         #[derive(Debug)]
         struct TestData<'a> {
             contents: &'a str,
+            env_vars: Vec<&'a str>,
             debug_console: bool,
             dev_mode: bool,
             log_level: slog::Level,
             hotplug_timeout: time::Duration,
             container_pipe_size: i32,
+            server_addr: &'a str,
             unified_cgroup_hierarchy: bool,
         }
 
         let tests = &[
             TestData {
                 contents: "agent.debug_consolex agent.devmode",
+                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "agent.debug_console agent.devmodex",
+                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "agent.logx=debug",
+                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "agent.log=debug",
+                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: false,
                 log_level: slog::Level::Debug,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
+                unified_cgroup_hierarchy: false,
+            },
+            TestData {
+                contents: "agent.log=debug",
+                env_vars: vec!["KATA_AGENT_LOG_LEVEL=trace"],
+                debug_console: false,
+                dev_mode: false,
+                log_level: slog::Level::Trace,
+                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
+                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "",
+                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "foo",
+                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "foo bar",
+                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "foo bar",
+                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "foo agent bar",
+                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "foo debug_console agent bar devmode",
+                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "agent.debug_console",
+                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "   agent.debug_console ",
+                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "agent.debug_console foo",
+                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: " agent.debug_console foo",
+                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "foo agent.debug_console bar",
+                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "foo agent.debug_console",
+                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "foo agent.debug_console ",
+                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "agent.devmode",
+                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "   agent.devmode ",
+                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "agent.devmode foo",
+                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: " agent.devmode foo",
+                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "foo agent.devmode bar",
+                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "foo agent.devmode",
+                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "foo agent.devmode ",
+                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "agent.devmode agent.debug_console",
+                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "agent.devmode agent.debug_console agent.hotplug_timeout=100 agent.unified_cgroup_hierarchy=a",
+                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: time::Duration::from_secs(100),
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "agent.devmode agent.debug_console agent.hotplug_timeout=0 agent.unified_cgroup_hierarchy=11",
+                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: true,
             },
             TestData {
                 contents: "agent.devmode agent.debug_console agent.container_pipe_size=2097152 agent.unified_cgroup_hierarchy=false",
+                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: 2097152,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "agent.devmode agent.debug_console agent.container_pipe_size=100 agent.unified_cgroup_hierarchy=true",
+                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: 100,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: true,
             },
             TestData {
                 contents: "agent.devmode agent.debug_console agent.container_pipe_size=0 agent.unified_cgroup_hierarchy=0",
+                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "agent.devmode agent.debug_console agent.container_pip_siz=100 agent.unified_cgroup_hierarchy=1",
+                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: true,
             },
+            TestData {
+                contents: "",
+                env_vars: Vec::new(),
+                debug_console: false,
+                dev_mode: false,
+                log_level: DEFAULT_LOG_LEVEL,
+                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
+                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
+                unified_cgroup_hierarchy: false,
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_SERVER_ADDR=foo"],
+                debug_console: false,
+                dev_mode: false,
+                log_level: DEFAULT_LOG_LEVEL,
+                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
+                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: "foo",
+                unified_cgroup_hierarchy: false,
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_SERVER_ADDR=="],
+                debug_console: false,
+                dev_mode: false,
+                log_level: DEFAULT_LOG_LEVEL,
+                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
+                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: "=",
+                unified_cgroup_hierarchy: false,
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_SERVER_ADDR==foo"],
+                debug_console: false,
+                dev_mode: false,
+                log_level: DEFAULT_LOG_LEVEL,
+                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
+                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: "=foo",
+                unified_cgroup_hierarchy: false,
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_SERVER_ADDR=foo=bar=baz="],
+                debug_console: false,
+                dev_mode: false,
+                log_level: DEFAULT_LOG_LEVEL,
+                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
+                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: "foo=bar=baz=",
+                unified_cgroup_hierarchy: false,
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_SERVER_ADDR=unix:///tmp/foo.socket"],
+                debug_console: false,
+                dev_mode: false,
+                log_level: DEFAULT_LOG_LEVEL,
+                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
+                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: "unix:///tmp/foo.socket",
+                unified_cgroup_hierarchy: false,
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_SERVER_ADDR=unix://@/tmp/foo.socket"],
+                debug_console: false,
+                dev_mode: false,
+                log_level: DEFAULT_LOG_LEVEL,
+                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
+                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: "unix://@/tmp/foo.socket",
+                unified_cgroup_hierarchy: false,
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_LOG_LEVEL="],
+                debug_console: false,
+                dev_mode: false,
+                log_level: DEFAULT_LOG_LEVEL,
+                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
+                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
+                unified_cgroup_hierarchy: false,
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_LOG_LEVEL=invalid"],
+                debug_console: false,
+                dev_mode: false,
+                log_level: DEFAULT_LOG_LEVEL,
+                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
+                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
+                unified_cgroup_hierarchy: false,
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_LOG_LEVEL=debug"],
+                debug_console: false,
+                dev_mode: false,
+                log_level: slog::Level::Debug,
+                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
+                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
+                unified_cgroup_hierarchy: false,
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_LOG_LEVEL=debugger"],
+                debug_console: false,
+                dev_mode: false,
+                log_level: DEFAULT_LOG_LEVEL,
+                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
+                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
+                server_addr: TEST_SERVER_ADDR,
+                unified_cgroup_hierarchy: false,
+            },
         ];
 
         let dir = tempdir().expect("failed to create tmpdir");
@@ -625,7 +826,8 @@ mod tests {
         let result = config.parse_cmdline(&filename.to_owned());
         assert!(result.is_err());
 
-        // Now, test various combinations of file contents
+        // Now, test various combinations of file contents and environment
+        // variables.
         for (i, d) in tests.iter().enumerate() {
             let msg = format!("test[{}]: {:?}", i, d);
 
@@ -634,10 +836,23 @@ mod tests {
             let filename = file_path.to_str().expect("failed to create filename");
 
             let mut file =
-                File::create(filename).expect(&format!("{}: failed to create file", msg));
+                File::create(filename).unwrap_or_else(|_| panic!("{}: failed to create file", msg));
 
             file.write_all(d.contents.as_bytes())
-                .expect(&format!("{}: failed to write file contents", msg));
+                .unwrap_or_else(|_| panic!("{}: failed to write file contents", msg));
+
+            let mut vars_to_unset = Vec::new();
+
+            for v in &d.env_vars {
+                let fields: Vec<&str> = v.split('=').collect();
+
+                let name = fields[0];
+                let value = fields[1..].join("=");
+
+                env::set_var(name, value);
+
+                vars_to_unset.push(name);
+            }
 
             let mut config = agentConfig::new();
             assert_eq!(config.debug_console, false, "{}", msg);
@@ -650,6 +865,7 @@ mod tests {
                 msg
             );
             assert_eq!(config.container_pipe_size, 0, "{}", msg);
+            assert_eq!(config.server_addr, TEST_SERVER_ADDR, "{}", msg);
 
             let result = config.parse_cmdline(filename);
             assert!(result.is_ok(), "{}", msg);
@@ -664,6 +880,11 @@ mod tests {
             assert_eq!(d.log_level, config.log_level, "{}", msg);
             assert_eq!(d.hotplug_timeout, config.hotplug_timeout, "{}", msg);
             assert_eq!(d.container_pipe_size, config.container_pipe_size, "{}", msg);
+            assert_eq!(d.server_addr, config.server_addr, "{}", msg);
+
+            for v in vars_to_unset {
+                env::remove_var(v);
+            }
         }
     }
 
@@ -737,7 +958,7 @@ mod tests {
 
             let msg = format!("{}: result: {:?}", msg, result);
 
-            assert_result!(d.result, result, format!("{}", msg));
+            assert_result!(d.result, result, msg);
         }
     }
 
@@ -831,7 +1052,7 @@ mod tests {
 
             let msg = format!("{}: result: {:?}", msg, result);
 
-            assert_result!(d.result, result, format!("{}", msg));
+            assert_result!(d.result, result, msg);
         }
     }
 
@@ -901,7 +1122,7 @@ mod tests {
 
             let msg = format!("{}: result: {:?}", msg, result);
 
-            assert_result!(d.result, result, format!("{}", msg));
+            assert_result!(d.result, result, msg);
         }
     }
 
@@ -975,7 +1196,7 @@ mod tests {
 
             let msg = format!("{}: result: {:?}", msg, result);
 
-            assert_result!(d.result, result, format!("{}", msg));
+            assert_result!(d.result, result, msg);
         }
     }
 }

src/agent/src/device.rs

@@ -38,8 +38,8 @@ struct DevIndex(HashMap<String, DevIndexEntry>);
 // DeviceHandler is the type of callback to be defined to handle every type of device driver.
 type DeviceHandler = fn(&Device, &mut Spec, &Arc<Mutex<Sandbox>>, &DevIndex) -> Result<()>;
 
-// DeviceHandlerList lists the supported drivers.
-#[cfg_attr(rustfmt, rustfmt_skip)]
+// DEVICEHANDLERLIST lists the supported drivers.
+#[rustfmt::skip]
 lazy_static! {
     static ref DEVICEHANDLERLIST: HashMap<&'static str, DeviceHandler> = {
         let mut m: HashMap<&'static str, DeviceHandler> = HashMap::new();
@@ -65,7 +65,7 @@ pub fn online_device(path: &str) -> Result<()> {
 // Here, bridgeAddr is the address at which the bridge is attached on the root bus,
 // while deviceAddr is the address at which the device is attached on the bridge.
 fn get_pci_device_address(pci_id: &str) -> Result<String> {
-    let tokens: Vec<&str> = pci_id.split("/").collect();
+    let tokens: Vec<&str> = pci_id.split('/').collect();
 
     if tokens.len() != 2 {
         return Err(anyhow!(
@@ -165,7 +165,7 @@ pub fn get_pci_device_name(sandbox: &Arc<Mutex<Sandbox>>, pci_id: &str) -> Resul
 
 /// Scan SCSI bus for the given SCSI address(SCSI-Id and LUN)
 fn scan_scsi_bus(scsi_addr: &str) -> Result<()> {
-    let tokens: Vec<&str> = scsi_addr.split(":").collect();
+    let tokens: Vec<&str> = scsi_addr.split(':').collect();
     if tokens.len() != 2 {
         return Err(anyhow!(
             "Unexpected format for SCSI Address: {}, expect SCSIID:LUA",
@@ -336,11 +336,11 @@ impl DevIndex {
     fn new(spec: &Spec) -> DevIndex {
         let mut map = HashMap::new();
 
-        for linux in spec.linux.as_ref() {
+        if let Some(linux) = spec.linux.as_ref() {
             for (i, d) in linux.devices.iter().enumerate() {
                 let mut residx = Vec::new();
 
-                for linuxres in linux.resources.as_ref() {
+                if let Some(linuxres) = linux.resources.as_ref() {
                     for (j, r) in linuxres.devices.iter().enumerate() {
                         if r.r#type == d.r#type
                             && r.major == Some(d.major)

src/agent/src/linux_abi.rs

@@ -5,8 +5,10 @@
 
 /// Linux ABI related constants.
 
-pub const SYSFS_DIR: &str = "/sys";
+#[cfg(target_arch = "aarch64")]
+use std::fs;
 
+pub const SYSFS_DIR: &str = "/sys";
 pub const SYSFS_PCI_BUS_PREFIX: &str = "/sys/bus/pci/devices";
 pub const SYSFS_PCI_BUS_RESCAN_FILE: &str = "/sys/bus/pci/rescan";
 #[cfg(any(
@@ -15,9 +17,46 @@ pub const SYSFS_PCI_BUS_RESCAN_FILE: &str = "/sys/bus/pci/rescan";
     target_arch = "x86_64",
     target_arch = "x86"
 ))]
-pub const PCI_ROOT_BUS_PATH: &str = "/devices/pci0000:00";
+pub fn create_pci_root_bus_path() -> String {
+    String::from("/devices/pci0000:00")
+}
+
 #[cfg(target_arch = "aarch64")]
-pub const PCI_ROOT_BUS_PATH: &str = "/devices/platform/4010000000.pcie/pci0000:00";
+pub fn create_pci_root_bus_path() -> String {
+    let ret = String::from("/devices/platform/4010000000.pcie/pci0000:00");
+
+    let mut sysfs_dir = String::from(SYSFS_DIR);
+    let mut start_root_bus_path = String::from("/devices/platform/");
+    let end_root_bus_path = String::from("/pci0000:00");
+
+    sysfs_dir.push_str(&start_root_bus_path);
+    let entries = match fs::read_dir(sysfs_dir) {
+        Ok(e) => e,
+        Err(_) => return ret,
+    };
+    for entry in entries {
+        let pathname = match entry {
+            Ok(p) => p.path(),
+            Err(_) => return ret,
+        };
+        let dir_name = match pathname.file_name() {
+            Some(p) => p.to_str(),
+            None => return ret,
+        };
+        let dir_name = match dir_name {
+            Some(p) => p,
+            None => return ret,
+        };
+        let dir_name = String::from(dir_name);
+        if dir_name.ends_with(".pcie") {
+            start_root_bus_path.push_str(&dir_name);
+            start_root_bus_path.push_str(&end_root_bus_path);
+            return start_root_bus_path;
+        }
+    }
+
+    ret
+}
 
 pub const SYSFS_CPU_ONLINE_PATH: &str = "/sys/devices/system/cpu";
 

src/agent/src/main.rs

@@ -198,6 +198,13 @@ fn main() -> Result<()> {
     // which is required to satisfy the the lifetime constraints of the auto-generated gRPC code.
     let _guard = slog_scope::set_global_logger(logger.new(o!("subsystem" => "rpc")));
 
+    let mut _log_guard: Result<(), log::SetLoggerError> = Ok(());
+
+    if config.log_level == slog::Level::Trace {
+        // Redirect ttrpc log calls to slog iff full debug requested
+        _log_guard = Ok(slog_stdlog::init().map_err(|e| e)?);
+    }
+
     start_sandbox(&logger, &config, init_mode)?;
 
     let _ = log_handle.join();
@@ -246,8 +253,8 @@ fn start_sandbox(logger: &Logger, config: &agentConfig, init_mode: bool) -> Resu
     let (tx, rx) = mpsc::channel::<i32>();
     sandbox.lock().unwrap().sender = Some(tx);
 
-    //vsock:///dev/vsock, port
-    let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str());
+    // vsock:///dev/vsock, port
+    let mut server = rpc::start(sandbox, config.server_addr.as_str());
 
     let _ = server.start().unwrap();
 
@@ -272,8 +279,6 @@ fn setup_signal_handler(logger: &Logger, sandbox: Arc<Mutex<Sandbox>>) -> Result
 
     let signals = Signals::new(&[SIGCHLD])?;
 
-    let s = sandbox.clone();
-
     thread::spawn(move || {
         'outer: for sig in signals.forever() {
             info!(logger, "received signal"; "signal" => sig);
@@ -301,15 +306,16 @@ fn setup_signal_handler(logger: &Logger, sandbox: Arc<Mutex<Sandbox>>) -> Result
                         continue 'outer;
                     }
                 };
+                info!(logger, "wait_status"; "wait_status result" => format!("{:?}", wait_status));
 
                 let pid = wait_status.pid();
-                if pid.is_some() {
-                    let raw_pid = pid.unwrap().as_raw();
+                if let Some(pid) = pid {
+                    let raw_pid = pid.as_raw();
                     let child_pid = format!("{}", raw_pid);
 
                     let logger = logger.new(o!("child-pid" => child_pid));
 
-                    let mut sandbox = s.lock().unwrap();
+                    let mut sandbox = sandbox.lock().unwrap();
                     let process = sandbox.find_process(raw_pid);
                     if process.is_none() {
                         info!(logger, "child exited unexpectedly");
@@ -337,6 +343,13 @@ fn setup_signal_handler(logger: &Logger, sandbox: Arc<Mutex<Sandbox>>) -> Result
 
                     p.exit_code = ret;
                     let _ = unistd::close(pipe_write);
+
+                    if let Some(ref poller) = p.epoller {
+                        info!(logger, "close epoller");
+                        // close the socket file to notify readStdio to close terminal specifically
+                        // in case this process's terminal has been inherited by its children.
+                        poller.close_wfd()
+                    }
                 }
             }
         }
@@ -366,7 +379,8 @@ fn init_agent_as_init(logger: &Logger, unified_cgroup_hierarchy: bool) -> Result
 
     env::set_var("PATH", "/bin:/sbin/:/usr/bin/:/usr/sbin/");
 
-    let contents = std::fs::read_to_string("/etc/hostname").unwrap_or(String::from("localhost"));
+    let contents =
+        std::fs::read_to_string("/etc/hostname").unwrap_or_else(|_| String::from("localhost"));
     let contents_array: Vec<&str> = contents.split(' ').collect();
     let hostname = contents_array[0].trim();
 
@@ -481,8 +495,8 @@ where
 
     // write and return
     match writer.write_all(&buf[..buf_len]) {
-        Ok(_) => return Ok(buf_len as u64),
-        Err(err) => return Err(err),
+        Ok(_) => Ok(buf_len as u64),
+        Err(err) => Err(err),
     }
 }
 

src/agent/src/metrics.rs

@@ -8,7 +8,6 @@ extern crate procfs;
 use prometheus::{Encoder, Gauge, GaugeVec, IntCounter, TextEncoder};
 
 use anyhow::Result;
-use protocols;
 
 const NAMESPACE_KATA_AGENT: &str = "kata_agent";
 const NAMESPACE_KATA_GUEST: &str = "kata_guest";
@@ -85,17 +84,15 @@ pub fn get_metrics(_: &protocols::agent::GetMetricsRequest) -> Result<String> {
     let encoder = TextEncoder::new();
     encoder.encode(&metric_families, &mut buffer).unwrap();
 
-    Ok(String::from_utf8(buffer.clone()).unwrap())
+    Ok(String::from_utf8(buffer).unwrap())
 }
 
 fn update_agent_metrics() {
     let me = procfs::process::Process::myself();
-    match me {
-        Err(err) => {
-            error!(sl!(), "failed to create process instance: {:?}", err);
-            return;
-        }
-        Ok(_) => {}
+
+    if let Err(err) = me {
+        error!(sl!(), "failed to create process instance: {:?}", err);
+        return;
     }
 
     let me = me.unwrap();

src/agent/src/mount.rs

@@ -39,7 +39,7 @@ pub const DRIVERLOCALTYPE: &str = "local";
 
 pub const TYPEROOTFS: &str = "rootfs";
 
-#[cfg_attr(rustfmt, rustfmt_skip)]
+#[rustfmt::skip]
 lazy_static! {
     pub static ref FLAGS: HashMap<&'static str, (bool, MsFlags)> = {
         let mut m = HashMap::new();
@@ -88,7 +88,7 @@ pub struct INIT_MOUNT {
     options: Vec<&'static str>,
 }
 
-#[cfg_attr(rustfmt, rustfmt_skip)]
+#[rustfmt::skip]
 lazy_static!{
     static ref CGROUPS: HashMap<&'static str, &'static str> = {
         let mut m = HashMap::new();
@@ -109,7 +109,7 @@ lazy_static!{
     };
 }
 
-#[cfg_attr(rustfmt, rustfmt_skip)]
+#[rustfmt::skip]
 lazy_static! {
     pub static ref INIT_ROOTFS_MOUNTS: Vec<INIT_MOUNT> = vec![
         INIT_MOUNT{fstype: "proc", src: "proc", dest: "/proc", options: vec!["nosuid", "nodev", "noexec"]},
@@ -126,7 +126,7 @@ lazy_static! {
 type StorageHandler = fn(&Logger, &Storage, Arc<Mutex<Sandbox>>) -> Result<String>;
 
 // STORAGEHANDLERLIST lists the supported drivers.
-#[cfg_attr(rustfmt, rustfmt_skip)]
+#[rustfmt::skip]
 lazy_static! {
     pub static ref STORAGEHANDLERLIST: HashMap<&'static str, StorageHandler> = {
     	let mut m = HashMap::new();
@@ -173,9 +173,9 @@ impl<'a> BareMount<'a> {
         BareMount {
             source: s,
             destination: d,
-            fs_type: fs_type,
-            flags: flags,
-            options: options,
+            fs_type,
+            flags,
+            options,
             logger: logger.new(o!("subsystem" => "baremount")),
         }
     }
@@ -190,11 +190,11 @@ impl<'a> BareMount<'a> {
         let cstr_dest: CString;
         let cstr_fs_type: CString;
 
-        if self.source.len() == 0 {
+        if self.source.is_empty() {
             return Err(anyhow!("need mount source"));
         }
 
-        if self.destination.len() == 0 {
+        if self.destination.is_empty() {
             return Err(anyhow!("need mount destination"));
         }
 
@@ -204,14 +204,14 @@ impl<'a> BareMount<'a> {
         cstr_dest = CString::new(self.destination)?;
         dest = cstr_dest.as_ptr();
 
-        if self.fs_type.len() == 0 {
+        if self.fs_type.is_empty() {
             return Err(anyhow!("need mount FS type"));
         }
 
         cstr_fs_type = CString::new(self.fs_type)?;
         fs_type = cstr_fs_type.as_ptr();
 
-        if self.options.len() > 0 {
+        if !self.options.is_empty() {
             cstr_options = CString::new(self.options)?;
             options = cstr_options.as_ptr() as *const c_void;
         }
@@ -243,8 +243,7 @@ fn ephemeral_storage_handler(
     storage: &Storage,
     sandbox: Arc<Mutex<Sandbox>>,
 ) -> Result<String> {
-    let s = sandbox.clone();
-    let mut sb = s.lock().unwrap();
+    let mut sb = sandbox.lock().unwrap();
     let new_storage = sb.set_sandbox_storage(&storage.mount_point);
 
     if !new_storage {
@@ -262,8 +261,7 @@ fn local_storage_handler(
     storage: &Storage,
     sandbox: Arc<Mutex<Sandbox>>,
 ) -> Result<String> {
-    let s = sandbox.clone();
-    let mut sb = s.lock().unwrap();
+    let mut sb = sandbox.lock().unwrap();
     let new_storage = sb.set_sandbox_storage(&storage.mount_point);
 
     if !new_storage {
@@ -279,8 +277,7 @@ fn local_storage_handler(
 
     let opts = parse_options(opts_vec);
     let mode = opts.get("mode");
-    if mode.is_some() {
-        let mode = mode.unwrap();
+    if let Some(mode) = mode {
         let mut permission = fs::metadata(&storage.mount_point)?.permissions();
 
         let o_mode = u32::from_str_radix(mode, 8)?;
@@ -410,17 +407,17 @@ fn parse_mount_flags_and_options(options_vec: Vec<&str>) -> (MsFlags, String) {
     let mut options: String = "".to_string();
 
     for opt in options_vec {
-        if opt.len() != 0 {
+        if !opt.is_empty() {
             match FLAGS.get(opt) {
                 Some(x) => {
                     let (_, f) = *x;
-                    flags = flags | f;
+                    flags |= f;
                 }
                 None => {
-                    if options.len() > 0 {
+                    if !options.is_empty() {
                         options.push_str(format!(",{}", opt).as_str());
                     } else {
-                        options.push_str(format!("{}", opt).as_str());
+                        options.push_str(opt.to_string().as_str());
                     }
                 }
             };
@@ -458,7 +455,7 @@ pub fn add_storages(
         // Todo need to rollback the mounted storage if err met.
         let mount_point = handler(&logger, &storage, sandbox.clone())?;
 
-        if mount_point.len() > 0 {
+        if !mount_point.is_empty() {
             mount_list.push(mount_point);
         }
     }
@@ -570,10 +567,10 @@ pub fn get_cgroup_mounts(
     'outer: for (_, line) in reader.lines().enumerate() {
         let line = line?;
 
-        let fields: Vec<&str> = line.split("\t").collect();
+        let fields: Vec<&str> = line.split('\t').collect();
 
         // Ignore comment header
-        if fields[0].starts_with("#") {
+        if fields[0].starts_with('#') {
             continue;
         }
 
@@ -643,7 +640,7 @@ pub fn cgroups_mount(logger: &Logger, unified_cgroup_hierarchy: bool) -> Result<
     Ok(())
 }
 
-pub fn remove_mounts(mounts: &Vec<String>) -> Result<()> {
+pub fn remove_mounts(mounts: &[String]) -> Result<()> {
     for m in mounts.iter() {
         mount::umount(m.as_str()).context(format!("failed to umount {:?}", m))?;
     }
@@ -675,7 +672,7 @@ fn ensure_destination_exists(destination: &str, fs_type: &str) -> Result<()> {
 fn parse_options(option_list: Vec<String>) -> HashMap<String, String> {
     let mut options = HashMap::new();
     for opt in option_list.iter() {
-        let fields: Vec<&str> = opt.split("=").collect();
+        let fields: Vec<&str> = opt.split('=').collect();
         if fields.len() != 2 {
             continue;
         }
@@ -856,7 +853,7 @@ mod tests {
 
                     let msg = format!("{}: umount result: {:?}", msg, result);
 
-                    assert!(ret == 0, format!("{}", msg));
+                    assert!(ret == 0, msg);
                 };
 
                 continue;
@@ -914,7 +911,8 @@ mod tests {
             .expect("failed to create mount destination filename");
 
         for d in [test_dir_filename, mnt_src_filename, mnt_dest_filename].iter() {
-            std::fs::create_dir_all(d).expect(&format!("failed to create directory {}", d));
+            std::fs::create_dir_all(d)
+                .unwrap_or_else(|_| panic!("failed to create directory {}", d));
         }
 
         // Create an actual mount
@@ -1055,13 +1053,13 @@ mod tests {
 
             let filename = file_path
                 .to_str()
-                .expect(&format!("{}: failed to create filename", msg));
+                .unwrap_or_else(|| panic!("{}: failed to create filename", msg));
 
             let mut file =
-                File::create(filename).expect(&format!("{}: failed to create file", msg));
+                File::create(filename).unwrap_or_else(|_| panic!("{}: failed to create file", msg));
 
             file.write_all(d.contents.as_bytes())
-                .expect(&format!("{}: failed to write file contents", msg));
+                .unwrap_or_else(|_| panic!("{}: failed to write file contents", msg));
 
             let result = get_mount_fs_type_from_file(filename, d.mount_point);
 
@@ -1217,10 +1215,10 @@ mod tests {
                 .expect("failed to create cgroup file filename");
 
             let mut file =
-                File::create(filename).expect(&format!("{}: failed to create file", msg));
+                File::create(filename).unwrap_or_else(|_| panic!("{}: failed to create file", msg));
 
             file.write_all(d.contents.as_bytes())
-                .expect(&format!("{}: failed to write file contents", msg));
+                .unwrap_or_else(|_| panic!("{}: failed to write file contents", msg));
 
             let result = get_cgroup_mounts(&logger, filename, false);
             let msg = format!("{}: result: {:?}", msg, result);

src/agent/src/namespace.rs

@@ -51,12 +51,12 @@ impl Namespace {
         }
     }
 
-    pub fn as_ipc(mut self) -> Self {
+    pub fn get_ipc(mut self) -> Self {
         self.ns_type = NamespaceType::IPC;
         self
     }
 
-    pub fn as_uts(mut self, hostname: &str) -> Self {
+    pub fn get_uts(mut self, hostname: &str) -> Self {
         self.ns_type = NamespaceType::UTS;
         if hostname != "" {
             self.hostname = Some(String::from(hostname));
@@ -64,7 +64,7 @@ impl Namespace {
         self
     }
 
-    pub fn as_pid(mut self) -> Self {
+    pub fn get_pid(mut self) -> Self {
         self.ns_type = NamespaceType::PID;
         self
     }
@@ -99,7 +99,7 @@ impl Namespace {
             File::open(Path::new(&origin_ns_path))?;
 
             // Create a new netns on the current thread.
-            let cf = ns_type.get_flags().clone();
+            let cf = ns_type.get_flags();
 
             unshare(cf)?;
 
@@ -112,12 +112,9 @@ impl Namespace {
 
             let mut flags = MsFlags::empty();
 
-            match FLAGS.get("rbind") {
-                Some(x) => {
-                    let (_, f) = *x;
-                    flags = flags | f;
-                }
-                None => (),
+            if let Some(x) = FLAGS.get("rbind") {
+                let (_, f) = *x;
+                flags |= f;
             };
 
             let bare_mount = BareMount::new(source, destination, "none", flags, "", &logger);
@@ -196,30 +193,30 @@ mod tests {
         let tmpdir = Builder::new().prefix("ipc").tempdir().unwrap();
 
         let ns_ipc = Namespace::new(&logger)
-            .as_ipc()
+            .get_ipc()
             .set_root_dir(tmpdir.path().to_str().unwrap())
             .setup();
 
         assert!(ns_ipc.is_ok());
-        assert!(remove_mounts(&vec![ns_ipc.unwrap().path]).is_ok());
+        assert!(remove_mounts(&[ns_ipc.unwrap().path]).is_ok());
 
         let logger = slog::Logger::root(slog::Discard, o!());
         let tmpdir = Builder::new().prefix("uts").tempdir().unwrap();
 
         let ns_uts = Namespace::new(&logger)
-            .as_uts("test_hostname")
+            .get_uts("test_hostname")
             .set_root_dir(tmpdir.path().to_str().unwrap())
             .setup();
 
         assert!(ns_uts.is_ok());
-        assert!(remove_mounts(&vec![ns_uts.unwrap().path]).is_ok());
+        assert!(remove_mounts(&[ns_uts.unwrap().path]).is_ok());
 
         // Check it cannot persist pid namespaces.
         let logger = slog::Logger::root(slog::Discard, o!());
         let tmpdir = Builder::new().prefix("pid").tempdir().unwrap();
 
         let ns_pid = Namespace::new(&logger)
-            .as_pid()
+            .get_pid()
             .set_root_dir(tmpdir.path().to_str().unwrap())
             .setup();
 

src/agent/src/network.rs

@@ -48,7 +48,7 @@ pub fn setup_guest_dns(logger: Logger, dns_list: Vec<String>) -> Result<()> {
 fn do_setup_guest_dns(logger: Logger, dns_list: Vec<String>, src: &str, dst: &str) -> Result<()> {
     let logger = logger.new(o!( "subsystem" => "network"));
 
-    if dns_list.len() == 0 {
+    if dns_list.is_empty() {
         info!(
             logger,
             "Did not set sandbox DNS as DNS not received as part of request."
@@ -117,12 +117,12 @@ mod tests {
         ];
 
         // write to /run/kata-containers/sandbox/resolv.conf
-        let mut src_file =
-            File::create(src_filename).expect(&format!("failed to create file {:?}", src_filename));
+        let mut src_file = File::create(src_filename)
+            .unwrap_or_else(|_| panic!("failed to create file {:?}", src_filename));
         let content = dns.join("\n");
         src_file
             .write_all(content.as_bytes())
-            .expect(&format!("failed to write file contents"));
+            .expect("failed to write file contents");
 
         // call do_setup_guest_dns
         let result = do_setup_guest_dns(logger, dns.clone(), src_filename, dst_filename);

src/agent/src/random.rs

@@ -4,11 +4,11 @@
 //
 
 use anyhow::Result;
-use libc;
 use nix::errno::Errno;
 use nix::fcntl::{self, OFlag};
 use nix::sys::stat::Mode;
 use std::fs;
+use std::os::unix::io::{AsRawFd, FromRawFd};
 
 pub const RNGDEV: &str = "/dev/random";
 pub const RNDADDTOENTCNT: libc::c_int = 0x40045201;
@@ -24,18 +24,22 @@ pub fn reseed_rng(data: &[u8]) -> Result<()> {
     let len = data.len() as libc::c_long;
     fs::write(RNGDEV, data)?;
 
-    let fd = fcntl::open(RNGDEV, OFlag::O_RDWR, Mode::from_bits_truncate(0o022))?;
+    let f = {
+        let fd = fcntl::open(RNGDEV, OFlag::O_RDWR, Mode::from_bits_truncate(0o022))?;
+        // Wrap fd with `File` to properly close descriptor on exit
+        unsafe { fs::File::from_raw_fd(fd) }
+    };
 
     let ret = unsafe {
         libc::ioctl(
-            fd,
+            f.as_raw_fd(),
             RNDADDTOENTCNT as IoctlRequestType,
             &len as *const libc::c_long,
         )
     };
     let _ = Errno::result(ret).map(drop)?;
 
-    let ret = unsafe { libc::ioctl(fd, RNDRESEEDRNG as IoctlRequestType, 0) };
+    let ret = unsafe { libc::ioctl(f.as_raw_fd(), RNDRESEEDRNG as IoctlRequestType, 0) };
     let _ = Errno::result(ret).map(drop)?;
 
     Ok(())

src/agent/src/rpc.rs

@@ -6,7 +6,7 @@
 use std::path::Path;
 use std::sync::mpsc::channel;
 use std::sync::{Arc, Mutex};
-use ttrpc;
+use ttrpc::{self, error::get_rpc_status as ttrpc_error};
 
 use anyhow::{anyhow, Context, Result};
 use oci::{LinuxNamespace, Root, Spec};
@@ -21,10 +21,10 @@ use protocols::health::{
     HealthCheckResponse, HealthCheckResponse_ServingStatus, VersionCheckResponse,
 };
 use protocols::types::Interface;
-use rustjail;
 use rustjail::cgroups::notifier;
 use rustjail::container::{BaseContainer, Container, LinuxContainer};
 use rustjail::process::Process;
+use rustjail::reaper;
 use rustjail::specconv::CreateOpts;
 
 use nix::errno::Errno;
@@ -47,7 +47,6 @@ use crate::AGENT_CONFIG;
 use netlink::{RtnlHandle, NETLINK_ROUTE};
 
 use libc::{self, c_ushort, pid_t, winsize, TIOCSWINSZ};
-use serde_json;
 use std::convert::TryFrom;
 use std::fs;
 use std::os::unix::io::RawFd;
@@ -152,14 +151,13 @@ impl agentService {
 
         let pipe_size = AGENT_CONFIG.read().unwrap().container_pipe_size;
         let p = if oci.process.is_some() {
-            let tp = Process::new(
+            Process::new(
                 &sl!(),
                 &oci.process.as_ref().unwrap(),
                 cid.as_str(),
                 true,
                 pipe_size,
-            )?;
-            tp
+            )?
         } else {
             info!(sl!(), "no process configurations!");
             return Err(anyhow!(nix::Error::from_errno(nix::errno::Errno::EINVAL)));
@@ -175,7 +173,7 @@ impl agentService {
     }
 
     fn do_start_container(&self, req: protocols::agent::StartContainerRequest) -> Result<()> {
-        let cid = req.container_id.clone();
+        let cid = req.container_id;
 
         let sandbox = self.sandbox.clone();
         let mut s = sandbox.lock().unwrap();
@@ -183,7 +181,7 @@ impl agentService {
 
         let ctr = s
             .get_container(&cid)
-            .ok_or(anyhow!("Invalid container id"))?;
+            .ok_or_else(|| anyhow!("Invalid container id"))?;
 
         ctr.exec()?;
 
@@ -192,10 +190,14 @@ impl agentService {
             let cg_path = ctr.cgroup_manager.as_ref().unwrap().get_cg_path("memory");
             if cg_path.is_some() {
                 let rx = notifier::notify_oom(cid.as_str(), cg_path.unwrap())?;
-                s.run_oom_event_monitor(rx, cid);
+                s.run_oom_event_monitor(rx, cid.clone());
             }
         }
 
+        // set epoller
+        let p = find_process(&mut s, cid.as_str(), "", true)?;
+        p.create_epoller()?;
+
         Ok(())
     }
 
@@ -206,9 +208,7 @@ impl agentService {
         let mut remove_container_resources = |sandbox: &mut Sandbox| -> Result<()> {
             // Find the sandbox storage used by this container
             let mounts = sandbox.container_mounts.get(&cid);
-            if mounts.is_some() {
-                let mounts = mounts.unwrap();
-
+            if let Some(mounts) = mounts {
                 remove_mounts(&mounts)?;
 
                 for m in mounts.iter() {
@@ -232,7 +232,7 @@ impl agentService {
             let mut sandbox = s.lock().unwrap();
             let ctr = sandbox
                 .get_container(&cid)
-                .ok_or(anyhow!("Invalid container id"))?;
+                .ok_or_else(|| anyhow!("Invalid container id"))?;
 
             ctr.destroy()?;
 
@@ -250,11 +250,11 @@ impl agentService {
             let mut sandbox = s.lock().unwrap();
             let _ctr = sandbox
                 .get_container(&cid2)
-                .ok_or(anyhow!("Invalid container id"))
-                .and_then(|ctr| {
+                .ok_or_else(|| anyhow!("Invalid container id"))
+                .map(|ctr| {
                     ctr.destroy().unwrap();
                     tx.send(1).unwrap();
-                    Ok(ctr)
+                    ctr
                 });
         });
 
@@ -277,7 +277,7 @@ impl agentService {
         let cid = req.container_id.clone();
         let exec_id = req.exec_id.clone();
 
-        info!(sl!(), "cid: {} eid: {}", cid.clone(), exec_id.clone());
+        info!(sl!(), "do_exec_process cid: {} eid: {}", cid, exec_id);
 
         let s = self.sandbox.clone();
         let mut sandbox = s.lock().unwrap();
@@ -294,10 +294,14 @@ impl agentService {
 
         let ctr = sandbox
             .get_container(&cid)
-            .ok_or(anyhow!("Invalid container id"))?;
+            .ok_or_else(|| anyhow!("Invalid container id"))?;
 
         ctr.run(p)?;
 
+        // set epoller
+        let p = find_process(&mut sandbox, cid.as_str(), exec_id.as_str(), false)?;
+        p.create_epoller()?;
+
         Ok(())
     }
 
@@ -340,7 +344,7 @@ impl agentService {
         req: protocols::agent::WaitProcessRequest,
     ) -> Result<protocols::agent::WaitProcessResponse> {
         let cid = req.container_id.clone();
-        let eid = req.exec_id.clone();
+        let eid = req.exec_id;
         let s = self.sandbox.clone();
         let mut resp = WaitProcessResponse::new();
         let pid: pid_t;
@@ -376,7 +380,7 @@ impl agentService {
         let mut sandbox = s.lock().unwrap();
         let ctr = sandbox
             .get_container(&cid)
-            .ok_or(anyhow!("Invalid container id"))?;
+            .ok_or_else(|| anyhow!("Invalid container id"))?;
 
         let mut p = match ctr.processes.get_mut(&pid) {
             Some(p) => p,
@@ -408,6 +412,8 @@ impl agentService {
             let _ = unistd::close(p.exit_pipe_r.unwrap());
         }
 
+        p.close_epoller();
+
         p.parent_stdin = None;
         p.parent_stdout = None;
         p.parent_stderr = None;
@@ -432,13 +438,6 @@ impl agentService {
         let cid = req.container_id.clone();
         let eid = req.exec_id.clone();
 
-        info!(
-            sl!(),
-            "write stdin";
-            "container-id" => cid.clone(),
-            "exec-id" => eid.clone()
-        );
-
         let s = self.sandbox.clone();
         let mut sandbox = s.lock().unwrap();
         let p = find_process(&mut sandbox, cid.as_str(), eid.as_str(), false)?;
@@ -482,6 +481,7 @@ impl agentService {
         let eid = req.exec_id;
 
         let mut fd: RawFd = -1;
+        let mut epoller: Option<reaper::Epoller> = None;
         {
             let s = self.sandbox.clone();
             let mut sandbox = s.lock().unwrap();
@@ -490,6 +490,7 @@ impl agentService {
 
             if p.term_master.is_some() {
                 fd = p.term_master.unwrap();
+                epoller = p.epoller.clone();
             } else if stdout {
                 if p.parent_stdout.is_some() {
                     fd = p.parent_stdout.unwrap();
@@ -499,6 +500,17 @@ impl agentService {
             }
         }
 
+        if let Some(epoller) = epoller {
+            // The process's epoller's poll() will return a file descriptor of the process's
+            // terminal or one end of its exited pipe. If it returns its terminal, it means
+            // there is data needed to be read out or it has been closed; if it returns the
+            // process's exited pipe, it means the process has exited and there is no data
+            // needed to be read out in its terminal, thus following read on it will read out
+            // "EOF" to terminate this process's io since the other end of this pipe has been
+            // closed in reap().
+            fd = epoller.poll()?;
+        }
+
         if fd == -1 {
             return Err(anyhow!(nix::Error::from_errno(nix::errno::Errno::EINVAL)));
         }
@@ -519,10 +531,7 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         req: protocols::agent::CreateContainerRequest,
     ) -> ttrpc::Result<Empty> {
         match self.do_create_container(req) {
-            Err(e) => Err(ttrpc::Error::RpcStatus(ttrpc::get_status(
-                ttrpc::Code::INTERNAL,
-                e.to_string(),
-            ))),
+            Err(e) => Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string())),
             Ok(_) => Ok(Empty::new()),
         }
     }
@@ -533,10 +542,7 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         req: protocols::agent::StartContainerRequest,
     ) -> ttrpc::Result<Empty> {
         match self.do_start_container(req) {
-            Err(e) => Err(ttrpc::Error::RpcStatus(ttrpc::get_status(
-                ttrpc::Code::INTERNAL,
-                e.to_string(),
-            ))),
+            Err(e) => Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string())),
             Ok(_) => Ok(Empty::new()),
         }
     }
@@ -547,10 +553,7 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         req: protocols::agent::RemoveContainerRequest,
     ) -> ttrpc::Result<Empty> {
         match self.do_remove_container(req) {
-            Err(e) => Err(ttrpc::Error::RpcStatus(ttrpc::get_status(
-                ttrpc::Code::INTERNAL,
-                e.to_string(),
-            ))),
+            Err(e) => Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string())),
             Ok(_) => Ok(Empty::new()),
         }
     }
@@ -561,10 +564,7 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         req: protocols::agent::ExecProcessRequest,
     ) -> ttrpc::Result<Empty> {
         match self.do_exec_process(req) {
-            Err(e) => Err(ttrpc::Error::RpcStatus(ttrpc::get_status(
-                ttrpc::Code::INTERNAL,
-                e.to_string(),
-            ))),
+            Err(e) => Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string())),
             Ok(_) => Ok(Empty::new()),
         }
     }
@@ -575,10 +575,7 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         req: protocols::agent::SignalProcessRequest,
     ) -> ttrpc::Result<Empty> {
         match self.do_signal_process(req) {
-            Err(e) => Err(ttrpc::Error::RpcStatus(ttrpc::get_status(
-                ttrpc::Code::INTERNAL,
-                e.to_string(),
-            ))),
+            Err(e) => Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string())),
             Ok(_) => Ok(Empty::new()),
         }
     }
@@ -588,9 +585,8 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         _ctx: &ttrpc::TtrpcContext,
         req: protocols::agent::WaitProcessRequest,
     ) -> ttrpc::Result<WaitProcessResponse> {
-        self.do_wait_process(req).map_err(|e| {
-            ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string()))
-        })
+        self.do_wait_process(req)
+            .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))
     }
 
     fn list_processes(
@@ -600,18 +596,18 @@ impl protocols::agent_ttrpc::AgentService for agentService {
     ) -> ttrpc::Result<ListProcessesResponse> {
         let cid = req.container_id.clone();
         let format = req.format.clone();
-        let mut args = req.args.clone().into_vec();
+        let mut args = req.args.into_vec();
         let mut resp = ListProcessesResponse::new();
 
         let s = Arc::clone(&self.sandbox);
         let mut sandbox = s.lock().unwrap();
 
-        let ctr = sandbox
-            .get_container(&cid)
-            .ok_or(ttrpc::Error::RpcStatus(ttrpc::get_status(
+        let ctr = sandbox.get_container(&cid).ok_or_else(|| {
+            ttrpc_error(
                 ttrpc::Code::INVALID_ARGUMENT,
                 "invalid container id".to_string(),
-            )))?;
+            )
+        })?;
 
         let pids = ctr.processes().unwrap();
 
@@ -622,15 +618,15 @@ impl protocols::agent_ttrpc::AgentService for agentService {
                 return Ok(resp);
             }
             _ => {
-                return Err(ttrpc::Error::RpcStatus(ttrpc::get_status(
+                return Err(ttrpc_error(
                     ttrpc::Code::INVALID_ARGUMENT,
                     "invalid format!".to_string(),
-                )));
+                ));
             }
         }
 
         // format "table"
-        if args.len() == 0 {
+        if args.is_empty() {
             // default argument
             args = vec!["-ef".to_string()];
         }
@@ -688,12 +684,12 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         let s = Arc::clone(&self.sandbox);
         let mut sandbox = s.lock().unwrap();
 
-        let ctr = sandbox
-            .get_container(&cid)
-            .ok_or(ttrpc::Error::RpcStatus(ttrpc::get_status(
+        let ctr = sandbox.get_container(&cid).ok_or_else(|| {
+            ttrpc_error(
                 ttrpc::Code::INVALID_ARGUMENT,
                 "invalid container id".to_string(),
-            )))?;
+            )
+        })?;
 
         let resp = Empty::new();
 
@@ -701,10 +697,7 @@ impl protocols::agent_ttrpc::AgentService for agentService {
             let ociRes = rustjail::resources_grpc_to_oci(&res.unwrap());
             match ctr.set(ociRes) {
                 Err(e) => {
-                    return Err(ttrpc::Error::RpcStatus(ttrpc::get_status(
-                        ttrpc::Code::INTERNAL,
-                        e.to_string(),
-                    )));
+                    return Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()));
                 }
 
                 Ok(_) => return Ok(resp),
@@ -719,20 +712,19 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         _ctx: &ttrpc::TtrpcContext,
         req: protocols::agent::StatsContainerRequest,
     ) -> ttrpc::Result<StatsContainerResponse> {
-        let cid = req.container_id.clone();
+        let cid = req.container_id;
         let s = Arc::clone(&self.sandbox);
         let mut sandbox = s.lock().unwrap();
 
-        let ctr = sandbox
-            .get_container(&cid)
-            .ok_or(ttrpc::Error::RpcStatus(ttrpc::get_status(
+        let ctr = sandbox.get_container(&cid).ok_or_else(|| {
+            ttrpc_error(
                 ttrpc::Code::INVALID_ARGUMENT,
                 "invalid container id".to_string(),
-            )))?;
+            )
+        })?;
 
-        ctr.stats().map_err(|e| {
-            ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string()))
-        })
+        ctr.stats()
+            .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))
     }
 
     fn pause_container(
@@ -744,17 +736,16 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         let s = Arc::clone(&self.sandbox);
         let mut sandbox = s.lock().unwrap();
 
-        let ctr = sandbox
-            .get_container(&cid)
-            .ok_or(ttrpc::Error::RpcStatus(ttrpc::get_status(
+        let ctr = sandbox.get_container(&cid).ok_or_else(|| {
+            ttrpc_error(
                 ttrpc::Code::INVALID_ARGUMENT,
                 "invalid container id".to_string(),
-            )))?;
-
-        ctr.pause().map_err(|e| {
-            ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string()))
+            )
         })?;
 
+        ctr.pause()
+            .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
+
         Ok(Empty::new())
     }
 
@@ -767,17 +758,16 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         let s = Arc::clone(&self.sandbox);
         let mut sandbox = s.lock().unwrap();
 
-        let ctr = sandbox
-            .get_container(&cid)
-            .ok_or(ttrpc::Error::RpcStatus(ttrpc::get_status(
+        let ctr = sandbox.get_container(&cid).ok_or_else(|| {
+            ttrpc_error(
                 ttrpc::Code::INVALID_ARGUMENT,
                 "invalid container id".to_string(),
-            )))?;
-
-        ctr.resume().map_err(|e| {
-            ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string()))
+            )
         })?;
 
+        ctr.resume()
+            .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
+
         Ok(Empty::new())
     }
 
@@ -786,9 +776,8 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         _ctx: &ttrpc::TtrpcContext,
         req: protocols::agent::WriteStreamRequest,
     ) -> ttrpc::Result<WriteStreamResponse> {
-        self.do_write_stream(req).map_err(|e| {
-            ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string()))
-        })
+        self.do_write_stream(req)
+            .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))
     }
 
     fn read_stdout(
@@ -796,9 +785,8 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         _ctx: &ttrpc::TtrpcContext,
         req: protocols::agent::ReadStreamRequest,
     ) -> ttrpc::Result<ReadStreamResponse> {
-        self.do_read_stream(req, true).map_err(|e| {
-            ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string()))
-        })
+        self.do_read_stream(req, true)
+            .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))
     }
 
     fn read_stderr(
@@ -806,9 +794,8 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         _ctx: &ttrpc::TtrpcContext,
         req: protocols::agent::ReadStreamRequest,
     ) -> ttrpc::Result<ReadStreamResponse> {
-        self.do_read_stream(req, false).map_err(|e| {
-            ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string()))
-        })
+        self.do_read_stream(req, false)
+            .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))
     }
 
     fn close_stdin(
@@ -817,15 +804,15 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         req: protocols::agent::CloseStdinRequest,
     ) -> ttrpc::Result<Empty> {
         let cid = req.container_id.clone();
-        let eid = req.exec_id.clone();
+        let eid = req.exec_id;
         let s = Arc::clone(&self.sandbox);
         let mut sandbox = s.lock().unwrap();
 
         let p = find_process(&mut sandbox, cid.as_str(), eid.as_str(), false).map_err(|e| {
-            ttrpc::Error::RpcStatus(ttrpc::get_status(
+            ttrpc_error(
                 ttrpc::Code::INVALID_ARGUMENT,
                 format!("invalid argument: {:?}", e),
-            ))
+            )
         })?;
 
         if p.term_master.is_some() {
@@ -838,6 +825,8 @@ impl protocols::agent_ttrpc::AgentService for agentService {
             p.parent_stdin = None;
         }
 
+        p.close_epoller();
+
         Ok(Empty::new())
     }
 
@@ -851,17 +840,14 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         let s = Arc::clone(&self.sandbox);
         let mut sandbox = s.lock().unwrap();
         let p = find_process(&mut sandbox, cid.as_str(), eid.as_str(), false).map_err(|e| {
-            ttrpc::Error::RpcStatus(ttrpc::get_status(
+            ttrpc_error(
                 ttrpc::Code::UNAVAILABLE,
                 format!("invalid argument: {:?}", e),
-            ))
+            )
         })?;
 
         if p.term_master.is_none() {
-            return Err(ttrpc::Error::RpcStatus(ttrpc::get_status(
-                ttrpc::Code::UNAVAILABLE,
-                "no tty".to_string(),
-            )));
+            return Err(ttrpc_error(ttrpc::Code::UNAVAILABLE, "no tty".to_string()));
         }
 
         let fd = p.term_master.unwrap();
@@ -874,12 +860,9 @@ impl protocols::agent_ttrpc::AgentService for agentService {
             };
 
             let err = libc::ioctl(fd, TIOCSWINSZ, &win);
-            Errno::result(err).map(drop).map_err(|e| {
-                ttrpc::Error::RpcStatus(ttrpc::get_status(
-                    ttrpc::Code::INTERNAL,
-                    format!("ioctl error: {:?}", e),
-                ))
-            })?;
+            Errno::result(err)
+                .map(drop)
+                .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, format!("ioctl error: {:?}", e)))?;
         }
 
         Ok(Empty::new())
@@ -891,13 +874,13 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         req: protocols::agent::UpdateInterfaceRequest,
     ) -> ttrpc::Result<Interface> {
         if req.interface.is_none() {
-            return Err(ttrpc::Error::RpcStatus(ttrpc::get_status(
+            return Err(ttrpc_error(
                 ttrpc::Code::INVALID_ARGUMENT,
-                format!("empty update interface request"),
-            )));
+                "empty update interface request".to_string(),
+            ));
         }
 
-        let interface = req.interface.clone();
+        let interface = req.interface;
         let s = Arc::clone(&self.sandbox);
         let mut sandbox = s.lock().unwrap();
 
@@ -910,10 +893,7 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         let iface = rtnl
             .update_interface(interface.as_ref().unwrap())
             .map_err(|e| {
-                ttrpc::Error::RpcStatus(ttrpc::get_status(
-                    ttrpc::Code::INTERNAL,
-                    format!("update interface: {:?}", e),
-                ))
+                ttrpc_error(ttrpc::Code::INTERNAL, format!("update interface: {:?}", e))
             })?;
 
         Ok(iface)
@@ -926,13 +906,13 @@ impl protocols::agent_ttrpc::AgentService for agentService {
     ) -> ttrpc::Result<Routes> {
         let mut routes = protocols::agent::Routes::new();
         if req.routes.is_none() {
-            return Err(ttrpc::Error::RpcStatus(ttrpc::get_status(
+            return Err(ttrpc_error(
                 ttrpc::Code::INVALID_ARGUMENT,
-                format!("empty update routes request"),
-            )));
+                "empty update routes request".to_string(),
+            ));
         }
 
-        let rs = req.routes.clone().unwrap().Routes.into_vec();
+        let rs = req.routes.unwrap().Routes.into_vec();
 
         let s = Arc::clone(&self.sandbox);
         let mut sandbox = s.lock().unwrap();
@@ -944,12 +924,9 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         let rtnl = sandbox.rtnl.as_mut().unwrap();
 
         // get current routes to return when error out
-        let crs = rtnl.list_routes().map_err(|e| {
-            ttrpc::Error::RpcStatus(ttrpc::get_status(
-                ttrpc::Code::INTERNAL,
-                format!("update routes: {:?}", e),
-            ))
-        })?;
+        let crs = rtnl
+            .list_routes()
+            .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, format!("update routes: {:?}", e)))?;
 
         let v = match rtnl.update_routes(rs.as_ref()) {
             Ok(value) => value,
@@ -975,12 +952,9 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         }
 
         let rtnl = sandbox.rtnl.as_mut().unwrap();
-        let v = rtnl.list_interfaces().map_err(|e| {
-            ttrpc::Error::RpcStatus(ttrpc::get_status(
-                ttrpc::Code::INTERNAL,
-                format!("list interface: {:?}", e),
-            ))
-        })?;
+        let v = rtnl
+            .list_interfaces()
+            .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, format!("list interface: {:?}", e)))?;
 
         interface.set_Interfaces(RepeatedField::from_vec(v));
 
@@ -1002,12 +976,9 @@ impl protocols::agent_ttrpc::AgentService for agentService {
 
         let rtnl = sandbox.rtnl.as_mut().unwrap();
 
-        let v = rtnl.list_routes().map_err(|e| {
-            ttrpc::Error::RpcStatus(ttrpc::get_status(
-                ttrpc::Code::INTERNAL,
-                format!("list routes: {:?}", e),
-            ))
-        })?;
+        let v = rtnl
+            .list_routes()
+            .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, format!("list routes: {:?}", e)))?;
 
         routes.set_Routes(RepeatedField::from_vec(v));
 
@@ -1055,19 +1026,17 @@ impl protocols::agent_ttrpc::AgentService for agentService {
                 });
             }
 
-            if req.sandbox_id.len() > 0 {
+            if !req.sandbox_id.is_empty() {
                 s.id = req.sandbox_id.clone();
             }
 
             for m in req.kernel_modules.iter() {
-                let _ = load_kernel_module(m).map_err(|e| {
-                    ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string()))
-                })?;
+                let _ = load_kernel_module(m)
+                    .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
             }
 
-            s.setup_shared_namespaces().map_err(|e| {
-                ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string()))
-            })?;
+            s.setup_shared_namespaces()
+                .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
         }
 
         match add_storages(sl!(), req.storages.to_vec(), self.sandbox.clone()) {
@@ -1076,30 +1045,20 @@ impl protocols::agent_ttrpc::AgentService for agentService {
                 let mut s = sandbox.lock().unwrap();
                 s.mounts = m
             }
-            Err(e) => {
-                return Err(ttrpc::Error::RpcStatus(ttrpc::get_status(
-                    ttrpc::Code::INTERNAL,
-                    e.to_string(),
-                )))
-            }
+            Err(e) => return Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string())),
         };
 
         match setup_guest_dns(sl!(), req.dns.to_vec()) {
             Ok(_) => {
                 let sandbox = self.sandbox.clone();
                 let mut s = sandbox.lock().unwrap();
-                let _ = req
+                let _dns = req
                     .dns
                     .to_vec()
                     .iter()
                     .map(|dns| s.network.set_dns(dns.to_string()));
             }
-            Err(e) => {
-                return Err(ttrpc::Error::RpcStatus(ttrpc::get_status(
-                    ttrpc::Code::INTERNAL,
-                    e.to_string(),
-                )))
-            }
+            Err(e) => return Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string())),
         };
 
         Ok(Empty::new())
@@ -1128,13 +1087,13 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         req: protocols::agent::AddARPNeighborsRequest,
     ) -> ttrpc::Result<Empty> {
         if req.neighbors.is_none() {
-            return Err(ttrpc::Error::RpcStatus(ttrpc::get_status(
+            return Err(ttrpc_error(
                 ttrpc::Code::INVALID_ARGUMENT,
-                format!("empty add arp neighbours request"),
-            )));
+                "empty add arp neighbours request".to_string(),
+            ));
         }
 
-        let neighs = req.neighbors.clone().unwrap().ARPNeighbors.into_vec();
+        let neighs = req.neighbors.unwrap().ARPNeighbors.into_vec();
 
         let s = Arc::clone(&self.sandbox);
         let mut sandbox = s.lock().unwrap();
@@ -1145,9 +1104,8 @@ impl protocols::agent_ttrpc::AgentService for agentService {
 
         let rtnl = sandbox.rtnl.as_mut().unwrap();
 
-        rtnl.add_arp_neighbors(neighs.as_ref()).map_err(|e| {
-            ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string()))
-        })?;
+        rtnl.add_arp_neighbors(neighs.as_ref())
+            .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
 
         Ok(Empty::new())
     }
@@ -1160,9 +1118,9 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         let s = Arc::clone(&self.sandbox);
         let sandbox = s.lock().unwrap();
 
-        sandbox.online_cpu_memory(&req).map_err(|e| {
-            ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string()))
-        })?;
+        sandbox
+            .online_cpu_memory(&req)
+            .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
 
         Ok(Empty::new())
     }
@@ -1172,9 +1130,8 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         _ctx: &ttrpc::TtrpcContext,
         req: protocols::agent::ReseedRandomDevRequest,
     ) -> ttrpc::Result<Empty> {
-        random::reseed_rng(req.data.as_slice()).map_err(|e| {
-            ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string()))
-        })?;
+        random::reseed_rng(req.data.as_slice())
+            .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
 
         Ok(Empty::new())
     }
@@ -1194,10 +1151,7 @@ impl protocols::agent_ttrpc::AgentService for agentService {
             }
             Err(e) => {
                 info!(sl!(), "fail to get memory info!");
-                return Err(ttrpc::Error::RpcStatus(ttrpc::get_status(
-                    ttrpc::Code::INTERNAL,
-                    e.to_string(),
-                )));
+                return Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()));
             }
         }
 
@@ -1213,9 +1167,8 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         _ctx: &ttrpc::TtrpcContext,
         req: protocols::agent::MemHotplugByProbeRequest,
     ) -> ttrpc::Result<Empty> {
-        do_mem_hotplug_by_probe(&req.memHotplugProbeAddr).map_err(|e| {
-            ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string()))
-        })?;
+        do_mem_hotplug_by_probe(&req.memHotplugProbeAddr)
+            .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
 
         Ok(Empty::new())
     }
@@ -1225,9 +1178,8 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         _ctx: &ttrpc::TtrpcContext,
         req: protocols::agent::SetGuestDateTimeRequest,
     ) -> ttrpc::Result<Empty> {
-        do_set_guest_date_time(req.Sec, req.Usec).map_err(|e| {
-            ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string()))
-        })?;
+        do_set_guest_date_time(req.Sec, req.Usec)
+            .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
 
         Ok(Empty::new())
     }
@@ -1237,9 +1189,7 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         _ctx: &ttrpc::TtrpcContext,
         req: protocols::agent::CopyFileRequest,
     ) -> ttrpc::Result<Empty> {
-        do_copy_file(&req).map_err(|e| {
-            ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string()))
-        })?;
+        do_copy_file(&req).map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
 
         Ok(Empty::new())
     }
@@ -1250,10 +1200,7 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         req: protocols::agent::GetMetricsRequest,
     ) -> ttrpc::Result<Metrics> {
         match get_metrics(&req) {
-            Err(e) => Err(ttrpc::Error::RpcStatus(ttrpc::get_status(
-                ttrpc::Code::INTERNAL,
-                e.to_string(),
-            ))),
+            Err(e) => Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string())),
             Ok(s) => {
                 let mut metrics = Metrics::new();
                 metrics.set_metrics(s);
@@ -1275,17 +1222,12 @@ impl protocols::agent_ttrpc::AgentService for agentService {
         drop(sandbox);
 
         match event_rx.recv() {
-            Err(err) => {
-                return Err(ttrpc::Error::RpcStatus(ttrpc::get_status(
-                    ttrpc::Code::INTERNAL,
-                    err.to_string(),
-                )))
-            }
+            Err(err) => Err(ttrpc_error(ttrpc::Code::INTERNAL, err.to_string())),
             Ok(container_id) => {
                 info!(sl!(), "get_oom_event return {}", &container_id);
                 let mut resp = OOMEvent::new();
                 resp.container_id = container_id;
-                return Ok(resp);
+                Ok(resp)
             }
         }
     }
@@ -1325,7 +1267,7 @@ fn get_memory_info(block_size: bool, hotplug: bool) -> Result<(u64, bool)> {
     if block_size {
         match fs::read_to_string(SYSFS_MEMORY_BLOCK_SIZE_PATH) {
             Ok(v) => {
-                if v.len() == 0 {
+                if v.is_empty() {
                     info!(sl!(), "string in empty???");
                     return Err(anyhow!("Invalid block size"));
                 }
@@ -1404,7 +1346,7 @@ fn read_stream(fd: RawFd, l: usize) -> Result<Vec<u8>> {
         }
         Err(e) => match e {
             nix::Error::Sys(errno) => match errno {
-                Errno::EAGAIN => v.resize(0, 0),
+                Errno::EAGAIN => v.clear(),
                 _ => return Err(anyhow!(nix::Error::Sys(errno))),
             },
             _ => return Err(anyhow!("read error")),
@@ -1422,13 +1364,13 @@ fn find_process<'a>(
 ) -> Result<&'a mut Process> {
     let ctr = sandbox
         .get_container(cid)
-        .ok_or(anyhow!("Invalid container id"))?;
+        .ok_or_else(|| anyhow!("Invalid container id"))?;
 
     if init || eid == "" {
         return ctr
             .processes
             .get_mut(&ctr.init_process_pid)
-            .ok_or(anyhow!("cannot find init process!"));
+            .ok_or_else(|| anyhow!("cannot find init process!"));
     }
 
     ctr.get_process(eid).map_err(|_| anyhow!("Invalid exec id"))
@@ -1478,7 +1420,7 @@ fn update_container_namespaces(
     let linux = spec
         .linux
         .as_mut()
-        .ok_or(anyhow!("Spec didn't container linux field"))?;
+        .ok_or_else(|| anyhow!("Spec didn't container linux field"))?;
 
     let namespaces = linux.namespaces.as_mut_slice();
     for namespace in namespaces.iter_mut() {
@@ -1546,7 +1488,7 @@ fn is_signal_handled(pid: pid_t, signum: u32) -> bool {
             }
         };
         if line.starts_with("SigCgt:") {
-            let mask_vec: Vec<&str> = line.split(":").collect();
+            let mask_vec: Vec<&str> = line.split(':').collect();
             if mask_vec.len() != 2 {
                 warn!(sl!(), "parse the SigCgt field failed\n");
                 return false;
@@ -1566,7 +1508,7 @@ fn is_signal_handled(pid: pid_t, signum: u32) -> bool {
     false
 }
 
-fn do_mem_hotplug_by_probe(addrs: &Vec<u64>) -> Result<()> {
+fn do_mem_hotplug_by_probe(addrs: &[u64]) -> Result<()> {
     for addr in addrs.iter() {
         fs::write(SYSFS_MEMORY_HOTPLUG_PROBE_PATH, format!("{:#X}", *addr))?;
     }
@@ -1579,8 +1521,12 @@ fn do_set_guest_date_time(sec: i64, usec: i64) -> Result<()> {
         tv_usec: usec,
     };
 
-    let ret =
-        unsafe { libc::settimeofday(&tv as *const libc::timeval, 0 as *const libc::timezone) };
+    let ret = unsafe {
+        libc::settimeofday(
+            &tv as *const libc::timeval,
+            std::ptr::null::<libc::timezone>(),
+        )
+    };
 
     Errno::result(ret).map(drop)?;
 
@@ -1596,8 +1542,8 @@ fn do_copy_file(req: &CopyFileRequest) -> Result<()> {
 
     let parent = path.parent();
 
-    let dir = if parent.is_some() {
-        parent.unwrap().to_path_buf()
+    let dir = if let Some(parent) = parent {
+        parent.to_path_buf()
     } else {
         PathBuf::from("/")
     };
@@ -1657,8 +1603,8 @@ fn setup_bundle(cid: &str, spec: &mut Spec) -> Result<PathBuf> {
     let spec_root = spec.root.as_ref().unwrap();
 
     let bundle_path = Path::new(CONTAINER_BASE).join(cid);
-    let config_path = bundle_path.clone().join("config.json");
-    let rootfs_path = bundle_path.clone().join("rootfs");
+    let config_path = bundle_path.join("config.json");
+    let rootfs_path = bundle_path.join("rootfs");
 
     fs::create_dir_all(&rootfs_path)?;
     BareMount::new(
@@ -1722,9 +1668,9 @@ fn load_kernel_module(module: &protocols::agent::KernelModule) -> Result<()> {
                 "load_kernel_module return code: {} stdout:{} stderr:{}",
                 code, std_out, std_err
             );
-            return Err(anyhow!(msg));
+            Err(anyhow!(msg))
         }
-        None => return Err(anyhow!("Process terminated by signal")),
+        None => Err(anyhow!("Process terminated by signal")),
     }
 }
 
@@ -1736,17 +1682,16 @@ mod tests {
     use std::sync::mpsc::{Receiver, Sender};
     use ttrpc::{MessageHeader, TtrpcContext};
 
-    fn mk_ttrpc_context() -> (TtrpcContext, Receiver<(MessageHeader, Vec<u8>)>) {
+    type Message = (MessageHeader, Vec<u8>);
+
+    fn mk_ttrpc_context() -> (TtrpcContext, Receiver<Message>) {
         let mh = MessageHeader::default();
 
-        let (tx, rx): (
-            Sender<(MessageHeader, Vec<u8>)>,
-            Receiver<(MessageHeader, Vec<u8>)>,
-        ) = channel();
+        let (tx, rx): (Sender<Message>, Receiver<Message>) = channel();
 
         let ctx = TtrpcContext {
             fd: -1,
-            mh: mh,
+            mh,
             res_tx: tx,
         };
 

src/agent/src/sandbox.rs

@@ -74,7 +74,7 @@ impl Sandbox {
             sender: None,
             rtnl: Some(RtnlHandle::new(NETLINK_ROUTE, 0).unwrap()),
             hooks: None,
-            event_rx: event_rx,
+            event_rx,
             event_tx: tx,
         })
     }
@@ -111,14 +111,14 @@ impl Sandbox {
     // acquiring a lock on sandbox.
     pub fn unset_sandbox_storage(&mut self, path: &str) -> Result<bool> {
         match self.storages.get_mut(path) {
-            None => return Err(anyhow!("Sandbox storage with path {} not found", path)),
+            None => Err(anyhow!("Sandbox storage with path {} not found", path)),
             Some(count) => {
                 *count -= 1;
                 if *count < 1 {
                     self.storages.remove(path);
                     return Ok(true);
                 }
-                return Ok(false);
+                Ok(false)
             }
         }
     }
@@ -160,13 +160,13 @@ impl Sandbox {
     pub fn setup_shared_namespaces(&mut self) -> Result<bool> {
         // Set up shared IPC namespace
         self.shared_ipcns = Namespace::new(&self.logger)
-            .as_ipc()
+            .get_ipc()
             .setup()
             .context("Failed to setup persistent IPC namespace")?;
 
         // // Set up shared UTS namespace
         self.shared_utsns = Namespace::new(&self.logger)
-            .as_uts(self.hostname.as_str())
+            .get_uts(self.hostname.as_str())
             .setup()
             .context("Failed to setup persistent UTS namespace")?;
 
@@ -183,7 +183,7 @@ impl Sandbox {
         // This means a separate pause process has not been created. We treat the
         // first container created as the infra container in that case
         // and use its pid namespace in case pid namespace needs to be shared.
-        if self.sandbox_pidns.is_none() && self.containers.len() == 0 {
+        if self.sandbox_pidns.is_none() && self.containers.is_empty() {
             let init_pid = c.init_process_pid;
             if init_pid == -1 {
                 return Err(anyhow!(
@@ -191,7 +191,7 @@ impl Sandbox {
                 ));
             }
 
-            let mut pid_ns = Namespace::new(&self.logger).as_pid();
+            let mut pid_ns = Namespace::new(&self.logger).get_pid();
             pid_ns.path = format!("/proc/{}/ns/pid", init_pid);
 
             self.sandbox_pidns = Some(pid_ns);
@@ -215,7 +215,7 @@ impl Sandbox {
     }
 
     pub fn destroy(&mut self) -> Result<()> {
-        for (_, ctr) in &mut self.containers {
+        for ctr in self.containers.values_mut() {
             ctr.destroy()?;
         }
         Ok(())
@@ -236,14 +236,29 @@ impl Sandbox {
             return Ok(());
         }
 
-        let cpuset = rustjail_cgroups::fs::get_guest_cpuset()?;
+        let guest_cpuset = rustjail_cgroups::fs::get_guest_cpuset()?;
 
         for (_, ctr) in self.containers.iter() {
+            let cpu = ctr
+                .config
+                .spec
+                .as_ref()
+                .unwrap()
+                .linux
+                .as_ref()
+                .unwrap()
+                .resources
+                .as_ref()
+                .unwrap()
+                .cpu
+                .as_ref();
+            let container_cpust = if let Some(c) = cpu { &c.cpus } else { "" };
+
             info!(self.logger, "updating {}", ctr.id.as_str());
             ctr.cgroup_manager
                 .as_ref()
                 .unwrap()
-                .update_cpuset_path(cpuset.as_str())?;
+                .update_cpuset_path(guest_cpuset.as_str(), &container_cpust)?;
         }
 
         Ok(())
@@ -335,7 +350,7 @@ fn online_resources(logger: &Logger, path: &str, pattern: &str, num: i32) -> Res
             }
             let c = c.unwrap();
 
-            if c.trim().contains("0") {
+            if c.trim().contains('0') {
                 let r = fs::write(file.as_str(), "1");
                 if r.is_err() {
                     continue;
@@ -614,8 +629,8 @@ mod tests {
 
         let linux = Linux::default();
         let mut spec = Spec::default();
-        spec.root = Some(root).into();
-        spec.linux = Some(linux).into();
+        spec.root = Some(root);
+        spec.linux = Some(linux);
 
         CreateOpts {
             cgroup_name: "".to_string(),
@@ -709,4 +724,31 @@ mod tests {
         assert!(s.hooks.as_ref().unwrap().poststart.is_empty());
         assert!(s.hooks.as_ref().unwrap().poststop.is_empty());
     }
+
+    #[test]
+    pub fn test_sandbox_is_running() {
+        let logger = slog::Logger::root(slog::Discard, o!());
+        let mut s = Sandbox::new(&logger).unwrap();
+        s.running = true;
+        assert!(s.is_running());
+        s.running = false;
+        assert!(!s.is_running());
+    }
+
+    #[test]
+    fn test_sandbox_set_hostname() {
+        let logger = slog::Logger::root(slog::Discard, o!());
+        let mut s = Sandbox::new(&logger).unwrap();
+        let hostname = "abc123";
+        s.set_hostname(hostname.to_string());
+        assert_eq!(s.hostname, hostname);
+    }
+
+    #[test]
+    fn test_sandbox_set_destroy() {
+        let logger = slog::Logger::root(slog::Discard, o!());
+        let mut s = Sandbox::new(&logger).unwrap();
+        let ret = s.destroy();
+        assert!(ret.is_ok());
+    }
 }

src/agent/src/test_utils.rs

@@ -2,6 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 //
+#![allow(clippy::module_inception)]
 
 #[cfg(test)]
 mod test_utils {

src/agent/src/uevent.rs

@@ -48,13 +48,16 @@ impl Uevent {
 
     // Check whether this is a block device hot-add event.
     fn is_block_add_event(&self) -> bool {
+        let pci_root_bus_path = create_pci_root_bus_path();
         self.action == U_EVENT_ACTION_ADD
             && self.subsystem == "block"
-            && self.devpath.starts_with(PCI_ROOT_BUS_PATH)
+            && self.devpath.starts_with(&pci_root_bus_path)
             && self.devname != ""
     }
 
     fn handle_block_add_event(&self, sandbox: &Arc<Mutex<Sandbox>>) {
+        let pci_root_bus_path = create_pci_root_bus_path();
+
         // Keep the same lock order as device::get_device_name(), otherwise it may cause deadlock.
         let mut w = GLOBAL_DEVICE_WATCHER.lock().unwrap();
         let mut sb = sandbox.lock().unwrap();
@@ -69,7 +72,7 @@ impl Uevent {
         let empties: Vec<_> = w
             .iter()
             .filter(|(dev_addr, _)| {
-                let pci_p = format!("{}/{}", PCI_ROOT_BUS_PATH, *dev_addr);
+                let pci_p = format!("{}/{}", pci_root_bus_path, *dev_addr);
 
                 // blk block device
                 devpath.starts_with(pci_p.as_str()) ||

src/runtime/Makefile

@@ -93,6 +93,9 @@ DEFAULTSDIR := $(SHAREDIR)/defaults
 COLLECT_SCRIPT = data/kata-collect-data.sh
 COLLECT_SCRIPT_SRC = $(COLLECT_SCRIPT).in
 
+# @RUNTIME_NAME@ should be replaced with the target in generated files
+RUNTIME_NAME = $(TARGET)
+
 GENERATED_FILES += $(COLLECT_SCRIPT)
 GENERATED_VARS = \
 		VERSION \
@@ -136,13 +139,14 @@ HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVIS
 QEMUPATH := $(QEMUBINDIR)/$(QEMUCMD)
 QEMUVALIDHYPERVISORPATHS := [\"$(QEMUPATH)\"]
 
-QEMUVALIDVIRTIOFSPATHS := $(QEMUBINDIR)/$(QEMUVIRTIOFSCMD)
+QEMUVIRTIOFSPATH := $(QEMUBINDIR)/$(QEMUVIRTIOFSCMD)
+QEMUVALIDVIRTIOFSPATHS := [\"$(QEMUVIRTIOFSPATH)\"]
 
 CLHPATH := $(CLHBINDIR)/$(CLHCMD)
-CLHVALIDHYPERVISORPATHS := [\"$(CLHBINDIR)/$(CLHCMD)\"]
+CLHVALIDHYPERVISORPATHS := [\"$(CLHPATH)\"]
 
 FCPATH = $(FCBINDIR)/$(FCCMD)
-FCVALIDPATHS = [\"$(FCPATH)\"]
+FCVALIDHYPERVISORPATHS := [\"$(FCPATH)\"]
 FCJAILERPATH = $(FCBINDIR)/$(FCJAILERCMD)
 FCVALIDJAILERPATHS = [\"$(FCJAILERPATH)\"]
 
@@ -600,8 +604,9 @@ $(SHIMV2_OUTPUT): $(SOURCES) $(GENERATED_FILES) $(MAKEFILE_LIST)
 	$(QUIET_BUILD)(cd $(SHIMV2_DIR)/ && ln -fs $(GENERATED_CONFIG))
 	$(QUIET_BUILD)(cd $(SHIMV2_DIR)/ && go build $(KATA_LDFLAGS) $(BUILDFLAGS) -o $@ .)
 
-$(MONITOR_OUTPUT): $(SOURCES) $(GENERATED_FILES) $(MAKEFILE_LIST)
-	$(QUIET_BUILD)(cd $(MONITOR_DIR)/ && go build $(KATA_LDFLAGS) $(BUILDFLAGS) -o $@ .)
+$(MONITOR_OUTPUT): $(SOURCES) $(GENERATED_FILES) $(MAKEFILE_LIST) .git-commit
+	$(QUIET_BUILD)(cd $(MONITOR_DIR)/ && go build \
+		--ldflags "-X main.GitCommit=$(shell cat .git-commit)" -o $@ .)
 
 .PHONY: \
 	check \

src/runtime/README.md

@@ -19,6 +19,8 @@ For details of the other Kata Containers repositories, see the
 * [Quick start for developers](#quick-start-for-developers)
 * [Architecture overview](#architecture-overview)
 * [Configuration](#configuration)
+    * [Hypervisor specific configuration](#hypervisor-specific-configuration)
+    * [Stateless systems](#stateless-systems)
 * [Logging](#logging)
     * [Kata OCI](#kata-oci)
     * [Kata containerd shimv2](#kata-containerd-shimv2)
@@ -65,7 +67,7 @@ The runtime has a built-in command to determine if your host system is capable
 of running and creating a Kata Container:
 
 ```bash
-$ kata-runtime kata-check
+$ kata-runtime check
 ```
 
 > **Note:**
@@ -106,6 +108,15 @@ The file contains comments explaining all options.
 > You may need to modify this file to optimise or tailor your system, or if you have
 > specific requirements.
 
+### Hypervisor specific configuration
+
+Kata Containers supports multiple hypervisors so your `configuration.toml`
+configuration file may be a symbolic link to a hypervisor-specific
+configuration file. See
+[the hypervisors document](../../docs/hypervisors.md) for further details.
+
+### Stateless systems
+
 Since the runtime supports a
 [stateless system](https://clearlinux.org/about),
 it checks for this configuration file in multiple locations, two of which are
@@ -135,7 +146,7 @@ To see details of your systems runtime environment (including the location of
 the configuration file being used), run:
 
 ```bash
-$ kata-runtime kata-env
+$ kata-runtime env
 ```
 
 ## Logging

src/runtime/VERSION

@@ -1 +0,0 @@
-../../VERSION

src/runtime/VERSION

@@ -0,0 +1 @@
+2.0.0

src/runtime/cli/config-generated.go.in

@@ -36,10 +36,6 @@ var commit = "@COMMIT@"
 // version is the runtime version.
 var version = "@VERSION@"
 
-// project-specific command names
-var envCmd = fmt.Sprintf("%s-env", projectPrefix)
-var checkCmd = fmt.Sprintf("%s-check", projectPrefix)
-
 // project-specific option names
 var configFilePathOption = fmt.Sprintf("%s-config", projectPrefix)
 var showConfigPathsOption = fmt.Sprintf("%s-show-default-config-paths", projectPrefix)

src/runtime/cli/config/configuration-acrn.toml.in

@@ -235,4 +235,4 @@ experimental=@DEFAULTEXPFEATURES@
 
 # If enabled, user can run pprof tools with shim v2 process through kata-monitor.
 # (default: false)
-# EnablePprof = true
+# enable_pprof = true

src/runtime/cli/config/configuration-clh.toml.in

@@ -234,4 +234,4 @@ experimental=@DEFAULTEXPFEATURES@
 
 # If enabled, user can run pprof tools with shim v2 process through kata-monitor.
 # (default: false)
-# EnablePprof = true
+# enable_pprof = true

src/runtime/cli/config/configuration-fc.toml.in

@@ -360,4 +360,4 @@ experimental=@DEFAULTEXPFEATURES@
 
 # If enabled, user can run pprof tools with shim v2 process through kata-monitor.
 # (default: false)
-# EnablePprof = true
+# enable_pprof = true

src/runtime/cli/config/configuration-qemu.toml.in

@@ -120,8 +120,8 @@ default_memory = @DEFMEMSZ@
 disable_block_device_use = @DEFDISABLEBLOCK@
 
 # Shared file system type:
-#   - virtio-9p (default)
-#   - virtio-fs
+#   - virtio-fs (default)
+#   - virtio-9p
 shared_fs = "@DEFSHAREDFS_QEMU_VIRTIOFS@"
 
 # Path to vhost-user-fs daemon.
@@ -319,6 +319,26 @@ valid_file_mem_backends = @DEFVALIDFILEMEMBACKENDS@
 # Default 0-sized value means unlimited rate.
 #tx_rate_limiter_max_rate = 0
 
+# Set where to save the guest memory dump file.
+# If set, when GUEST_PANICKED event occurred,
+# guest memeory will be dumped to host filesystem under guest_memory_dump_path,
+# This directory will be created automatically if it does not exist.
+#
+# The dumped file(also called vmcore) can be processed with crash or gdb.
+#
+# WARNING:
+#   Dump guest’s memory can take very long depending on the amount of guest memory
+#   and use much disk space.
+#guest_memory_dump_path="/var/crash/kata"
+
+# If enable paging.
+# Basically, if you want to use "gdb" rather than "crash",
+# or need the guest-virtual addresses in the ELF vmcore,
+# then you should enable paging.
+#
+# See: https://www.qemu.org/docs/master/qemu-qmp-ref.html#Dump-guest-memory for details
+#guest_memory_dump_paging=false
+
 [factory]
 # VM templating support. Once enabled, new VMs are created from template
 # using vm cloning. They will share the same initial kernel, initramfs and
@@ -486,4 +506,4 @@ experimental=@DEFAULTEXPFEATURES@
 
 # If enabled, user can run pprof tools with shim v2 process through kata-monitor.
 # (default: false)
-# EnablePprof = true
+# enable_pprof = true

src/runtime/cli/kata-check.go

@@ -134,17 +134,25 @@ func getCPUFlags(cpuinfo string) string {
 // haveKernelModule returns true if the specified module exists
 // (either loaded or available to be loaded)
 func haveKernelModule(module string) bool {
+	kmodLog := kataLog.WithField("module", module)
+
 	// First, check to see if the module is already loaded
 	path := filepath.Join(sysModuleDir, module)
 	if katautils.FileExists(path) {
 		return true
 	}
 
+	// Only root can load modules
+	if os.Getuid() != 0 {
+		kmodLog.Error("Module is not loaded and it can not be inserted. Please consider running with sudo or as root")
+		return false
+	}
+
 	// Now, check if the module is unloaded, but available.
 	// And modprobe it if so.
 	cmd := exec.Command(modProbeCmd, module)
 	if output, err := cmd.CombinedOutput(); err != nil {
-		kataLog.WithField("module", module).WithError(err).Warnf("modprobe insert module failed: %s", string(output))
+		kmodLog.WithError(err).WithField("output", string(output)).Warnf("modprobe insert module failed")
 		return false
 	}
 	return true
@@ -306,8 +314,9 @@ func genericHostIsVMContainerCapable(details vmContainerCapableDetails) error {
 }
 
 var kataCheckCLICommand = cli.Command{
-	Name:  checkCmd,
-	Usage: "tests if system can run " + project,
+	Name:    "check",
+	Aliases: []string{"kata-check"},
+	Usage:   "tests if system can run " + project,
 	Flags: []cli.Flag{
 		cli.BoolFlag{
 			Name:  "check-version-only",
@@ -344,36 +353,36 @@ EXAMPLES:
 
 - Perform basic checks:
 
-  $ %s %s
+  $ %s check
 
 - Local basic checks only:
 
-  $ %s %s --no-network-checks
+  $ %s check --no-network-checks
 
 - Perform further checks:
 
-  $ sudo %s %s
+  $ sudo %s check
 
 - Just check if a newer version is available:
 
-  $ %s %s --check-version-only
+  $ %s check --check-version-only
 
 - List available releases (shows output in format "version;release-date;url"):
 
-  $ %s %s --only-list-releases
+  $ %s check --only-list-releases
 
 - List all available releases (includes pre-release versions):
 
-  $ %s %s --only-list-releases --include-all-releases
+  $ %s check --only-list-releases --include-all-releases
 `,
 		project,
 		noNetworkEnvVar,
-		name, checkCmd,
-		name, checkCmd,
-		name, checkCmd,
-		name, checkCmd,
-		name, checkCmd,
-		name, checkCmd,
+		name,
+		name,
+		name,
+		name,
+		name,
+		name,
 	),
 
 	Action: func(context *cli.Context) error {
@@ -386,7 +395,7 @@ EXAMPLES:
 			return err
 		}
 
-		span, _ := katautils.Trace(ctx, "kata-check")
+		span, _ := katautils.Trace(ctx, "check")
 		defer span.Finish()
 
 		if context.Bool("no-network-checks") == false && os.Getenv(noNetworkEnvVar) == "" {
@@ -413,7 +422,7 @@ EXAMPLES:
 
 		runtimeConfig, ok := context.App.Metadata["runtimeConfig"].(oci.RuntimeConfig)
 		if !ok {
-			return errors.New("kata-check: cannot determine runtime config")
+			return errors.New("check: cannot determine runtime config")
 		}
 
 		err = setCPUtype(runtimeConfig.HypervisorType)

src/runtime/cli/kata-check_test.go

@@ -513,6 +513,10 @@ func TestCheckCheckCPUAttribs(t *testing.T) {
 }
 
 func TestCheckHaveKernelModule(t *testing.T) {
+	if tc.NotValid(ktu.NeedRoot()) {
+		t.Skip(testDisabledAsNonRoot)
+	}
+
 	assert := assert.New(t)
 
 	dir, err := ioutil.TempDir("", "")

src/runtime/cli/kata-env.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/BurntSushi/toml"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
@@ -292,7 +293,7 @@ func getNetmonInfo(config oci.RuntimeConfig) NetmonInfo {
 }
 
 func getCommandVersion(cmd string) (string, error) {
-	return katautils.RunCommand([]string{cmd, "--version"})
+	return utils.RunCommand([]string{cmd, "--version"})
 }
 
 func getAgentInfo(config oci.RuntimeConfig) (AgentInfo, error) {
@@ -437,8 +438,9 @@ func writeJSONSettings(env EnvInfo, file *os.File) error {
 }
 
 var kataEnvCLICommand = cli.Command{
-	Name:  envCmd,
-	Usage: "display settings. Default to TOML",
+	Name:    "env",
+	Aliases: []string{"kata-env"},
+	Usage:   "display settings. Default to TOML",
 	Flags: []cli.Flag{
 		cli.BoolFlag{
 			Name:  "json",

src/runtime/cli/kata-monitor/main.go

@@ -9,6 +9,8 @@ import (
 	"flag"
 	"net/http"
 	"os"
+	"runtime"
+	"text/template"
 	"time"
 
 	kataMonitor "github.com/kata-containers/kata-containers/src/runtime/pkg/kata-monitor"
@@ -20,12 +22,77 @@ var containerdAddr = flag.String("containerd-address", "/run/containerd/containe
 var containerdConfig = flag.String("containerd-conf", "/etc/containerd/config.toml", "Containerd config file.")
 var logLevel = flag.String("log-level", "info", "Log level of logrus(trace/debug/info/warn/error/fatal/panic).")
 
+// These values are overridden via ldflags
+var (
+	appName = "kata-monitor"
+	// version is the kata monitor version.
+	version = "0.1.0"
+
+	GitCommit = "unknown-commit"
+)
+
+type versionInfo struct {
+	AppName   string
+	Version   string
+	GitCommit string
+	GoVersion string
+	Os        string
+	Arch      string
+}
+
+var versionTemplate = `{{.AppName}}
+ Version:	{{.Version}}
+ Go version:	{{.GoVersion}}
+ Git commit:	{{.GitCommit}}
+ OS/Arch:	{{.Os}}/{{.Arch}}
+`
+
+func printVersion(ver versionInfo) {
+	t, err := template.New("version").Parse(versionTemplate)
+
+	if err = t.Execute(os.Stdout, ver); err != nil {
+		panic(err)
+	}
+}
+
 func main() {
+	ver := versionInfo{
+		AppName:   appName,
+		Version:   version,
+		GoVersion: runtime.Version(),
+		Os:        runtime.GOOS,
+		Arch:      runtime.GOARCH,
+		GitCommit: GitCommit,
+	}
+
+	if len(os.Args) == 2 && (os.Args[1] == "--version" || os.Args[1] == "version") {
+		printVersion(ver)
+		return
+	}
+
 	flag.Parse()
 
 	// init logrus
 	initLog()
 
+	announceFields := logrus.Fields{
+		// properties from version info
+		"app":        ver.AppName,
+		"version":    ver.Version,
+		"go-version": ver.GoVersion,
+		"os":         ver.Os,
+		"arch":       ver.Arch,
+		"git-commit": ver.GitCommit,
+
+		// properties from command-line options
+		"listen-address":     *monitorListenAddr,
+		"containerd-address": *containerdAddr,
+		"containerd-conf":    *containerdConfig,
+		"log-level":          *logLevel,
+	}
+
+	logrus.WithFields(announceFields).Info("announce")
+
 	// create new kataMonitor
 	km, err := kataMonitor.NewKataMonitor(*containerdAddr, *containerdConfig)
 	if err != nil {

src/runtime/cli/main.go

@@ -272,7 +272,7 @@ func beforeSubcommands(c *cli.Context) error {
 	ignoreConfigLogs := false
 	var traceRootSpan string
 
-	subCmdIsCheckCmd := (c.NArg() >= 1 && (c.Args()[0] == checkCmd))
+	subCmdIsCheckCmd := (c.NArg() >= 1 && ((c.Args()[0] == "kata-check") || (c.Args()[0] == "check")))
 	if subCmdIsCheckCmd {
 		// checkCmd will use the default logrus logger to stderr
 		// raise the logger default level to warn
@@ -313,7 +313,7 @@ func beforeSubcommands(c *cli.Context) error {
 		// (meaning any spans created at this point will be silently ignored).
 		setExternalLoggers(context.Background(), kataLog)
 
-		if c.NArg() == 1 && c.Args()[0] == envCmd {
+		if c.NArg() == 1 && (c.Args()[0] == "kata-env" || c.Args()[0] == "env") {
 			// simply report the logging setup
 			ignoreConfigLogs = true
 		}

src/runtime/cli/main_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/dlespiau/covertool/pkg/cover"
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
@@ -273,7 +274,7 @@ func createOCIConfig(bundleDir string) error {
 		return errors.New("Cannot find command to generate OCI config file")
 	}
 
-	_, err := katautils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
+	_, err := utils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
 	if err != nil {
 		return err
 	}
@@ -378,7 +379,7 @@ func makeOCIBundle(bundleDir string) error {
 		}
 	}
 
-	output, err := katautils.RunCommandFull([]string{"cp", "-a", from, to}, true)
+	output, err := utils.RunCommandFull([]string{"cp", "-a", from, to}, true)
 	if err != nil {
 		return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output)
 	}

src/runtime/cli/release.go

@@ -6,6 +6,7 @@
 package main
 
 import (
+	"bytes"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -276,12 +277,17 @@ func getReleases(releaseURL string, includeAll bool) ([]semver.Version, map[stri
 
 	releasesArray := []map[string]interface{}{}
 
-	bytes, err := ioutil.ReadAll(resp.Body)
+	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to read release details: %v", err)
+	} else if resp.StatusCode == http.StatusForbidden && bytes.Contains(body, []byte("limit exceeded")) {
+		// Do not fail if rate limit is exceeded
+		kataLog.WithField("url", releaseURL).
+			Warn("API rate limit exceeded. Try again later. Read https://docs.github.com/apps/building-github-apps/understanding-rate-limits-for-github-apps for more information")
+		return []semver.Version{}, map[string]releaseDetails{}, nil
 	}
 
-	if err := json.Unmarshal(bytes, &releasesArray); err != nil {
+	if err := json.Unmarshal(body, &releasesArray); err != nil {
 		return nil, nil, fmt.Errorf("failed to unpack release details: %v", err)
 	}
 
@@ -321,8 +327,14 @@ func getNewReleaseType(current semver.Version, latest semver.Version) (string, e
 		}
 	} else if latest.Patch == current.Patch && len(latest.Pre) > 0 {
 		desc = "pre-release"
+	} else if latest.Major == current.Major &&
+		latest.Minor == current.Minor &&
+		latest.Patch == current.Patch {
+		if len(current.Pre) > 0 && len(latest.Pre) == 0 {
+			desc = "major"
+		}
 	} else {
-		return "", fmt.Errorf("BUG: unhandled scenario: current version: %s, latest version: %v", current, latest)
+		return "", fmt.Errorf("BUG: unhandled scenario: current version: %s, latest version: %s", current, latest)
 	}
 
 	return desc, nil

src/runtime/cli/release_test.go

@@ -458,6 +458,21 @@ func TestGetNewReleaseType(t *testing.T) {
 	}
 
 	data := []testData{
+		// Check build metadata (ignored for version comparisions)
+		{"2.0.0+build", "2.0.0", true, ""},
+		{"2.0.0+build-1", "2.0.0+build-2", true, ""},
+		{"1.12.0+build", "1.12.0", true, ""},
+
+		{"2.0.0-rc3+foo", "2.0.0", false, "major"},
+		{"2.0.0-rc3+foo", "2.0.0-rc4", false, "pre-release"},
+		{"1.12.0+foo", "1.13.0", false, "minor"},
+
+		{"1.12.0+build", "2.0.0", false, "major"},
+		{"1.12.0+build", "1.13.0", false, "minor"},
+		{"1.12.0-rc2+build", "1.12.1", false, "patch"},
+		{"1.12.0-rc2+build", "1.12.1-foo", false, "patch pre-release"},
+		{"1.12.0-rc4+wibble", "1.12.0", false, "major"},
+
 		{"2.0.0-alpha3", "1.0.0", true, ""},
 		{"1.0.0", "1.0.0", true, ""},
 		{"2.0.0", "1.0.0", true, ""},
@@ -473,6 +488,12 @@ func TestGetNewReleaseType(t *testing.T) {
 		{"1.0.0", "1.0.3", false, "patch"},
 		{"1.0.0-beta29", "1.0.0-beta30", false, "pre-release"},
 		{"1.0.0", "1.0.3-alpha99.1b", false, "patch pre-release"},
+
+		{"2.0.0-rc0", "2.0.0", false, "major"},
+		{"2.0.0-rc1", "2.0.0", false, "major"},
+
+		{"1.12.0-rc0", "1.12.0", false, "major"},
+		{"1.12.0-rc5", "1.12.0", false, "major"},
 	}
 
 	for i, d := range data {

src/runtime/cli/utils_test.go

@@ -13,6 +13,7 @@ import (
 	"testing"
 
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -176,13 +177,13 @@ VERSION_ID="%s"
 }
 
 func TestUtilsRunCommand(t *testing.T) {
-	output, err := katautils.RunCommand([]string{"true"})
+	output, err := utils.RunCommand([]string{"true"})
 	assert.NoError(t, err)
 	assert.Equal(t, "", output)
 }
 
 func TestUtilsRunCommandCaptureStdout(t *testing.T) {
-	output, err := katautils.RunCommand([]string{"echo", "hello"})
+	output, err := utils.RunCommand([]string{"echo", "hello"})
 	assert.NoError(t, err)
 	assert.Equal(t, "hello", output)
 }
@@ -190,7 +191,7 @@ func TestUtilsRunCommandCaptureStdout(t *testing.T) {
 func TestUtilsRunCommandIgnoreStderr(t *testing.T) {
 	args := []string{"/bin/sh", "-c", "echo foo >&2;exit 0"}
 
-	output, err := katautils.RunCommand(args)
+	output, err := utils.RunCommand(args)
 	assert.NoError(t, err)
 	assert.Equal(t, "", output)
 }
@@ -213,7 +214,7 @@ func TestUtilsRunCommandInvalidCmds(t *testing.T) {
 	}
 
 	for _, args := range invalidCommands {
-		output, err := katautils.RunCommand(args)
+		output, err := utils.RunCommand(args)
 		assert.Error(t, err)
 		assert.Equal(t, "", output)
 	}

src/runtime/containerd-shim-v2/service.go

@@ -25,6 +25,7 @@ import (
 	"github.com/containerd/typeurl"
 	ptypes "github.com/gogo/protobuf/types"
 	"github.com/opencontainers/runtime-spec/specs-go"
+	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
@@ -78,6 +79,21 @@ func New(ctx context.Context, id string, publisher events.Publisher) (cdshim.Shi
 	vci.SetLogger(ctx, shimLog)
 	katautils.SetLogger(ctx, shimLog, shimLog.Logger.Level)
 
+	// load runtime config so that tracing can start if enabled
+	_, runtimeConfig, err := katautils.LoadConfiguration("", false, true)
+	if err != nil {
+		return nil, err
+	}
+
+	// create tracer
+	_, err = katautils.CreateTracer("kata")
+	if err != nil {
+		return nil, err
+	}
+	// create span
+	span, ctx := trace(ctx, "New")
+	defer span.Finish()
+
 	ctx, cancel := context.WithCancel(ctx)
 
 	s := &service{
@@ -85,6 +101,7 @@ func New(ctx context.Context, id string, publisher events.Publisher) (cdshim.Shi
 		pid:        uint32(os.Getpid()),
 		ctx:        ctx,
 		containers: make(map[string]*container),
+		config:     &runtimeConfig,
 		events:     make(chan interface{}, chSize),
 		ec:         make(chan exit, bufferSize),
 		cancel:     cancel,
@@ -167,6 +184,13 @@ func newCommand(ctx context.Context, containerdBinary, id, containerdAddress str
 // StartShim willl start a kata shimv2 daemon which will implemented the
 // ShimV2 APIs such as create/start/update etc containers.
 func (s *service) StartShim(ctx context.Context, id, containerdBinary, containerdAddress string) (string, error) {
+	// Stop tracing here since a new tracer will be created the next time New()
+	// is called again after StartShim()
+	defer katautils.StopTracing(s.ctx)
+
+	span, _ := trace(s.ctx, "StartShim")
+	defer span.Finish()
+
 	bundlePath, err := os.Getwd()
 	if err != nil {
 		return "", err
@@ -215,8 +239,6 @@ func (s *service) StartShim(ctx context.Context, id, containerdBinary, container
 		}
 	}()
 
-	// make sure to wait after start
-	go cmd.Wait()
 	if err = cdshim.WritePidFile("shim.pid", cmd.Process.Pid); err != nil {
 		return "", err
 	}
@@ -280,7 +302,22 @@ func getTopic(e interface{}) string {
 	return cdruntime.TaskUnknownTopic
 }
 
+func trace(ctx context.Context, name string) (opentracing.Span, context.Context) {
+	if ctx == nil {
+		logrus.WithField("type", "bug").Error("trace called before context set")
+		ctx = context.Background()
+	}
+
+	span, ctx := opentracing.StartSpanFromContext(ctx, name)
+	span.SetTag("source", "runtime")
+
+	return span, ctx
+}
+
 func (s *service) Cleanup(ctx context.Context) (_ *taskAPI.DeleteResponse, err error) {
+	span, _ := trace(s.ctx, "Cleanup")
+	defer span.Finish()
+
 	//Since the binary cleanup will return the DeleteResponse from stdout to
 	//containerd, thus we must make sure there is no any outputs in stdout except
 	//the returned response, thus here redirect the log to stderr in case there's
@@ -336,6 +373,9 @@ func (s *service) Cleanup(ctx context.Context) (_ *taskAPI.DeleteResponse, err e
 
 // Create a new sandbox or container with the underlying OCI runtime
 func (s *service) Create(ctx context.Context, r *taskAPI.CreateTaskRequest) (_ *taskAPI.CreateTaskResponse, err error) {
+	span, _ := trace(s.ctx, "Create")
+	defer span.Finish()
+
 	start := time.Now()
 	defer func() {
 		err = toGRPC(err)
@@ -345,38 +385,53 @@ func (s *service) Create(ctx context.Context, r *taskAPI.CreateTaskRequest) (_ *
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
-	var c *container
-
-	c, err = create(ctx, s, r)
-	if err != nil {
-		return nil, err
+	type Result struct {
+		container *container
+		err       error
 	}
+	ch := make(chan Result, 1)
+	go func() {
+		container, err := create(ctx, s, r)
+		ch <- Result{container, err}
+	}()
 
-	c.status = task.StatusCreated
+	select {
+	case <-ctx.Done():
+		return nil, errors.Errorf("create container timeout: %v", r.ID)
+	case res := <-ch:
+		if res.err != nil {
+			return nil, res.err
+		}
+		container := res.container
+		container.status = task.StatusCreated
 
-	s.containers[r.ID] = c
+		s.containers[r.ID] = container
 
-	s.send(&eventstypes.TaskCreate{
-		ContainerID: r.ID,
-		Bundle:      r.Bundle,
-		Rootfs:      r.Rootfs,
-		IO: &eventstypes.TaskIO{
-			Stdin:    r.Stdin,
-			Stdout:   r.Stdout,
-			Stderr:   r.Stderr,
-			Terminal: r.Terminal,
-		},
-		Checkpoint: r.Checkpoint,
-		Pid:        s.pid,
-	})
+		s.send(&eventstypes.TaskCreate{
+			ContainerID: r.ID,
+			Bundle:      r.Bundle,
+			Rootfs:      r.Rootfs,
+			IO: &eventstypes.TaskIO{
+				Stdin:    r.Stdin,
+				Stdout:   r.Stdout,
+				Stderr:   r.Stderr,
+				Terminal: r.Terminal,
+			},
+			Checkpoint: r.Checkpoint,
+			Pid:        s.pid,
+		})
 
-	return &taskAPI.CreateTaskResponse{
-		Pid: s.pid,
-	}, nil
+		return &taskAPI.CreateTaskResponse{
+			Pid: s.pid,
+		}, nil
+	}
 }
 
 // Start a process
 func (s *service) Start(ctx context.Context, r *taskAPI.StartRequest) (_ *taskAPI.StartResponse, err error) {
+	span, _ := trace(s.ctx, "Start")
+	defer span.Finish()
+
 	start := time.Now()
 	defer func() {
 		err = toGRPC(err)
@@ -425,6 +480,9 @@ func (s *service) Start(ctx context.Context, r *taskAPI.StartRequest) (_ *taskAP
 
 // Delete the initial process and container
 func (s *service) Delete(ctx context.Context, r *taskAPI.DeleteRequest) (_ *taskAPI.DeleteResponse, err error) {
+	span, _ := trace(s.ctx, "Delete")
+	defer span.Finish()
+
 	start := time.Now()
 	defer func() {
 		err = toGRPC(err)
@@ -474,6 +532,9 @@ func (s *service) Delete(ctx context.Context, r *taskAPI.DeleteRequest) (_ *task
 
 // Exec an additional process inside the container
 func (s *service) Exec(ctx context.Context, r *taskAPI.ExecProcessRequest) (_ *ptypes.Empty, err error) {
+	span, _ := trace(s.ctx, "Exec")
+	defer span.Finish()
+
 	start := time.Now()
 	defer func() {
 		rpcDurationsHistogram.WithLabelValues("exec").Observe(float64(time.Since(start).Nanoseconds() / int64(time.Millisecond)))
@@ -509,6 +570,9 @@ func (s *service) Exec(ctx context.Context, r *taskAPI.ExecProcessRequest) (_ *p
 
 // ResizePty of a process
 func (s *service) ResizePty(ctx context.Context, r *taskAPI.ResizePtyRequest) (_ *ptypes.Empty, err error) {
+	span, _ := trace(s.ctx, "ResizePty")
+	defer span.Finish()
+
 	start := time.Now()
 	defer func() {
 		err = toGRPC(err)
@@ -545,6 +609,9 @@ func (s *service) ResizePty(ctx context.Context, r *taskAPI.ResizePtyRequest) (_
 
 // State returns runtime state information for a process
 func (s *service) State(ctx context.Context, r *taskAPI.StateRequest) (_ *taskAPI.StateResponse, err error) {
+	span, _ := trace(s.ctx, "State")
+	defer span.Finish()
+
 	start := time.Now()
 	defer func() {
 		err = toGRPC(err)
@@ -590,11 +657,13 @@ func (s *service) State(ctx context.Context, r *taskAPI.StateRequest) (_ *taskAP
 		Terminal:   execs.tty.terminal,
 		ExitStatus: uint32(execs.exitCode),
 	}, nil
-
 }
 
 // Pause the container
 func (s *service) Pause(ctx context.Context, r *taskAPI.PauseRequest) (_ *ptypes.Empty, err error) {
+	span, _ := trace(s.ctx, "Pause")
+	defer span.Finish()
+
 	start := time.Now()
 	defer func() {
 		err = toGRPC(err)
@@ -631,6 +700,9 @@ func (s *service) Pause(ctx context.Context, r *taskAPI.PauseRequest) (_ *ptypes
 
 // Resume the container
 func (s *service) Resume(ctx context.Context, r *taskAPI.ResumeRequest) (_ *ptypes.Empty, err error) {
+	span, _ := trace(s.ctx, "Resume")
+	defer span.Finish()
+
 	start := time.Now()
 	defer func() {
 		err = toGRPC(err)
@@ -665,6 +737,9 @@ func (s *service) Resume(ctx context.Context, r *taskAPI.ResumeRequest) (_ *ptyp
 
 // Kill a process with the provided signal
 func (s *service) Kill(ctx context.Context, r *taskAPI.KillRequest) (_ *ptypes.Empty, err error) {
+	span, _ := trace(s.ctx, "Kill")
+	defer span.Finish()
+
 	start := time.Now()
 	defer func() {
 		err = toGRPC(err)
@@ -723,6 +798,9 @@ func (s *service) Kill(ctx context.Context, r *taskAPI.KillRequest) (_ *ptypes.E
 // Since for kata, it cannot get the process's pid from VM,
 // thus only return the Shim's pid directly.
 func (s *service) Pids(ctx context.Context, r *taskAPI.PidsRequest) (_ *taskAPI.PidsResponse, err error) {
+	span, _ := trace(s.ctx, "Pids")
+	defer span.Finish()
+
 	var processes []*task.ProcessInfo
 
 	start := time.Now()
@@ -743,6 +821,9 @@ func (s *service) Pids(ctx context.Context, r *taskAPI.PidsRequest) (_ *taskAPI.
 
 // CloseIO of a process
 func (s *service) CloseIO(ctx context.Context, r *taskAPI.CloseIORequest) (_ *ptypes.Empty, err error) {
+	span, _ := trace(s.ctx, "CloseIO")
+	defer span.Finish()
+
 	start := time.Now()
 	defer func() {
 		err = toGRPC(err)
@@ -781,6 +862,9 @@ func (s *service) CloseIO(ctx context.Context, r *taskAPI.CloseIORequest) (_ *pt
 
 // Checkpoint the container
 func (s *service) Checkpoint(ctx context.Context, r *taskAPI.CheckpointTaskRequest) (_ *ptypes.Empty, err error) {
+	span, _ := trace(s.ctx, "Checkpoint")
+	defer span.Finish()
+
 	start := time.Now()
 	defer func() {
 		err = toGRPC(err)
@@ -792,6 +876,9 @@ func (s *service) Checkpoint(ctx context.Context, r *taskAPI.CheckpointTaskReque
 
 // Connect returns shim information such as the shim's pid
 func (s *service) Connect(ctx context.Context, r *taskAPI.ConnectRequest) (_ *taskAPI.ConnectResponse, err error) {
+	span, _ := trace(s.ctx, "Connect")
+	defer span.Finish()
+
 	start := time.Now()
 	defer func() {
 		err = toGRPC(err)
@@ -809,6 +896,8 @@ func (s *service) Connect(ctx context.Context, r *taskAPI.ConnectRequest) (_ *ta
 }
 
 func (s *service) Shutdown(ctx context.Context, r *taskAPI.ShutdownRequest) (_ *ptypes.Empty, err error) {
+	span, _ := trace(s.ctx, "Shutdown")
+
 	start := time.Now()
 	defer func() {
 		err = toGRPC(err)
@@ -822,6 +911,9 @@ func (s *service) Shutdown(ctx context.Context, r *taskAPI.ShutdownRequest) (_ *
 	}
 	s.mu.Unlock()
 
+	span.Finish()
+	katautils.StopTracing(s.ctx)
+
 	s.cancel()
 
 	os.Exit(0)
@@ -832,6 +924,9 @@ func (s *service) Shutdown(ctx context.Context, r *taskAPI.ShutdownRequest) (_ *
 }
 
 func (s *service) Stats(ctx context.Context, r *taskAPI.StatsRequest) (_ *taskAPI.StatsResponse, err error) {
+	span, _ := trace(s.ctx, "Stats")
+	defer span.Finish()
+
 	start := time.Now()
 	defer func() {
 		err = toGRPC(err)
@@ -858,6 +953,9 @@ func (s *service) Stats(ctx context.Context, r *taskAPI.StatsRequest) (_ *taskAP
 
 // Update a running container
 func (s *service) Update(ctx context.Context, r *taskAPI.UpdateTaskRequest) (_ *ptypes.Empty, err error) {
+	span, _ := trace(s.ctx, "Update")
+	defer span.Finish()
+
 	start := time.Now()
 	defer func() {
 		err = toGRPC(err)
@@ -887,6 +985,9 @@ func (s *service) Update(ctx context.Context, r *taskAPI.UpdateTaskRequest) (_ *
 
 // Wait for a process to exit
 func (s *service) Wait(ctx context.Context, r *taskAPI.WaitRequest) (_ *taskAPI.WaitResponse, err error) {
+	span, _ := trace(s.ctx, "Wait")
+	defer span.Finish()
+
 	var ret uint32
 
 	start := time.Now()

src/runtime/containerd-shim-v2/utils.go

@@ -31,10 +31,10 @@ func cReap(s *service, status int, id, execid string, exitat time.Time) {
 	}
 }
 
-func cleanupContainer(ctx context.Context, sid, cid, bundlePath string) error {
+func cleanupContainer(ctx context.Context, sandboxID, cid, bundlePath string) error {
 	shimLog.WithField("service", "cleanup").WithField("container", cid).Info("Cleanup container")
 
-	err := vci.CleanupContainer(ctx, sid, cid, true)
+	err := vci.CleanupContainer(ctx, sandboxID, cid, true)
 	if err != nil {
 		shimLog.WithError(err).WithField("container", cid).Warn("failed to cleanup container")
 		return err

src/runtime/containerd-shim-v2/utils_test.go

@@ -21,6 +21,7 @@ import (
 
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
@@ -150,7 +151,7 @@ func createOCIConfig(bundleDir string) error {
 		return errors.New("Cannot find command to generate OCI config file")
 	}
 
-	_, err := katautils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
+	_, err := utils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
 	if err != nil {
 		return err
 	}
@@ -278,7 +279,7 @@ func makeOCIBundle(bundleDir string) error {
 		}
 	}
 
-	output, err := katautils.RunCommandFull([]string{"cp", "-a", from, to}, true)
+	output, err := utils.RunCommandFull([]string{"cp", "-a", from, to}, true)
 	if err != nil {
 		return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output)
 	}

src/runtime/containerd-shim-v2/wait.go

@@ -20,6 +20,8 @@ import (
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 )
 
+const defaultCheckInterval = 1 * time.Second
+
 func wait(s *service, c *container, execID string) (int32, error) {
 	var execs *exec
 	var err error
@@ -152,6 +154,7 @@ func watchOOMEvents(ctx context.Context, s *service) {
 				if isGRPCErrorCode(codes.NotFound, err) || err.Error() == "Dead agent" {
 					return
 				}
+				time.Sleep(defaultCheckInterval)
 				continue
 			}
 

src/runtime/data/completions/bash/kata-runtime

@@ -63,6 +63,8 @@ _kata_subcmd_needs_existing_container()
     for cmd in \
 	    'kata-check' \
 	    'kata-env' \
+	    'check' \
+	    'env' \
 	    'create' \
 	    'help' \
 	    'list' \

src/runtime/data/kata-collect-data.sh.in

@@ -15,6 +15,9 @@ typeset -r runtime=${runtime_path:-"$runtime_snap_path"}
 typeset -r containerd_shim_v2_name="containerd-shim-kata-v2"
 typeset -r containerd_shim_v2=$(command -v "$containerd_shim_v2_name" 2>/dev/null)
 
+typeset -r kata_monitor_name="kata-monitor"
+typeset -r kata_monitor=$(command -v "$kata_monitor_name" 2>/dev/null)
+
 typeset -r issue_url="@PROJECT_BUG_URL@"
 typeset -r script_version="@VERSION@ (commit @COMMIT@)"
 
@@ -770,6 +773,21 @@ show_throttler_details()
 	end_section
 }
 
+show_kata_monitor_version()
+{
+	start_section "Kata Monitor"
+
+	local cmd="${kata_monitor_name} --version"
+
+	msg "Kata Monitor \`$kata_monitor_name\`."
+
+	run_cmd_and_show_quoted_output "" "$cmd"
+
+	separator
+
+	end_section
+}
+
 # Retrieve details of the image containing
 # the rootfs used to boot the virtual machine.
 show_image_details()
@@ -850,6 +868,7 @@ show_details()
 	show_log_details
 	show_container_mgr_details
 	show_package_versions
+	show_kata_monitor_version
 
 	show_footer
 }

src/runtime/go.mod

@@ -39,7 +39,7 @@ require (
 	github.com/opencontainers/runtime-spec v1.0.2-0.20190408193819-a1b50f621a48
 	github.com/opencontainers/selinux v1.4.0
 	github.com/opentracing/opentracing-go v1.1.0
-	github.com/pkg/errors v0.8.1
+	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.7.1
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/common v0.10.0
@@ -56,11 +56,12 @@ require (
 	github.com/vishvananda/netlink v1.0.1-0.20190604022042-c8c507c80ea2
 	github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc
 	go.uber.org/atomic v1.6.0 // indirect
-	golang.org/x/net v0.0.0-20191108221443-4ba9e2ef068c
+	golang.org/x/net v0.0.0-20200707034311-ab3426394381
 	golang.org/x/oauth2 v0.0.0-20191122200657-5d9234df094c
-	golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1
+	golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4
 	google.golang.org/grpc v1.19.0
 	gotest.tools v2.2.0+incompatible // indirect
+	k8s.io/apimachinery v0.18.2
 )
 
 replace (

src/runtime/go.sum

@@ -7,8 +7,11 @@ github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Microsoft/hcsshim v0.8.6 h1:ZfF0+zZeYdzMIVMZHKtDKJvLHj76XCuVae/jNkjj0IA=
 github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
+github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/purell v1.1.0 h1:rmGxhojJlM0tuKtfdvliR84CFHljx9ag64t2xmVkjK4=
 github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -78,9 +81,14 @@ github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ
 github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
 github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is=
 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
@@ -90,6 +98,7 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI=
 github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
 github.com/go-openapi/analysis v0.17.2 h1:eYp14J1o8TTSCzndHBtsNuckikV1PfZOSnx4BcBeu0c=
@@ -98,9 +107,11 @@ github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQH
 github.com/go-openapi/errors v0.17.2/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
 github.com/go-openapi/errors v0.18.0 h1:+RnmJ5MQccF7jwWAoMzwOpzJEspZ18ZIWfg9Z2eiXq8=
 github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
+github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
 github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
 github.com/go-openapi/jsonpointer v0.17.2 h1:3ekBy41gar/iJi2KSh/au/PrC2vpLr85upF/UZmm3W0=
 github.com/go-openapi/jsonpointer v0.17.2/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
+github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
 github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
 github.com/go-openapi/jsonreference v0.17.2 h1:lF3z7AH8dd0IKXc1zEBi1dj0B4XgVb5cVjn39dCK3Ls=
 github.com/go-openapi/jsonreference v0.17.2/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
@@ -110,6 +121,7 @@ github.com/go-openapi/loads v0.17.2/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf
 github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA=
 github.com/go-openapi/runtime v0.18.0 h1:ddoL4Uo/729XbNAS9UIsG7Oqa8R8l2edBe6Pq/i8AHM=
 github.com/go-openapi/runtime v0.18.0/go.mod h1:uI6pHuxWYTy94zZxgcwJkUWa9wbIlhteGfloI10GD4U=
+github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
 github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
 github.com/go-openapi/spec v0.17.2 h1:eb2NbuCnoe8cWAxhtK6CfMWUYmiFEZJ9Hx3Z2WRwJ5M=
 github.com/go-openapi/spec v0.17.2/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
@@ -117,6 +129,7 @@ github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pL
 github.com/go-openapi/strfmt v0.17.2/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
 github.com/go-openapi/strfmt v0.18.0 h1:FqqmmVCKn3di+ilU/+1m957T1CnMz3IteVUcV3aGXWA=
 github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
+github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
 github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
 github.com/go-openapi/swag v0.17.2/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
 github.com/go-openapi/swag v0.18.0 h1:1DU8Km1MRGv9Pj7BNLmkA+umwTStwDHttXvx3NhJA70=
@@ -134,7 +147,9 @@ github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -152,20 +167,28 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
+github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.0 h1:Jf4mxPC/ziBnoPIdpQdPJ9OeiomAUHLvxmPRSPH9m4s=
 github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
@@ -173,6 +196,8 @@ github.com/juju/errors v0.0.0-20180806074554-22422dad46e1/go.mod h1:W54LbzXuIE0b
 github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
 github.com/juju/testing v0.0.0-20190613124551-e81189438503/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kata-containers/govmm v0.0.0-20201020052039-99f43ec18864 h1:ETwjbdr9aU/J90P5D/HAxRW8M4r0HQSPmuBDIaNr9EM=
+github.com/kata-containers/govmm v0.0.0-20201020052039-99f43ec18864/go.mod h1:VmAHbsL5lLfzHW/MNL96NVLF840DNEV5i683kISgFKk=
 github.com/kata-containers/govmm v0.0.0-20210112013750-7d320e8f5dca h1:UdXFthwasAPnmv37gLJUEFsW9FaabYA+mM6FoSi8kzU=
 github.com/kata-containers/govmm v0.0.0-20210112013750-7d320e8f5dca/go.mod h1:VmAHbsL5lLfzHW/MNL96NVLF840DNEV5i683kISgFKk=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
@@ -185,6 +210,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 h1:2gxZ0XQIU/5z3Z3bUBu+FXuk2pFbkN6tcwi/pjyaDic=
 github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
@@ -198,14 +224,20 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b h1:Ey6yH0acn50T/v6CB75bGP4EMJqnv9WvnjN7oZaj+xE=
 github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw=
+github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a h1:KfNOeFvoAssuZLT7IntKZElKwi/5LRuxY71k+t6rfaM=
 github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
@@ -227,6 +259,8 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -258,7 +292,9 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
 github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
@@ -284,9 +320,11 @@ golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -296,10 +334,14 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191108221443-4ba9e2ef068c h1:SRpq/kuj/xNci/RdvEs+RSvpfxqvLAzTKuKGlzoGdZQ=
 golang.org/x/net v0.0.0-20191108221443-4ba9e2ef068c/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20191122200657-5d9234df094c h1:HjRaKPaiWks0f5tA6ELVF7ZfqSppfPwOEEAvsrKUTO4=
 golang.org/x/oauth2 v0.0.0-20191122200657-5d9234df094c/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -309,23 +351,34 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191105231009-c1f44814a5cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 h1:5/PjkGUjvEU5Gl6BxmvKRPpqo2uNMv4rcHBMwzk/st8=
+golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -359,6 +412,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
@@ -368,6 +423,18 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+k8s.io/apimachinery v0.18.2 h1:44CmtbmkzVDAhCpRVSiP2R5PPrC2RtlIv/MoB8xpdRA=
+k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA=
+k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
+k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E=
+sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
+sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
+sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
+sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

src/runtime/pkg/katautils/config-settings.go.in

@@ -56,6 +56,8 @@ const defaultVhostUserStorePath string = "/var/run/kata-containers/vhost-user/"
 const defaultRxRateLimiterMaxRate = uint64(0)
 const defaultTxRateLimiterMaxRate = uint64(0)
 
+var defaultSGXEPCSize = int64(0)
+
 const defaultTemplatePath string = "/run/vc/vm/template"
 const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock"
 

src/runtime/pkg/katautils/config.go

@@ -125,6 +125,8 @@ type hypervisor struct {
 	RxRateLimiterMaxRate    uint64   `toml:"rx_rate_limiter_max_rate"`
 	TxRateLimiterMaxRate    uint64   `toml:"tx_rate_limiter_max_rate"`
 	EnableAnnotations       []string `toml:"enable_annotations"`
+	GuestMemoryDumpPath     string   `toml:"guest_memory_dump_path"`
+	GuestMemoryDumpPaging   bool     `toml:"guest_memory_dump_paging"`
 }
 
 type runtime struct {
@@ -688,6 +690,8 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		RxRateLimiterMaxRate:    rxRateLimiterMaxRate,
 		TxRateLimiterMaxRate:    txRateLimiterMaxRate,
 		EnableAnnotations:       h.EnableAnnotations,
+		GuestMemoryDumpPath:     h.GuestMemoryDumpPath,
+		GuestMemoryDumpPaging:   h.GuestMemoryDumpPaging,
 	}, nil
 }
 
@@ -843,6 +847,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		PCIeRootPort:            h.PCIeRootPort,
 		DisableVhostNet:         true,
 		VirtioFSExtraArgs:       h.VirtioFSExtraArgs,
+		SGXEPCSize:              defaultSGXEPCSize,
 		EnableAnnotations:       h.EnableAnnotations,
 	}, nil
 }
@@ -1036,6 +1041,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
 		DisableImageNvdimm:      defaultDisableImageNvdimm,
 		RxRateLimiterMaxRate:    defaultRxRateLimiterMaxRate,
 		TxRateLimiterMaxRate:    defaultTxRateLimiterMaxRate,
+		SGXEPCSize:              defaultSGXEPCSize,
 	}
 }
 

src/runtime/pkg/katautils/config_test.go

@@ -83,6 +83,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 	disableNewNetNs := false
 	sharedFS := "virtio-9p"
 	virtioFSdaemon := path.Join(dir, "virtiofsd")
+	epcSize := int64(0)
 
 	configFileOptions := ktu.RuntimeConfigOptions{
 		Hypervisor:           "qemu",
@@ -165,6 +166,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		SharedFS:              sharedFS,
 		VirtioFSDaemon:        virtioFSdaemon,
 		VirtioFSCache:         defaultVirtioFSCacheMode,
+		SGXEPCSize:            epcSize,
 	}
 
 	agentConfig := vc.KataAgentConfig{

src/runtime/pkg/katautils/container_engine.go

@@ -7,6 +7,8 @@ package katautils
 
 import (
 	"os/exec"
+
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 )
 
 type CtrEngine struct {
@@ -19,7 +21,7 @@ var (
 
 func (e *CtrEngine) Init(name string) (string, error) {
 	var out string
-	out, err := RunCommandFull([]string{name, "version"}, true)
+	out, err := utils.RunCommandFull([]string{name, "version"}, true)
 	if err != nil {
 		return out, err
 	}
@@ -30,19 +32,19 @@ func (e *CtrEngine) Init(name string) (string, error) {
 
 func (e *CtrEngine) Inspect(image string) (string, error) {
 	// Only hit the network if the image doesn't exist locally
-	return RunCommand([]string{e.Name, "inspect", "--type=image", image})
+	return utils.RunCommand([]string{e.Name, "inspect", "--type=image", image})
 }
 
 func (e *CtrEngine) Pull(image string) (string, error) {
-	return RunCommand([]string{e.Name, "pull", image})
+	return utils.RunCommand([]string{e.Name, "pull", image})
 }
 
 func (e *CtrEngine) Create(image string) (string, error) {
-	return RunCommand([]string{e.Name, "create", image})
+	return utils.RunCommand([]string{e.Name, "create", image})
 }
 
 func (e *CtrEngine) Rm(ctrID string) (string, error) {
-	return RunCommand([]string{e.Name, "rm", ctrID})
+	return utils.RunCommand([]string{e.Name, "rm", ctrID})
 }
 
 func (e *CtrEngine) GetRootfs(ctrID string, dir string) error {

src/runtime/pkg/katautils/create_test.go

@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
@@ -87,7 +88,7 @@ func makeOCIBundle(bundleDir string) error {
 		}
 	}
 
-	output, err := RunCommandFull([]string{"cp", "-a", from, to}, true)
+	output, err := utils.RunCommandFull([]string{"cp", "-a", from, to}, true)
 	if err != nil {
 		return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output)
 	}

src/runtime/pkg/katautils/utils.go

@@ -8,13 +8,12 @@ package katautils
 
 import (
 	"fmt"
-	"golang.org/x/sys/unix"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path/filepath"
-	"strings"
 	"syscall"
+
+	"golang.org/x/sys/unix"
 )
 
 // FileExists test is a file exiting or not
@@ -110,27 +109,3 @@ func GetFileContents(file string) (string, error) {
 
 	return string(bytes), nil
 }
-
-// RunCommandFull returns the commands space-trimmed standard output and
-// error on success. Note that if the command fails, the requested output will
-// still be returned, along with an error.
-func RunCommandFull(args []string, includeStderr bool) (string, error) {
-	cmd := exec.Command(args[0], args[1:]...)
-	var err error
-	var bytes []byte
-
-	if includeStderr {
-		bytes, err = cmd.CombinedOutput()
-	} else {
-		bytes, err = cmd.Output()
-	}
-
-	trimmed := strings.TrimSpace(string(bytes))
-
-	return trimmed, err
-}
-
-// RunCommand returns the commands space-trimmed standard output on success
-func RunCommand(args []string) (string, error) {
-	return RunCommandFull(args, false)
-}

src/runtime/pkg/katautils/utils_test.go

@@ -18,6 +18,7 @@ import (
 	"syscall"
 	"testing"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/stretchr/testify/assert"
 )
@@ -90,7 +91,7 @@ func createOCIConfig(bundleDir string) error {
 		return errors.New("Cannot find command to generate OCI config file")
 	}
 
-	_, err := RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
+	_, err := utils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
 	if err != nil {
 		return err
 	}

src/runtime/pkg/utils/utils.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2020 Ant Financial
+// Copyright (c) 2020 Ant Group
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -6,7 +6,11 @@
 package utils
 
 import (
+	"fmt"
 	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
 	"strings"
 )
 
@@ -31,3 +35,48 @@ func GzipAccepted(header http.Header) bool {
 func String2Pointer(s string) *string {
 	return &s
 }
+
+// RunCommandFull returns the commands space-trimmed standard output and
+// error on success. Note that if the command fails, the requested output will
+// still be returned, along with an error.
+func RunCommandFull(args []string, includeStderr bool) (string, error) {
+	cmd := exec.Command(args[0], args[1:]...)
+	var err error
+	var bytes []byte
+
+	if includeStderr {
+		bytes, err = cmd.CombinedOutput()
+	} else {
+		bytes, err = cmd.Output()
+	}
+
+	trimmed := strings.TrimSpace(string(bytes))
+
+	return trimmed, err
+}
+
+// RunCommand returns the commands space-trimmed standard output on success
+func RunCommand(args []string) (string, error) {
+	return RunCommandFull(args, false)
+}
+
+// EnsureDir check if a directory exist, if not then create it
+func EnsureDir(path string, mode os.FileMode) error {
+	if !filepath.IsAbs(path) {
+		return fmt.Errorf("Not an absolute path: %s", path)
+	}
+
+	if fi, err := os.Stat(path); err != nil {
+		if os.IsNotExist(err) {
+			if err = os.MkdirAll(path, mode); err != nil {
+				return err
+			}
+		} else {
+			return err
+		}
+	} else if !fi.IsDir() {
+		return fmt.Errorf("Not a directory: %s", path)
+	}
+
+	return nil
+}

src/runtime/pkg/utils/utils_test.go

@@ -6,7 +6,10 @@
 package utils
 
 import (
+	"fmt"
+	"io/ioutil"
 	"net/http"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -45,3 +48,71 @@ func TestGzipAccepted(t *testing.T) {
 		assert.Equal(tc.result, b)
 	}
 }
+
+func TestEnsureDir(t *testing.T) {
+	const testMode = 0755
+	tmpdir, err := ioutil.TempDir("", "TestEnsureDir")
+	assert := assert.New(t)
+
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	testCases := []struct {
+		before func()
+		path   string
+		err    bool
+		msg    string
+	}{
+		{
+			before: nil,
+			path:   "a/b/c",
+			err:    true,
+			msg:    "Not an absolute path",
+		},
+		{
+			before: nil,
+			path:   fmt.Sprintf("%s/abc/def", tmpdir),
+			err:    false,
+			msg:    "",
+		},
+		{
+			before: nil,
+			path:   fmt.Sprintf("%s/abc", tmpdir),
+			err:    false,
+			msg:    "",
+		},
+		{
+			before: func() {
+				err := os.MkdirAll(fmt.Sprintf("%s/abc/def", tmpdir), testMode)
+				assert.NoError(err)
+			},
+			path: fmt.Sprintf("%s/abc/def", tmpdir),
+			err:  false,
+			msg:  "",
+		},
+		{
+			before: func() {
+				// create a regular file
+				err := os.MkdirAll(fmt.Sprintf("%s/abc", tmpdir), testMode)
+				assert.NoError(err)
+				_, err = os.Create(fmt.Sprintf("%s/abc/file.txt", tmpdir))
+				assert.NoError(err)
+			},
+			path: fmt.Sprintf("%s/abc/file.txt", tmpdir),
+			err:  true,
+			msg:  "Not a directory",
+		},
+	}
+
+	for _, tc := range testCases {
+		if tc.before != nil {
+			tc.before()
+		}
+		err := EnsureDir(tc.path, testMode)
+		if tc.err {
+			assert.Contains(err.Error(), tc.msg, "error msg should contains: %s, but got %s", tc.msg, err.Error())
+		} else {
+			assert.Equal(err, nil, "failed for path: %s, except no error, but got %+v", tc.path, err)
+		}
+	}
+}

src/runtime/vendor/github.com/pkg/errors/.travis.yml

@@ -1,15 +1,10 @@
 language: go
 go_import_path: github.com/pkg/errors
 go:
-  - 1.4.x
-  - 1.5.x
-  - 1.6.x
-  - 1.7.x
-  - 1.8.x
-  - 1.9.x
-  - 1.10.x
   - 1.11.x
+  - 1.12.x
+  - 1.13.x
   - tip
 
 script:
-  - go test -v ./...
+  - make check