Merge pull request #992 from bergwolf/2.0.0-branch-bump

# Kata Containers 2.0.0

.github/workflows/commit-message-check.yaml

@@ -10,7 +10,7 @@ env:
   error_msg: |+
     See the document below for help on formatting commits for the project.
 
-    https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md#patch-format
+    https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md#patch-forma
 
 jobs:
   commit-message-check:

.github/workflows/kata-deploy-test.yaml

@@ -1,12 +1,7 @@
-on:
-  issue_comment:
-    types: [created, edited]
-
+on: issue_comment
 name: test-kata-deploy
-
 jobs:
   check_comments:
-    if: ${{ github.event.issue.pull_request }}
     runs-on: ubuntu-latest
     steps:
       - name: Check for Command
@@ -14,7 +9,7 @@ jobs:
         uses: kata-containers/slash-command-action@v1
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-          command: "test_kata_deploy"
+          command: "test-kata-deploy"
           reaction: "true"
           reaction-type: "eyes"
           allow-edits: "false"
@@ -22,7 +17,6 @@ jobs:
       - name: verify command arg is kata-deploy
         run: |
            echo "The command was '${{ steps.command.outputs.command-name }}' with arguments '${{ steps.command.outputs.command-arguments }}'"
-
   create-and-test-container:
     needs: check_comments
     runs-on: ubuntu-latest
@@ -33,26 +27,22 @@ jobs:
             ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
             echo "reference for PR: " ${ref}
             echo "##[set-output name=pr-ref;]${ref}"
-
-      - name: check out
-        uses: actions/checkout@v2
+      - uses: actions/checkout@v2-beta
         with:
-           ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
-
+          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
       - name: build-container-image
         id: build-container-image
         run: |
             PR_SHA=$(git log --format=format:%H -n1)
-            VERSION="2.0.0"
+            VERSION=$(curl https://raw.githubusercontent.com/kata-containers/kata-containers/2.0-dev/VERSION)
             ARTIFACT_URL="https://github.com/kata-containers/kata-containers/releases/download/${VERSION}/kata-static-${VERSION}-x86_64.tar.xz"
-            wget "${ARTIFACT_URL}" -O tools/packaging/kata-deploy/kata-static.tar.xz
-            docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:${PR_SHA} ./tools/packaging/kata-deploy
+            wget "${ARTIFACT_URL}" -O ./kata-deploy/kata-static.tar.xz
+            docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:${PR_SHA} ./kata-deploy
             docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
             docker push katadocker/kata-deploy-ci:$PR_SHA
             echo "##[set-output name=pr-sha;]${PR_SHA}"
-
       - name: test-kata-deploy-ci-in-aks
-        uses: ./tools/packaging/kata-deploy/action
+        uses: ./kata-deploy/action
         with:
           packaging-sha: ${{ steps.build-container-image.outputs.pr-sha }}
         env:

.github/workflows/main.yaml

@@ -38,12 +38,12 @@ jobs:
         run: |
          if grep -q $buildstr ./artifact-list/artifact-list.txt; then
            $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::true
          else
-           echo "artifact-built=false" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::false
          fi
       - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
+        if: env.artifact-built == 'true'
         uses: actions/upload-artifact@master
         with:
           name: kata-artifacts
@@ -66,12 +66,12 @@ jobs:
         run: |
          if grep -q $buildstr ./artifact-list/artifact-list.txt; then
            $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::true
          else
-           echo "artifact-built=false" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::false
          fi
       - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
+        if: env.artifact-built == 'true'
         uses: actions/upload-artifact@master
         with:
           name: kata-artifacts
@@ -92,17 +92,70 @@ jobs:
         run: |
          if grep -q $buildstr ./artifact-list/artifact-list.txt; then
            $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::true
          else
-           echo "artifact-built=false" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::false
          fi
       - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
+        if: env.artifact-built == 'true'
         uses: actions/upload-artifact@master
         with:
           name: kata-artifacts
           path: kata-static-qemu.tar.gz
 
+  build-nemu:
+    runs-on: ubuntu-16.04
+    needs: get-artifact-list
+    env:
+      buildstr: "install_nemu"
+    steps:
+      - uses: actions/checkout@v1
+      - name: get-artifact-list
+        uses: actions/download-artifact@master
+        with:
+          name: artifact-list
+      - name: build-nemu
+        run: |
+         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
+           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
+           echo ::set-env name=artifact-built::true
+         else
+           echo ::set-env name=artifact-built::false
+         fi
+      - name: store-artifacts
+        if: env.artifact-built == 'true'
+        uses: actions/upload-artifact@master
+        with:
+          name: kata-artifacts
+          path: kata-static-nemu.tar.gz
+
+  # Job for building the QEMU binaries with virtiofs support
+  build-qemu-virtiofsd:
+    runs-on: ubuntu-16.04
+    needs: get-artifact-list
+    env:
+      buildstr: "install_qemu_virtiofsd"
+    steps:
+      - uses: actions/checkout@v1
+      - name: get-artifact-list
+        uses: actions/download-artifact@master
+        with:
+          name: artifact-list
+      - name: build-qemu-virtiofsd
+        run: |
+         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
+           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
+           echo ::set-env name=artifact-built::true
+         else
+           echo ::set-env name=artifact-built::false
+         fi
+      - name: store-artifacts
+        if: env.artifact-built == 'true'
+        uses: actions/upload-artifact@master
+        with:
+          name: kata-artifacts
+          path: kata-static-qemu-virtiofsd.tar.gz
+
   # Job for building the image
   build-image:
     runs-on: ubuntu-16.04
@@ -119,12 +172,12 @@ jobs:
         run: |
          if grep -q $buildstr ./artifact-list/artifact-list.txt; then
            $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::true
          else
-           echo "artifact-built=false" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::false
          fi
       - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
+        if: env.artifact-built == 'true'
         uses: actions/upload-artifact@master
         with:
           name: kata-artifacts
@@ -146,12 +199,12 @@ jobs:
         run: |
          if grep -q $buildstr ./artifact-list/artifact-list.txt; then
            $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::true
          else
-           echo "artifact-built=false" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::false
          fi
       - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
+        if: env.artifact-built == 'true'
         uses: actions/upload-artifact@master
         with:
           name: kata-artifacts
@@ -173,12 +226,12 @@ jobs:
         run: |
          if grep -q $buildstr ./artifact-list/artifact-list.txt; then
            $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::true
          else
-           echo "artifact-built=false" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::false
          fi
       - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
+        if: env.artifact-built == 'true'
         uses: actions/upload-artifact@master
         with:
           name: kata-artifacts
@@ -200,12 +253,12 @@ jobs:
         run: |
          if grep -q $buildstr ./artifact-list/artifact-list.txt; then
            $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::true
          else
-           echo "artifact-built=false" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::false
          fi
       - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
+        if: env.artifact-built == 'true'
         uses: actions/upload-artifact@master
         with:
           name: kata-artifacts
@@ -213,7 +266,7 @@ jobs:
 
   gather-artifacts:
     runs-on: ubuntu-16.04
-    needs: [build-experimental-kernel, build-kernel, build-qemu, build-image, build-firecracker, build-kata-components, build-clh]
+    needs: [build-experimental-kernel, build-kernel, build-qemu, build-qemu-virtiofsd, build-image, build-firecracker, build-kata-components, build-nemu, build-clh]
     steps:
       - uses: actions/checkout@v1
       - name: get-artifacts
@@ -250,7 +303,9 @@ jobs:
           docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:$pkg_sha ./packaging/kata-deploy
           docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
           docker push katadocker/kata-deploy-ci:$pkg_sha
-          echo "::set-output name=PKG_SHA::${pkg_sha}"
+
+          echo "##[set-output name=PKG_SHA;]${pkg_sha}"
+          echo ::set-env name=TAG::$tag
       - name: test-kata-deploy-ci-in-aks
         uses: ./packaging/kata-deploy/action
         with:

.github/workflows/release.yaml

@@ -39,12 +39,12 @@ jobs:
         run: |
          if grep -q $buildstr artifact-list.txt; then
            $GITHUB_WORKSPACE/.github/workflows/generate-local-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::true
          else
-           echo "artifact-built=false" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::false
          fi
       - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
+        if: env.artifact-built == 'true'
         uses: actions/upload-artifact@v2
         with:
           name: kata-artifacts
@@ -67,12 +67,12 @@ jobs:
         run: |
          if grep -q $buildstr artifact-list.txt; then
            $GITHUB_WORKSPACE/.github/workflows/generate-local-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::true
          else
-           echo "artifact-built=false" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::false
          fi
       - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
+        if: env.artifact-built == 'true'
         uses: actions/upload-artifact@v2
         with:
           name: kata-artifacts
@@ -93,17 +93,43 @@ jobs:
         run: |
          if grep -q $buildstr artifact-list.txt; then
            $GITHUB_WORKSPACE/.github/workflows/generate-local-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::true
          else
-           echo "artifact-built=false" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::false
          fi
       - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
+        if: env.artifact-built == 'true'
         uses: actions/upload-artifact@v2
         with:
           name: kata-artifacts
           path: kata-static-qemu.tar.gz
 
+  build-qemu-virtiofsd:
+    runs-on: ubuntu-16.04
+    needs: get-artifact-list
+    env:
+      buildstr: "install_qemu_virtiofsd"
+    steps:
+      - uses: actions/checkout@v2
+      - name: get-artifact-list
+        uses: actions/download-artifact@v2
+        with:
+          name: artifact-list
+      - name: build-qemu-virtiofsd
+        run: |
+         if grep -q $buildstr artifact-list.txt; then
+           $GITHUB_WORKSPACE/.github/workflows/generate-local-artifact-tarball.sh $buildstr
+           echo ::set-env name=artifact-built::true
+         else
+           echo ::set-env name=artifact-built::false
+         fi
+      - name: store-artifacts
+        if: env.artifact-built == 'true'
+        uses: actions/upload-artifact@v2
+        with:
+          name: kata-artifacts
+          path: kata-static-qemu-virtiofsd.tar.gz
+
   build-image:
     runs-on: ubuntu-16.04
     needs: get-artifact-list
@@ -119,12 +145,12 @@ jobs:
         run: |
          if grep -q $buildstr artifact-list.txt; then
            $GITHUB_WORKSPACE/.github/workflows/generate-local-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::true
          else
-           echo "artifact-built=false" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::false
          fi
       - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
+        if: env.artifact-built == 'true'
         uses: actions/upload-artifact@v2
         with:
           name: kata-artifacts
@@ -145,12 +171,12 @@ jobs:
         run: |
          if grep -q $buildstr artifact-list.txt; then
            $GITHUB_WORKSPACE/.github/workflows/generate-local-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::true
          else
-           echo "artifact-built=false" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::false
          fi
       - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
+        if: env.artifact-built == 'true'
         uses: actions/upload-artifact@v2
         with:
           name: kata-artifacts
@@ -172,12 +198,12 @@ jobs:
         run: |
          if grep -q $buildstr artifact-list.txt; then
            $GITHUB_WORKSPACE/.github/workflows/generate-local-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::true
          else
-           echo "artifact-built=false" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::false
          fi
       - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
+        if: env.artifact-built == 'true'
         uses: actions/upload-artifact@v2
         with:
           name: kata-artifacts
@@ -198,12 +224,12 @@ jobs:
         run: |
          if grep -q $buildstr artifact-list.txt; then
            $GITHUB_WORKSPACE/.github/workflows/generate-local-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::true
          else
-           echo "artifact-built=false" >> $GITHUB_ENV
+           echo ::set-env name=artifact-built::false
          fi
       - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
+        if: env.artifact-built == 'true'
         uses: actions/upload-artifact@v2
         with:
           name: kata-artifacts
@@ -211,7 +237,7 @@ jobs:
 
   gather-artifacts:
     runs-on: ubuntu-16.04
-    needs: [build-experimental-kernel, build-kernel, build-qemu, build-image, build-firecracker, build-kata-components, build-clh]
+    needs: [build-experimental-kernel, build-kernel, build-qemu, build-qemu-virtiofsd, build-image, build-firecracker, build-kata-components, build-clh]
     steps:
       - uses: actions/checkout@v2
       - name: get-artifacts
@@ -249,9 +275,11 @@ jobs:
           docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
           docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
           docker push katadocker/kata-deploy-ci:$pkg_sha
+
+          echo "##[set-output name=PKG_SHA;]${pkg_sha}"
+          echo ::set-env name=TAG::$tag
           mkdir -p packaging/kata-deploy
           ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
-          echo "::set-output name=PKG_SHA::${pkg_sha}"
       - name: test-kata-deploy-ci-in-aks
         uses: ./packaging/kata-deploy/action
         with:

.github/workflows/snap-release.yaml

@@ -1,37 +0,0 @@
-name: Release Kata 2.x in snapcraft store
-on:
-  push:
-    tags:
-      - '2.*'
-jobs:
-  release-snap:
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Check out Git repository
-        uses: actions/checkout@v2
-
-      - name: Install Snapcraft
-        uses: samuelmeuli/action-snapcraft@v1
-        with:
-          snapcraft_token: ${{ secrets.snapcraft_token }}
-
-      - name: Build snap
-        run: |
-          sudo apt-get install -y git git-extras
-          kata_url="https://github.com/kata-containers/kata-containers"
-          latest_version=$(git ls-remote --tags ${kata_url}  | egrep -o "refs.*" | egrep -v "\-alpha|\-rc|{}" | egrep -o "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+" | sort -V -r | head -1)
-          current_version="$(echo ${GITHUB_REF} | cut -d/ -f3)"
-          # Check semantic versioning format (x.y.z) and if the current tag is the latest tag
-          if echo "${current_version}" | grep -q "^[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+$" && echo -e "$latest_version\n$current_version" | sort -C -V; then
-            # Current version is the latest version, build it
-            snapcraft -d snap --destructive-mode
-          fi
-
-      - name: Upload snap
-        run: |
-          snap_version="$(echo ${GITHUB_REF} | cut -d/ -f3)"
-          snap_file="kata-containers_${snap_version}_amd64.snap"
-          # Upload the snap if it exists
-          if [ -f ${snap_file} ]; then
-            snapcraft upload --release=candidate ${snap_file}
-          fi

.github/workflows/snap.yaml

@@ -1,5 +1,15 @@
 name: snap CI
-on: ["pull_request"]
+on:
+  pull_request:
+    paths:
+      - "**/Makefile"
+      - "**/*.go"
+      - "**/*.mk"
+      - "**/*.rs"
+      - "**/*.sh"
+      - "**/*.toml"
+      - "**/*.yaml"
+      - "**/*.yml"
 jobs:
   test:
     runs-on: ubuntu-20.04

.github/workflows/static-checks.yaml

@@ -1,66 +0,0 @@
-on: ["pull_request"]
-name: Static checks
-jobs:
-  test:
-    strategy:
-      matrix:
-        go-version: [1.13.x, 1.14.x, 1.15.x]
-        os: [ubuntu-20.04]
-    runs-on: ${{ matrix.os }}
-    env:
-      TRAVIS: "true"
-      TRAVIS_BRANCH: ${{ github.base_ref }}
-      TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }}
-      TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }}
-      RUST_BACKTRACE: "1"
-      target_branch: ${TRAVIS_BRANCH}
-    steps:
-    - name: Install Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: ${{ matrix.go-version }}
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Setup GOPATH
-      run: |
-        echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}"
-        echo "TRAVIS: ${TRAVIS}"
-    - name: Set env
-      run: |
-        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
-    - name: Checkout code
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-        path: ./src/github.com/${{ github.repository }}
-    - name: Setup travis references
-      run: |
-        echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}"
-        target_branch=${TRAVIS_BRANCH}
-    - name: Setup
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Building rust
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh
-        PATH=$PATH:"$HOME/.cargo/bin"
-        rustup target add x86_64-unknown-linux-musl
-        rustup component add rustfmt clippy
-    # Must build before static checks as we depend on some generated code in runtime and agent
-    - name: Build
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make
-    - name: Static Checks
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/static-checks.sh
-    - name: Run Compiler Checks
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make check
-    - name: Run Unit Tests
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make test

.gitignore

@@ -1,12 +1,7 @@
 **/*.bk
-**/*~
 **/*.orig
 **/*.rej
 **/target
 **/.vscode
-pkg/logging/Cargo.lock
 src/agent/src/version.rs
 src/agent/kata-agent.service
-src/agent/protocols/src/*.rs
-!src/agent/protocols/src/lib.rs
-

.travis.yml

@@ -0,0 +1,62 @@
+# Copyright (c) 2019 Ant Financial
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+dist: bionic
+os: linux
+
+# set cache directories manually, because
+# we are using a non-standard directory struct
+# cargo root is in srs/agent
+#
+# If needed, caches can be cleared
+# by ways documented in
+# https://docs.travis-ci.com/user/caching#clearing-caches
+language: rust
+rust:
+  - 1.44.1
+cache:
+  cargo: true
+  directories:
+    - src/agent/target
+
+before_install:
+  - git remote set-branches --add origin "${TRAVIS_BRANCH}"
+  - git fetch
+  - export RUST_BACKTRACE=1
+  - export target_branch=$TRAVIS_BRANCH
+  - "ci/setup.sh"
+
+# we use install to run check agent
+# so that it is easy to skip for non-amd64 platform
+install:
+  - export PATH=$PATH:"$HOME/.cargo/bin"
+  - export RUST_AGENT=yes
+  - rustup target add x86_64-unknown-linux-musl
+  - sudo ln -sf /usr/bin/g++ /bin/musl-g++
+  - rustup component add rustfmt
+  - make -C ${TRAVIS_BUILD_DIR}/src/agent
+  - make -C ${TRAVIS_BUILD_DIR}/src/agent check
+  - sudo -E PATH=$PATH make -C ${TRAVIS_BUILD_DIR}/src/agent check
+
+before_script:
+  - "ci/install_go.sh"
+  - make -C ${TRAVIS_BUILD_DIR}/src/runtime
+  - make -C ${TRAVIS_BUILD_DIR}/src/runtime test
+  - sudo -E PATH=$PATH GOPATH=$GOPATH make -C ${TRAVIS_BUILD_DIR}/src/runtime test
+
+script:
+  - "ci/static-checks.sh"
+
+jobs:
+  include:
+  - name: x86_64 test
+    os: linux
+  - name: ppc64le test
+    os: linux-ppc64le
+    install: skip
+    script: skip
+  allow_failures:
+    - name: ppc64le test
+  fast_finish: true

README.md

@@ -2,143 +2,136 @@
 
 # Kata Containers
 
-* [Kata Containers](#kata-containers)
-    * [Introduction](#introduction)
-    * [Getting started](#getting-started)
-    * [Documentation](#documentation)
+* [Raising issues](#raising-issues)
+* [Kata Containers repositories](#kata-containers-repositories)
+    * [Code Repositories](#code-repositories)
+        * [Kata Containers-developed components](#kata-containers-developed-components)
+            * [Agent](#agent)
+            * [KSM throttler](#ksm-throttler)
+            * [Runtime](#runtime)
+            * [Trace forwarder](#trace-forwarder)
+        * [Additional](#additional)
+            * [Hypervisor](#hypervisor)
+            * [Kernel](#kernel)
+    * [CI](#ci)
     * [Community](#community)
-    * [Getting help](#getting-help)
-        * [Raising issues](#raising-issues)
-            * [Kata Containers 1.x versions](#kata-containers-1x-versions)
-    * [Developers](#developers)
-        * [Components](#components)
-            * [Kata Containers 1.x components](#kata-containers-1x-components)
-        * [Common repositories](#common-repositories)
-        * [Packaging and releases](#packaging-and-releases)
+    * [Documentation](#documentation)
+    * [Packaging](#packaging)
+    * [Test code](#test-code)
+    * [Utilities](#utilities)
+        * [OS builder](#os-builder)
+    * [Web content](#web-content)
 
 ---
 
 Welcome to Kata Containers!
 
-This repository is the home of the Kata Containers code for the 2.0 and newer
-releases.
+The purpose of this repository is to act as a "top level" site for the project. Specifically it is used:
 
-If you want to learn about Kata Containers, visit the main
-[Kata Containers website](https://katacontainers.io).
+- To provide a list of the various *other* [Kata Containers repositories](#kata-containers-repositories),
+  along with a brief explanation of their purpose.
 
-For further details on the older (first generation) Kata Containers 1.x
-versions, see the
-[Kata Containers 1.x components](#kata-containers-1x-components)
-section.
+- To provide a general area for [Raising Issues](#raising-issues).
 
-## Introduction
+## Raising issues
 
-Kata Containers is an open source project and community working to build a
-standard implementation of lightweight Virtual Machines (VMs) that feel and
-perform like containers, but provide the workload isolation and security
-advantages of VMs.
+This repository is used for [raising
+issues](https://github.com/kata-containers/kata-containers/issues/new):
 
-## Getting started
+- That might affect multiple code repositories.
 
-See the [installation documentation](docs/install).
-
-## Documentation
-
-See the [official documentation](docs)
-(including [installation guides](docs/install),
-[the developer guide](docs/Developer-Guide.md),
-[design documents](docs/design) and more).
-
-## Community
-
-To learn more about the project, its community and governance, see the
-[community repository](https://github.com/kata-containers/community). This is
-the first place to go if you wish to contribute to the project.
-
-## Getting help
-
-See the [community](#community) section for ways to contact us.
-
-### Raising issues
-
-Please raise an issue
-[in this repository](https://github.com/kata-containers/kata-containers/issues).
+- Where the raiser is unsure which repositories are affected.
 
 > **Note:**
-> If you are reporting a security issue, please follow the [vulnerability reporting process](https://github.com/kata-containers/community#vulnerability-handling)
+> 
+> - If an issue affects only a single component, it should be raised in that
+>   components repository.
 
-#### Kata Containers 1.x versions
+## Kata Containers repositories
 
-For older Kata Containers 1.x releases, please raise an issue in the
-[Kata Containers 1.x component repository](#kata-containers-1x-components)
-that seems most appropriate.
+### CI
 
-If in doubt, raise an issue
-[in the Kata Containers 1.x runtime repository](https://github.com/kata-containers/runtime/issues).
+The [CI](https://github.com/kata-containers/ci) repository stores the Continuous
+Integration (CI) system configuration information.
 
-## Developers
+### Community
 
-### Components
+The [Community](https://github.com/kata-containers/community) repository is
+the first place to go if you want to use or contribute to the project.
 
-| Component | Type | Description |
-|-|-|-|
-| [agent-ctl](tools/agent-ctl) | utility | Tool that provides low-level access for testing the agent. |
-| [agent](src/agent) | core | Management process running inside the virtual machine / POD that sets up the container environment. |
-| [documentation](docs) | documentation | Documentation common to all components (such as design and install documentation). |
-| [osbuilder](tools/osbuilder) | infrastructure | Tool to create "mini O/S" rootfs and initrd images for the hypervisor. |
-| [packaging](tools/packaging) | infrastructure | Scripts and metadata for producing packaged binaries<br/>(components, hypervisors, kernel and rootfs). |
-| [runtime](src/runtime) | core | Main component run by a container manager and providing a containerd shimv2 runtime implementation. |
-| [trace-forwarder](src/trace-forwarder) | utility | Agent tracing helper. |
+### Code Repositories
 
-#### Kata Containers 1.x components
+#### Kata Containers-developed components
 
-For the first generation of Kata Containers (1.x versions), each component was
-kept in a separate repository.
+##### Agent
 
-For information on the Kata Containers 1.x releases, see the
-[Kata Containers 1.x releases page](https://github.com/kata-containers/runtime/releases).
+The [`kata-agent`](src/agent/README.md) runs inside the
+virtual machine and sets up the container environment.
 
-For further information on particular Kata Containers 1.x components, see the
-individual component repositories:
+##### KSM throttler
 
-| Component | Type | Description |
-|-|-|-|
-| [agent](https://github.com/kata-containers/agent) | core | See [components](#components). |
-| [documentation](https://github.com/kata-containers/documentation) | documentation | |
-| [KSM throttler](https://github.com/kata-containers/ksm-throttler) | optional core | Daemon that monitors containers and deduplicates memory to maximize container density on the host. |
-| [osbuilder](https://github.com/kata-containers/osbuilder) | infrastructure | See [components](#components). |
-| [packaging](https://github.com/kata-containers/packaging) | infrastructure | See [components](#components). |
-| [proxy](https://github.com/kata-containers/proxy) | core | Multiplexes communications between the shims, agent and runtime. |
-| [runtime](https://github.com/kata-containers/runtime) | core | See [components](#components). |
-| [shim](https://github.com/kata-containers/shim) | core | Handles standard I/O and signals on behalf of the container process. |
+The [`kata-ksm-throttler`](https://github.com/kata-containers/ksm-throttler)
+is an optional utility that monitors containers and deduplicates memory to
+maximize container density on a host.
 
-> **Note:**
->
-> - There are more components for the original Kata Containers 1.x implementation.
-> - The current implementation simplifies the design significantly:
->   compare the [current](docs/design/architecture.md) and
->   [previous generation](https://github.com/kata-containers/documentation/blob/master/design/architecture.md)
->   designs.
+##### Runtime
 
-### Common repositories
+The [`kata-runtime`](src/runtime/README.md) is usually
+invoked by a container manager and provides high-level verbs to manage
+containers.
 
-The following repositories are used by both the current and first generation Kata Containers implementations:
+##### Trace forwarder
 
-| Component | Description | Current | First generation | Notes |
-|-|-|-|-|-|
-| CI | Continuous Integration configuration files and scripts. | [Kata 2.x](https://github.com/kata-containers/ci/tree/main) | [Kata 1.x](https://github.com/kata-containers/ci/tree/master) | |
-| kernel | The Linux kernel used by the hypervisor to boot the guest image. | [Kata 2.x][kernel] | [Kata 1.x][kernel] | Patches are stored in the packaging component. |
-| tests | Test code. | [Kata 2.x](https://github.com/kata-containers/tests/tree/main) | [Kata 1.x](https://github.com/kata-containers/tests/tree/master) | Excludes unit tests which live with the main code. |
-| www.katacontainers.io | Contains the source for the [main web site](https://www.katacontainers.io). | [Kata 2.x][github-katacontainers.io] | [Kata 1.x][github-katacontainers.io] | | |
+The [`kata-trace-forwarder`](src/trace-forwarder) is a component only used
+when tracing the [agent](#agent) process.
 
-### Packaging and releases
+#### Additional
 
-Kata Containers is now
-[available natively for most distributions](docs/install/README.md#packaged-installation-methods).
-However, packaging scripts and metadata are still used to generate snap and GitHub releases. See
-the [components](#components) section for further details.
+##### Hypervisor
 
----
+The [`qemu`](https://github.com/kata-containers/qemu) hypervisor is used to
+create virtual machines for hosting the containers.
 
-[kernel]: https://www.kernel.org
-[github-katacontainers.io]: https://github.com/kata-containers/www.katacontainers.io
+##### Kernel
+
+The hypervisor uses a [Linux\* kernel](https://github.com/kata-containers/linux) to boot the guest image.
+
+### Documentation
+
+The [docs](docs/README.md) directory holds documentation common to all code components.
+
+### Packaging
+
+We use the [packaging](tools/packaging/README.md) to create packages for the [system
+components](#kata-containers-developed-components) including
+[rootfs](#os-builder) and [kernel](#kernel) images.
+
+### Test code
+
+The [tests](https://github.com/kata-containers/tests) repository hosts all
+test code except the unit testing code (which is kept in the same repository
+as the component it tests).
+
+### Utilities
+
+#### OS builder
+
+The [osbuilder](tools/osbuilder/README.md) tool can create
+a rootfs and a "mini O/S" image. This image is used by the hypervisor to setup
+the environment before switching to the workload.
+
+#### `kata-agent-ctl`
+
+[`kata-agent-ctl`](tools/agent-ctl) is a low-level test tool for
+interacting with the agent.
+
+### Web content
+
+The
+[www.katacontainers.io](https://github.com/kata-containers/www.katacontainers.io)
+repository contains all sources for the https://www.katacontainers.io site.
+
+## Credits
+
+Kata Containers uses [packagecloud](https://packagecloud.io) for package
+hosting.

VERSION

@@ -1 +1 @@
-2.1.0
+2.0.0

ci/install_musl.sh

@@ -12,11 +12,10 @@ install_aarch64_musl() {
 		local musl_tar="${arch}-linux-musl-native.tgz"
 		local musl_dir="${arch}-linux-musl-native"
 		pushd /tmp
-		if curl -sLO --fail https://musl.cc/${musl_tar}; then
-			tar -zxf ${musl_tar}
-			mkdir -p /usr/local/musl/
-			cp -r ${musl_dir}/* /usr/local/musl/
-		fi
+		curl -sLO https://musl.cc/${musl_tar}
+		tar -zxf ${musl_tar}
+		mkdir -p /usr/local/musl/
+		cp -r ${musl_dir}/* /usr/local/musl/
 		popd
 	fi
 }

ci/install_yq.sh

@@ -18,9 +18,7 @@ function install_yq() {
 	GOPATH=${GOPATH:-${HOME}/go}
 	local yq_path="${GOPATH}/bin/yq"
 	local yq_pkg="github.com/mikefarah/yq"
-	local yq_version=3.4.1
-
-	[ -x  "${GOPATH}/bin/yq" ] && [ "`${GOPATH}/bin/yq --version`"X == "yq version ${yq_version}"X ] && return
+	[ -x  "${GOPATH}/bin/yq" ] && return
 
 	read -r -a sysInfo <<< "$(uname -sm)"
 
@@ -58,6 +56,8 @@ function install_yq() {
 		die "Please install curl"
 	fi
 
+	local yq_version=3.1.0
+
 	## NOTE: ${var,,} => gives lowercase value of var
 	local yq_url="https://${yq_pkg}/releases/download/${yq_version}/yq_${goos,,}_${goarch}"
 	curl -o "${yq_path}" -LSsf "${yq_url}"

ci/lib.sh

@@ -5,27 +5,18 @@
 
 export tests_repo="${tests_repo:-github.com/kata-containers/tests}"
 export tests_repo_dir="$GOPATH/src/$tests_repo"
-export branch="${branch:-main}"
+export branch="${branch:-2.0-dev}"
 
-# Clones the tests repository and checkout to the branch pointed out by
-# the global $branch variable.
-# If the clone exists and `CI` is exported then it does nothing. Otherwise
-# it will clone the repository or `git pull` the latest code.
-#
 clone_tests_repo()
 {
-	if [ -d "$tests_repo_dir" ]; then
-		[ -n "$CI" ] && return
-		pushd "${tests_repo_dir}"
-		git checkout "${branch}"
-		git pull
-		popd
-	else
-		git clone -q "https://${tests_repo}" "$tests_repo_dir"
-		pushd "${tests_repo_dir}"
-		git checkout "${branch}"
-		popd
+	if [ -d "$tests_repo_dir" -a -n "$CI" ]
+	then
+		return
 	fi
+
+	go get -d -u "$tests_repo" || true
+
+	pushd "${tests_repo_dir}" && git checkout "${branch}" && popd
 }
 
 run_static_checks()

ci/openshift-ci/images/Dockerfile.buildroot

@@ -1,9 +0,0 @@
-# Copyright (c) 2021 Red Hat, Inc.
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-# This is the build root image for Kata Containers on OpenShift CI.
-#
-FROM centos:8
-
-RUN yum -y update && yum -y install git sudo wget

docs/Developer-Guide.md

@@ -1,54 +1,55 @@
-- [Warning](#warning)
-- [Assumptions](#assumptions)
-- [Initial setup](#initial-setup)
-- [Requirements to build individual components](#requirements-to-build-individual-components)
-- [Build and install the Kata Containers runtime](#build-and-install-the-kata-containers-runtime)
-- [Check hardware requirements](#check-hardware-requirements)
-  - [Configure to use initrd or rootfs image](#configure-to-use-initrd-or-rootfs-image)
-  - [Enable full debug](#enable-full-debug)
-    - [debug logs and shimv2](#debug-logs-and-shimv2)
-      - [Enabling full `containerd` debug](#enabling-full-containerd-debug)
-      - [Enabling just `containerd shim` debug](#enabling-just-containerd-shim-debug)
-      - [Enabling `CRI-O` and `shimv2` debug](#enabling-cri-o-and-shimv2-debug)
-    - [journald rate limiting](#journald-rate-limiting)
-      - [`systemd-journald` suppressing messages](#systemd-journald-suppressing-messages)
-      - [Disabling `systemd-journald` rate limiting](#disabling-systemd-journald-rate-limiting)
-- [Create and install rootfs and initrd image](#create-and-install-rootfs-and-initrd-image)
-  - [Build a custom Kata agent - OPTIONAL](#build-a-custom-kata-agent---optional)
-  - [Get the osbuilder](#get-the-osbuilder)
-  - [Create a rootfs image](#create-a-rootfs-image)
-    - [Create a local rootfs](#create-a-local-rootfs)
-    - [Add a custom agent to the image - OPTIONAL](#add-a-custom-agent-to-the-image---optional)
-    - [Build a rootfs image](#build-a-rootfs-image)
-    - [Install the rootfs image](#install-the-rootfs-image)
-  - [Create an initrd image - OPTIONAL](#create-an-initrd-image---optional)
-    - [Create a local rootfs for initrd image](#create-a-local-rootfs-for-initrd-image)
-    - [Build an initrd image](#build-an-initrd-image)
-    - [Install the initrd image](#install-the-initrd-image)
-- [Install guest kernel images](#install-guest-kernel-images)
-- [Install a hypervisor](#install-a-hypervisor)
-  - [Build a custom QEMU](#build-a-custom-qemu)
-    - [Build a custom QEMU for aarch64/arm64 - REQUIRED](#build-a-custom-qemu-for-aarch64arm64---required)
-- [Run Kata Containers with Containerd](#run-kata-containers-with-containerd)
-- [Run Kata Containers with Kubernetes](#run-kata-containers-with-kubernetes)
-- [Troubleshoot Kata Containers](#troubleshoot-kata-containers)
-- [Appendices](#appendices)
-  - [Checking Docker default runtime](#checking-docker-default-runtime)
-  - [Set up a debug console](#set-up-a-debug-console)
-    - [Simple debug console setup](#simple-debug-console-setup)
-      - [Enable agent debug console](#enable-agent-debug-console)
-      - [Connect to debug console](#connect-to-debug-console)
-    - [Traditional debug console setup](#traditional-debug-console-setup)
-      - [Create a custom image containing a shell](#create-a-custom-image-containing-a-shell)
-      - [Build the debug image](#build-the-debug-image)
-      - [Configure runtime for custom debug image](#configure-runtime-for-custom-debug-image)
-      - [Create a container](#create-a-container)
-      - [Connect to the virtual machine using the debug console](#connect-to-the-virtual-machine-using-the-debug-console)
-        - [Enabling debug console for QEMU](#enabling-debug-console-for-qemu)
-        - [Enabling debug console for cloud-hypervisor / firecracker](#enabling-debug-console-for-cloud-hypervisor--firecracker)
-        - [Connecting to the debug console](#connecting-to-the-debug-console)
-  - [Obtain details of the image](#obtain-details-of-the-image)
-  - [Capturing kernel boot logs](#capturing-kernel-boot-logs)
+* [Warning](#warning)
+* [Assumptions](#assumptions)
+* [Initial setup](#initial-setup)
+* [Requirements to build individual components](#requirements-to-build-individual-components)
+* [Build and install the Kata Containers runtime](#build-and-install-the-kata-containers-runtime)
+* [Check hardware requirements](#check-hardware-requirements)
+    * [Configure to use initrd or rootfs image](#configure-to-use-initrd-or-rootfs-image)
+    * [Enable full debug](#enable-full-debug)
+        * [debug logs and shimv2](#debug-logs-and-shimv2)
+            * [Enabling full `containerd` debug](#enabling-full-containerd-debug)
+            * [Enabling just `containerd shim` debug](#enabling-just-containerd-shim-debug)
+            * [Enabling `CRI-O` and `shimv2` debug](#enabling-cri-o-and-shimv2-debug)
+        * [journald rate limiting](#journald-rate-limiting)
+            * [`systemd-journald` suppressing messages](#systemd-journald-suppressing-messages)
+            * [Disabling `systemd-journald` rate limiting](#disabling-systemd-journald-rate-limiting)
+* [Create and install rootfs and initrd image](#create-and-install-rootfs-and-initrd-image)
+    * [Build a custom Kata agent - OPTIONAL](#build-a-custom-kata-agent---optional)
+    * [Get the osbuilder](#get-the-osbuilder)
+    * [Create a rootfs image](#create-a-rootfs-image)
+        * [Create a local rootfs](#create-a-local-rootfs)
+        * [Add a custom agent to the image - OPTIONAL](#add-a-custom-agent-to-the-image---optional)
+        * [Build a rootfs image](#build-a-rootfs-image)
+        * [Install the rootfs image](#install-the-rootfs-image)
+    * [Create an initrd image - OPTIONAL](#create-an-initrd-image---optional)
+        * [Create a local rootfs for initrd image](#create-a-local-rootfs-for-initrd-image)
+        * [Build an initrd image](#build-an-initrd-image)
+        * [Install the initrd image](#install-the-initrd-image)
+* [Install guest kernel images](#install-guest-kernel-images)
+* [Install a hypervisor](#install-a-hypervisor)
+    * [Build a custom QEMU](#build-a-custom-qemu)
+        * [Build a custom QEMU for aarch64/arm64 - REQUIRED](#build-a-custom-qemu-for-aarch64arm64---required)
+* [Run Kata Containers with Containerd](#run-kata-containers-with-containerd)
+* [Run Kata Containers with Kubernetes](#run-kata-containers-with-kubernetes)
+* [Troubleshoot Kata Containers](#troubleshoot-kata-containers)
+* [Appendices](#appendices)
+    * [Checking Docker default runtime](#checking-docker-default-runtime)
+    * [Set up a debug console](#set-up-a-debug-console)
+      * [Simple debug console setup](#simple-debug-console-setup)
+          * [Enable agent debug console](#enable-agent-debug-console)
+          * [Start `kata-monitor`](#start-kata-monitor)
+          * [Connect to debug console](#connect-to-debug-console)
+      * [Traditional debug console setup](#traditional-debug-console-setup)
+          * [Create a custom image containing a shell](#create-a-custom-image-containing-a-shell)
+          * [Build the debug image](#build-the-debug-image)
+          * [Configure runtime for custom debug image](#configure-runtime-for-custom-debug-image)
+          * [Connect to the virtual machine using the debug console](#connect-to-the-virtual-machine-using-the-debug-console)
+              * [Enabling debug console for QEMU](#enabling-debug-console-for-qemu)
+              * [Enabling debug console for cloud-hypervisor / firecracker](#enabling-debug-console-for-cloud-hypervisor--firecracker)
+          * [Create a container](#create-a-container)
+          * [Connect to the virtual machine using the debug console](#connect-to-the-virtual-machine-using-the-debug-console)
+    * [Obtain details of the image](#obtain-details-of-the-image)
+    * [Capturing kernel boot logs](#capturing-kernel-boot-logs)
 
 # Warning
 
@@ -103,7 +104,7 @@ The build will create the following:
 You can check if your system is capable of creating a Kata Container by running the following:
 
 ```
-$ sudo kata-runtime check
+$ sudo kata-runtime kata-check
 ```
 
 If your system is *not* able to run Kata Containers, the previous command will error out and explain why.
@@ -353,12 +354,9 @@ You MUST choose one of `alpine`, `centos`, `clearlinux`, `euleros`, and `fedora`
 >
 > - Check the [compatibility matrix](../tools/osbuilder/README.md#platform-distro-compatibility-matrix) before creating rootfs.
 
-Optionally, add your custom agent binary to the rootfs with the following, `LIBC` default is `musl`, if `ARCH` is `ppc64le`, should set the `LIBC=gnu` and `ARCH=powerpc64le`:
+Optionally, add your custom agent binary to the rootfs with the following:
 ```
-$ export ARCH=$(shell uname -m)
-$ [ ${ARCH} == "ppc64le" ] && export LIBC=gnu || export LIBC=musl
-$ [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
-$ sudo install -o root -g root -m 0550 -T ../../../src/agent/target/$(ARCH)-unknown-linux-$(LIBC)/release/kata-agent ${ROOTFS_DIR}/sbin/init
+$ sudo install -o root -g root -m 0550 -T ../../agent/kata-agent ${ROOTFS_DIR}/sbin/init
 ```
 
 ### Build an initrd image
@@ -384,30 +382,31 @@ You can build and install the guest kernel image as shown [here](../tools/packag
 
 # Install a hypervisor
 
-When setting up Kata using a [packaged installation method](install/README.md#installing-on-a-linux-system), the
-`QEMU` VMM is installed automatically. Cloud-Hypervisor and Firecracker VMMs are available from the [release tarballs](https://github.com/kata-containers/kata-containers/releases), as well as through [`kata-deploy`](../tools/packaging/kata-deploy/README.md).
-You may choose to manually build your VMM/hypervisor.
+When setting up Kata using a [packaged installation method](install/README.md#installing-on-a-linux-system), the `qemu-lite` hypervisor is installed automatically. For other installation methods, you will need to manually install a suitable hypervisor.
 
 ## Build a custom QEMU
 
-Kata Containers makes use of upstream QEMU branch. The exact version
-and repository utilized can be found by looking at the [versions file](../versions.yaml).
+Your QEMU directory need to be prepared with source code. Alternatively, you can use the [Kata containers QEMU](https://github.com/kata-containers/qemu/tree/master) and checkout the recommended branch:
 
-Kata often utilizes patches for not-yet-upstream fixes for components,
-including QEMU. These can be found in the [packaging/QEMU directory](../tools/packaging/qemu/patches)
+```
+$ go get -d github.com/kata-containers/qemu
+$ qemu_branch=$(grep qemu-lite- ${GOPATH}/src/github.com/kata-containers/kata-containers/versions.yaml | cut -d '"' -f2)
+$ cd ${GOPATH}/src/github.com/kata-containers/qemu
+$ git checkout -b $qemu_branch remotes/origin/$qemu_branch
+$ your_qemu_directory=${GOPATH}/src/github.com/kata-containers/qemu
+```
+
+To build a version of QEMU using the same options as the default `qemu-lite` version , you could use the `configure-hypervisor.sh` script:
 
-To build utilizing the same options as Kata, you should make use of the `configure-hypervisor.sh` script. For example:
 ```
 $ go get -d github.com/kata-containers/kata-containers/tools/packaging
 $ cd $your_qemu_directory
-$ ${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging/scripts/configure-hypervisor.sh kata-qemu > kata.cfg
+$ ${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging/scripts/configure-hypervisor.sh qemu > kata.cfg
 $ eval ./configure "$(cat kata.cfg)"
 $ make -j $(nproc)
 $ sudo -E make install
 ```
 
-See the [static-build script for QEMU](../tools/packaging/static-build/qemu/build-static-qemu.sh) for a reference on how to get, setup, configure and build QEMU for Kata.
-
 ### Build a custom QEMU for aarch64/arm64 - REQUIRED
 > **Note:**
 >
@@ -475,6 +474,17 @@ debug_console_enabled = true
 
 This will pass `agent.debug_console agent.debug_console_vport=1026` to agent as kernel parameters, and sandboxes created using this parameters will start a shell in guest if new connection is accept from VSOCK.
 
+#### Start `kata-monitor`
+
+The `kata-runtime exec` command needs `kata-monitor` to get the sandbox's `vsock` address to connect to, first start `kata-monitor`.
+
+```
+$ sudo kata-monitor
+```
+
+`kata-monitor` will serve at `localhost:8090` by default.
+
+
 #### Connect to debug console
 
 Command `kata-runtime exec` is used to connect to the debug console.
@@ -489,10 +499,6 @@ bash-4.2# exit
 exit
 ```
 
-`kata-runtime exec` has a command-line option `runtime-namespace`, which is used to specify under which [runtime namespace](https://github.com/containerd/containerd/blob/master/docs/namespaces.md) the particular pod was created. By default, it is set to `k8s.io` and works for containerd when configured
- with Kubernetes. For CRI-O, the namespace should set to `default` explicitly. This should not be confused with [Kubernetes namespaces](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/).
-For other CRI-runtimes and configurations, you may need to set the namespace utilizing the `runtime-namespace` option.
-
 If you want to access guest OS through a traditional way, see [Traditional debug console setup)](#traditional-debug-console-setup).
 
 ### Traditional debug console setup
@@ -612,11 +618,8 @@ sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.debug_cons
 > **Note** Ports 1024 and 1025 are reserved for communication with the agent
 > and gathering of agent logs respectively. 
 
-##### Connecting to the debug console
-
-Next, connect to the debug console. The VSOCKS paths vary slightly between each
-VMM solution.
-
+Next, connect to the debug console. The VSOCKS paths vary slightly between
+cloud-hypervisor and firecracker.
 In case of cloud-hypervisor, connect to the `vsock` as shown:
 ```
 $ sudo su -c 'cd /var/run/vc/vm/{sandbox_id}/root/ && socat stdin unix-connect:clh.sock'
@@ -633,12 +636,6 @@ CONNECT 1026
 
 **Note**: You need to press the `RETURN` key to see the shell prompt.
 
-
-For QEMU, connect to the `vsock` as shown:
-```
-$ sudo su -c 'cd /var/run/vc/vm/{sandbox_id} && socat "stdin,raw,echo=0,escape=0x11" "unix-connect:console.sock"
-```
-
 To disconnect from the virtual machine, type `CONTROL+q` (hold down the
 `CONTROL` key and press `q`).
 

docs/Licensing-strategy.md

@@ -22,4 +22,4 @@ licensing and allows automated tooling to check the license of individual
 files.
 
 This SPDX licence identifier requirement is enforced by the
-[CI (Continuous Integration) system](https://github.com/kata-containers/tests/blob/main/.ci/static-checks.sh).
+[CI (Continuous Integration) system](https://github.com/kata-containers/tests/blob/master/.ci/static-checks.sh).

docs/Limitations.md

@@ -19,8 +19,6 @@
         * [Support for joining an existing VM network](#support-for-joining-an-existing-vm-network)
         * [docker --net=host](#docker---nethost)
         * [docker run --link](#docker-run---link)
-    * [Storage limitations](#storage-limitations)
-        * [Kubernetes `volumeMounts.subPaths`](#kubernetes-volumemountssubpaths)
     * [Host resource sharing](#host-resource-sharing)
         * [docker run --privileged](#docker-run---privileged)
 * [Miscellaneous](#miscellaneous)
@@ -28,7 +26,7 @@
 * [Appendices](#appendices)
     * [The constraints challenge](#the-constraints-challenge)
 
-***
+---
 
 # Overview
 
@@ -94,9 +92,7 @@ This section lists items that might be possible to fix.
 ### checkpoint and restore
 
 The runtime does not provide `checkpoint` and `restore` commands. There
-are discussions about using VM save and restore to give us a
-`[criu](https://github.com/checkpoint-restore/criu)`-like functionality,
-which might provide a solution.
+are discussions about using VM save and restore to give [`criu`](https://github.com/checkpoint-restore/criu)-like functionality, which might provide a solution.
 
 Note that the OCI standard does not specify `checkpoint` and `restore`
 commands.
@@ -220,17 +216,6 @@ Equivalent functionality can be achieved with the newer docker networking comman
 See more documentation at
 [docs.docker.com](https://docs.docker.com/engine/userguide/networking/default_network/dockerlinks/).
 
-## Storage limitations
-
-### Kubernetes `volumeMounts.subPaths`
-
-Kubernetes `volumeMount.subPath` is not supported by Kata Containers at the
-moment.
-
-See [this issue](https://github.com/kata-containers/runtime/issues/2812) for more details.
-[Another issue](https://github.com/kata-containers/kata-containers/issues/1728) focuses on the case of `emptyDir`.
-
-
 ## Host resource sharing
 
 ### docker run --privileged
@@ -239,7 +224,7 @@ Privileged support in Kata is essentially different from `runc` containers.
 Kata does support `docker run --privileged` command, but in this case full access
 to the guest VM is provided in addition to some host access.
 
-The container runs with elevated capabilities within the guest and is granted
+The container runs with elevated capabilities within the guest and is granted 
 access to guest devices instead of the host devices.
 This is also true with using `securityContext privileged=true` with Kubernetes.
 

docs/README.md

@@ -40,7 +40,6 @@ See the [howto documentation](how-to).
 * [Intel QAT with Kata](./use-cases/using-Intel-QAT-and-kata.md)
 * [VPP with Kata](./use-cases/using-vpp-and-kata.md)
 * [SPDK vhost-user with Kata](./use-cases/using-SPDK-vhostuser-and-kata.md)
-* [Intel SGX with Kata](./use-cases/using-Intel-SGX-and-kata.md)
 
 ## Developer Guide
 
@@ -49,7 +48,6 @@ Documents that help to understand and contribute to Kata Containers.
 ### Design and Implementations
 
 * [Kata Containers Architecture](design/architecture.md): Architectural overview of Kata Containers
-* [Kata Containers E2E Flow](design/end-to-end-flow.md): The entire end-to-end flow of Kata Containers
 * [Kata Containers design](./design/README.md): More Kata Containers design documents
 
 ### How to Contribute

docs/Release-Process.md

@@ -18,7 +18,8 @@
 ## Requirements
 
 - [hub](https://github.com/github/hub)
-  * Using an [application token](https://github.com/settings/tokens) is required for hub.
+
+- OBS account with permissions on [`/home:katacontainers`](https://build.opensuse.org/project/subprojects/home:katacontainers)
 
 - GitHub permissions to push tags and create releases in Kata repositories.
 
@@ -31,9 +32,14 @@
 
 ### Bump all Kata repositories
 
-  Bump the repositories using a script in the Kata packaging repo, where:
-  - `BRANCH=<the-branch-you-want-to-bump>`
-  - `NEW_VERSION=<the-new-kata-version>`
+  - We have set up a Jenkins job to bump the version in the `VERSION` file in all Kata repositories. Go to the [Jenkins bump-job page](http://jenkins.katacontainers.io/job/release/build) to trigger a new job.
+  - Start a new job with variables for the job passed as:
+     - `BRANCH=<the-branch-you-want-to-bump>`
+     - `NEW_VERSION=<the-new-kata-version>`
+
+     For example, in the case where you want to make a patch release `1.10.2`, the variable `NEW_VERSION` should be `1.10.2` and `BRANCH` should point to  `stable-1.10`. In case of an alpha or release candidate release, `BRANCH` should point to `master` branch.
+
+  Alternatively, you can also bump the repositories using a script in the Kata packaging repo
   ```
   $ cd ${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging/release
   $ export NEW_VERSION=<the-new-kata-version>
@@ -73,9 +79,9 @@
   ```
   $ cd ${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging/release
   # Note: OLD_VERSION is where the script should start to get changes.
-  $ ./release-notes.sh ${OLD_VERSION} ${NEW_VERSION} > notes.md
+  $ ./runtime-release-notes.sh ${OLD_VERSION} ${NEW_VERSION} > notes.md
   # Edit the `notes.md` file to review and make any changes to the release notes.
-  # Add the release notes in the project's GitHub.
+  # Add the release notes in GitHub runtime.
   $ hub release edit -F notes.md "${NEW_VERSION}"
   ```
 

docs/Upgrading.md

@@ -48,10 +48,10 @@ Alternatively, if you are using Kata Containers version 1.12.0 or newer, you
 can check for newer releases using the command line:
 
 ```bash
-$ kata-runtime check --check-version-only
+$ kata-runtime kata-check --check-version-only
 ```
 
-There are various other related options. Run `kata-runtime check --help`
+There are various other related options. Run `kata-runtime kata-check --help`
 for further details.
 
 # Configuration changes

docs/design/architecture.md

@@ -58,7 +58,7 @@ to go through the VSOCK interface exported by QEMU.
 
 The container workload, that is, the actual OCI bundle rootfs, is exported from the
 host to the virtual machine.  In the case where a block-based graph driver is
-configured, `virtio-scsi` will be used. In all other cases a `virtio-fs` VIRTIO mount point
+configured, `virtio-scsi` will be used. In all other cases a 9pfs VIRTIO mount point
 will be used. `kata-agent` uses this mount point as the root filesystem for the
 container processes.
 
@@ -137,7 +137,7 @@ The runtime uses a TOML format configuration file called `configuration.toml`. B
 
 The actual configuration file paths can be determined by running:
 ```
-$ kata-runtime --show-default-config-paths
+$ kata-runtime --kata-show-default-config-paths
 ```
 Most users will not need to modify the configuration file.
 

docs/design/end-to-end-flow.md

@@ -1,4 +0,0 @@
-# Kata Containers E2E Flow
-
-
-![Kata containers e2e flow](arch-images/katacontainers-e2e-with-bg.jpg)

docs/design/kata-api-design.md

@@ -3,6 +3,7 @@ To fulfill the [Kata design requirements](kata-design-requirements.md), and base
 -  Sandbox based top API
 -  Storage and network hotplug API
 -  Plugin frameworks for external proprietary Kata runtime extensions
+-  Built-in shim and proxy types and capabilities
 
 ## Sandbox Based API
 ### Sandbox Management API
@@ -22,17 +23,17 @@ To fulfill the [Kata design requirements](kata-design-requirements.md), and base
 |`sandbox.Stats()`| Get the stats of a running sandbox, return a `SandboxStats` structure.|
 |`sandbox.Status()`| Get the status of the sandbox and containers, return a `SandboxStatus` structure.|
 |`sandbox.Stop(force)`| Stop a sandbox and Destroy the containers in the sandbox. When force is true, ignore guest related stop failures.|
-|`sandbox.CreateContainer(contConfig)`| Create new container in the sandbox with the `ContainerConfig` parameter. It will add new container config to `sandbox.config.Containers`.|
-|`sandbox.DeleteContainer(containerID)`| Delete a container from the sandbox by `containerID`, return a `Container` structure.|
+|`sandbox.CreateContainer(contConfig)`| Create new container in the sandbox with the `ContainerConfig` param. It will add new container config to `sandbox.config.Containers`.|
+|`sandbox.DeleteContainer(containerID)`| Delete a container from the sandbox by containerID, return a `Container` structure.|
 |`sandbox.EnterContainer(containerID, cmd)`| Run a new process in a container, executing customer's `types.Cmd` command.|
-|`sandbox.KillContainer(containerID, signal, all)`| Signal a container in the sandbox by the `containerID`.|
-|`sandbox.PauseContainer(containerID)`| Pause a running container in the sandbox by the `containerID`.|
+|`sandbox.KillContainer(containerID, signal, all)`| Signal a container in the sandbox by the containerID.|
+|`sandbox.PauseContainer(containerID)`| Pause a running container in the sandbox by the containerID.|
 |`sandbox.ProcessListContainer(containerID, options)`| List every process running inside a specific container in the sandbox, return a `ProcessList` structure.|
-|`sandbox.ResumeContainer(containerID)`| Resume a paused container in the sandbox by the `containerID`.|
-|`sandbox.StartContainer(containerID)`| Start a container in the sandbox by the `containerID`.|
+|`sandbox.ResumeContainer(containerID)`| Resume a paused container in the sandbox by the containerID.|
+|`sandbox.StartContainer(containerID)`| Start a container in the sandbox by the containerID.|
 |`sandbox.StatsContainer(containerID)`| Get the stats of a running container, return a `ContainerStats` structure.|
 |`sandbox.StatusContainer(containerID)`| Get the status of a container in the sandbox, return a `ContainerStatus` structure.|
-|`sandbox.StopContainer(containerID, force)`| Stop a container in the sandbox by the `containerID`.|
+|`sandbox.StopContainer(containerID, force)`| Stop a container in the sandbox by the containerID.|
 |`sandbox.UpdateContainer(containerID, resources)`| Update a running container in the sandbox.|
 |`sandbox.WaitProcess(containerID, processID)`| Wait on a process to terminate.|
 ### Sandbox Hotplug API
@@ -56,7 +57,7 @@ To fulfill the [Kata design requirements](kata-design-requirements.md), and base
 |Name|Description|
 |---|---|
 |`sandbox.GetOOMEvent()`| Monitor the OOM events that occur in the sandbox..|
-|`sandbox.UpdateRuntimeMetrics()`| Update the `shim/hypervisor` metrics of the running sandbox.|
+|`sandbox.UpdateRuntimeMetrics()`| Update the shim/hypervisor's metrics of the running sandbox.|
 |`sandbox.GetAgentMetrics()`| Get metrics of the agent and the guest in the running sandbox.|
 
 ## Plugin framework for external proprietary Kata runtime extensions
@@ -98,3 +99,32 @@ Built-in implementations include:
 
 ### Sandbox Connection Plugin Workflow
 ![Sandbox Connection Plugin Workflow](https://raw.githubusercontent.com/bergwolf/raw-contents/master/kata/Sandbox-Connection.png "Sandbox Connection Plugin Workflow")
+
+## Built-in Shim and Proxy Types and Capabilities
+### Built-in shim/proxy sandbox configurations
+-  Supported shim configurations:
+
+|Name|Description|
+|---|---|
+|`noopshim`|Do not start any shim process.|
+|`ccshim`| Start the cc-shim binary.|
+|`katashim`| Start the `kata-shim` binary.|
+|`katashimbuiltin`|No standalone shim process but shim functionality APIs are exported.|
+-  Supported proxy configurations:
+
+|Name|Description|
+|---|---|
+|`noopProxy`| a dummy proxy implementation of the proxy interface, only used for testing purpose.|
+|`noProxy`|generic implementation for any case where no actual proxy is needed.|
+|`ccProxy`|run `ccProxy` to proxy between runtime and agent.|
+|`kataProxy`|run `kata-proxy` to translate Yamux connections between runtime and Kata agent. |
+|`kataProxyBuiltin`| no standalone proxy process and connect to Kata agent with internal Yamux translation.|
+
+### Built-in Shim Capability
+Built-in shim capability is implemented by removing standalone shim process, and
+supporting the shim related APIs.
+
+### Built-in Proxy Capability
+Built-in proxy capability is achieved by removing standalone proxy process, and
+connecting to Kata agent with a custom gRPC dialer that is internal Yamux translation.
+The behavior is enabled when proxy is configured as `kataProxyBuiltin`.

docs/design/virtualization.md

@@ -22,10 +22,10 @@ the multiple hypervisors and virtual machine monitors that Kata supports.
 ## Mapping container concepts to virtual machine technologies
 
 A typical deployment of Kata Containers will be in Kubernetes by way of a Container Runtime Interface (CRI) implementation. On every node,
-Kubelet will interact with a CRI implementer (such as containerd or CRI-O), which will in turn interface with Kata Containers (an OCI based runtime).
+Kubelet will interact with a CRI implementor (such as containerd or CRI-O), which will in turn interface with Kata Containers (an OCI based runtime).
 
 The CRI API, as defined at the [Kubernetes CRI-API repo](https://github.com/kubernetes/cri-api/), implies a few constructs being supported by the
-CRI implementation, and ultimately in Kata Containers. In order to support the full [API](https://github.com/kubernetes/cri-api/blob/a6f63f369f6d50e9d0886f2eda63d585fbd1ab6a/pkg/apis/runtime/v1alpha2/api.proto#L34-L110) with the CRI-implementer, Kata must provide the following constructs:
+CRI implementation, and ultimately in Kata Containers. In order to support the full [API](https://github.com/kubernetes/cri-api/blob/a6f63f369f6d50e9d0886f2eda63d585fbd1ab6a/pkg/apis/runtime/v1alpha2/api.proto#L34-L110) with the CRI-implementor, Kata must provide the following constructs:
 
 ![API to construct](./arch-images/api-to-construct.png)
 
@@ -41,9 +41,14 @@ Each hypervisor or VMM varies on how or if it handles each of these.
 
 ## Kata Containers Hypervisor and VMM support
 
-Kata Containers [supports multiple hypervisors](../hypervisors.md).
+Kata Containers is designed to support multiple virtual machine monitors (VMMs) and hypervisors.
+Kata Containers supports:
+ - [ACRN hypervisor](https://projectacrn.org/)
+ - [Cloud Hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor)/[KVM](https://www.linux-kvm.org/page/Main_Page)
+ - [Firecracker](https://github.com/firecracker-microvm/firecracker)/KVM
+ - [QEMU](http://www.qemu-project.org/)/KVM
 
-Details of each solution and a summary are provided below.
+Which configuration to use will depend on the end user's requirements. Details of each solution and a summary are provided below.
 
 ### QEMU/KVM
 
@@ -57,7 +62,7 @@ be changed by editing the runtime [`configuration`](./architecture.md/#configura
 Devices and features used:
 - virtio VSOCK or virtio serial
 - virtio block or virtio SCSI
-- [virtio net](https://www.redhat.com/en/virtio-networking-series)
+- virtio net
 - virtio fs or virtio 9p (recommend: virtio fs)
 - VFIO
 - hotplug
@@ -100,34 +105,25 @@ Devices used:
 
 ### Cloud Hypervisor/KVM
 
-[Cloud Hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor), based
-on [rust-vmm](https://github.com/rust-vmm), is designed to have a
-lighter footprint and smaller attack surface for running modern cloud
-workloads. Kata Containers with Cloud
-Hypervisor provides mostly complete compatibility with Kubernetes
-comparable to the QEMU configuration. As of the 1.12 and 2.0.0 release
-of Kata Containers, the Cloud Hypervisor configuration supports both CPU
-and memory resize, device hotplug (disk and VFIO), file-system sharing through virtio-fs,
-block-based volumes, booting from VM images backed by pmem device, and
-fine-grained seccomp filters for each VMM threads (e.g. all virtio
-device worker threads). Please check [this GitHub Project](https://github.com/orgs/kata-containers/projects/21)
-for details of ongoing integration efforts.
+Cloud Hypervisor, based on [rust-VMM](https://github.com/rust-vmm), is designed to have a lighter footprint and attack surface. For Kata Containers,
+relative to Firecracker, the Cloud Hypervisor configuration provides better compatibility at the expense of exposing additional devices: file system
+sharing and direct device assignment.  As of the 1.10 release of Kata Containers, Cloud Hypervisor does not support device hotplug, and as a result
+does not support updating container resources after boot, or utilizing block based volumes. While Cloud Hypervisor does support VFIO, Kata is still adding
+this support. As of 1.10, Kata does not support block based volumes or direct device assignment. See [Cloud Hypervisor device support documentation](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/device_model.md)
+for more details on Cloud Hypervisor.
 
-Devices and features used:
-- virtio VSOCK or virtio serial
+Devices used:
+- virtio VSOCK
 - virtio block
 - virtio net
 - virtio fs
-- virtio pmem
-- VFIO
-- hotplug
-- seccomp filters
-- [HTTP OpenAPI](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/vmm/src/api/openapi/cloud-hypervisor.yaml)
 
 ### Summary
 
 | Solution | release introduced | brief summary |
 |-|-|-|
-| Cloud Hypervisor | 1.10 | upstream Cloud Hypervisor with rich feature support, e.g. hotplug, VFIO and FS sharing|
-| Firecracker | 1.5 | upstream Firecracker, rust-VMM based, no VFIO, no FS sharing, no memory/CPU hotplug |
 | QEMU | 1.0 | upstream QEMU, with support for hotplug and filesystem sharing |
+| NEMU | 1.4 | Deprecated, removed as of 1.10 release. Slimmed down fork of QEMU, with experimental support of virtio-fs |
+| Firecracker | 1.5 | upstream Firecracker, rust-VMM based, no VFIO, no FS sharing, no memory/CPU hotplug |
+| QEMU-virtio-fs | 1.7 | upstream QEMU with support for virtio-fs. Will be removed once virtio-fs lands in upstream QEMU |
+| Cloud Hypervisor | 1.10 |  rust-VMM based, includes VFIO and FS sharing through virtio-fs, no hotplug |

docs/how-to/README.md

@@ -14,17 +14,8 @@
 - [How to import Kata Containers logs into Fluentd](how-to-import-kata-logs-with-fluentd.md)
 
 ## Hypervisors Integration
-
-  Currently supported hypervisors with Kata Containers include:
-- `qemu`
-- `cloud-hypervisor`
-- `firecracker`
-- `ACRN`
-
-  While `qemu` and `cloud-hypervisor` work out of the box with installation of Kata,
-  some additional configuration is needed in case of `firecracker` and `ACRN`.
-  Refer to the following guides for additional configuration steps:
 - [Kata Containers with Firecracker](https://github.com/kata-containers/documentation/wiki/Initial-release-of-Kata-Containers-with-Firecracker-support)
+- [Kata Containers with NEMU](how-to-use-kata-containers-with-nemu.md)
 - [Kata Containers with ACRN Hypervisor](how-to-use-kata-containers-with-acrn.md)
 
 ## Advanced Topics

docs/how-to/how-to-import-kata-logs-with-fluentd.md

@@ -185,7 +185,7 @@ in Kibana:
 ![Kata tags in EFK](./images/efk_syslog_entry_detail.png).
 
 We can however further sub-parse the Kata entries using the
-[Fluentd plugins](https://docs.fluentbit.io/manual/pipeline/parsers/logfmt) that will parse
+[Fluentd plugins](https://docs.fluentbit.io/manual/parser/logfmt) that will parse
 `logfmt` formatted data. We can utilise these to parse the sub-fields using a Fluentd filter
 section. At the same time, we will prefix the new fields with `kata_` to make it clear where
 they have come from:
@@ -222,7 +222,7 @@ test to check the parsing works. The resulting output from Fluentd is:
   "_COMM":"kata-runtime",
   "_EXE":"/opt/kata/bin/kata-runtime",
   "SYSLOG_TIMESTAMP":"Feb 21 10:31:27 ",
-  "_CMDLINE":"/opt/kata/bin/kata-runtime --config /opt/kata/share/defaults/kata-containers/configuration-qemu.toml --root /run/runc state 7cdd31660d8705facdadeb8598d2c0bd008e8142c54e3b3069abd392c8d58997",
+  "_CMDLINE":"/opt/kata/bin/kata-runtime --kata-config /opt/kata/share/defaults/kata-containers/configuration-qemu.toml --root /run/runc state 7cdd31660d8705facdadeb8598d2c0bd008e8142c54e3b3069abd392c8d58997",
   "SYSLOG_PID":"14314",
   "_PID":"14314",
   "MESSAGE":"time=\"2020-02-21T10:31:27.810781647Z\" level=info msg=\"release sandbox\" arch=amd64 command=state container=7cdd31660d8705facdadeb8598d2c0bd008e8142c54e3b3069abd392c8d58997 name=kata-runtime pid=14314 sandbox=1c3e77cad66aa2b6d8cc846f818370f79cb0104c0b840f67d0f502fd6562b68c source=virtcontainers subsystem=sandbox",
@@ -281,7 +281,7 @@ own file (rather than into the system journal).
 
 ```bash
 #!/bin/bash
-/opt/kata/bin/kata-runtime --config "/opt/kata/share/defaults/kata-containers/configuration-qemu.toml" --log-format=json --log=/var/log/kata-runtime.log $@
+/opt/kata/bin/kata-runtime --kata-config "/opt/kata/share/defaults/kata-containers/configuration-qemu.toml" --log-format=json --log=/var/log/kata-runtime.log $@
 ```
 
 And then we'll add the Fluentd config section to parse that file. Note, we inform the parser that Kata is

docs/how-to/how-to-load-kernel-modules-with-kata.md

@@ -56,9 +56,8 @@ There are some limitations with this approach:
 
 As was mentioned above, not all containers need the same modules, therefore using
 the configuration file for specifying the list of kernel modules per [POD][3] can
-be a pain.
-Unlike the configuration file, [annotations](how-to-set-sandbox-config-kata.md)
-provide a way to specify custom configurations per POD.
+be a pain. Unlike the configuration file, annotations provide a way to specify
+custom configurations per POD.
 
 The list of kernel modules and parameters can be set using the annotation
 `io.katacontainers.config.agent.kernel_modules` as a semicolon separated
@@ -102,7 +101,7 @@ spec:
     tty: true
 ```
 
-> **Note**: To pass annotations to Kata containers, [CRI-O must be configured correctly](how-to-set-sandbox-config-kata.md#cri-o-configuration)
+> **Note**: To pass annotations to Kata containers, [cri must to be configurated correctly](how-to-set-sandbox-config-kata.md#cri-configuration)
 
 [1]: ../../src/runtime
 [2]: ../../src/agent

docs/how-to/how-to-set-prometheus-in-k8s.md

@@ -34,7 +34,7 @@ Also you should ensure that `kubectl` working correctly.
 Start Prometheus by utilizing our sample manifest:
 
 ```
-$ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/docs/how-to/data/prometheus.yml
+$ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/2.0-dev/docs/how-to/data/prometheus.yml
 ```
 
 This will create a new namespace, `prometheus`, and create the following resources:
@@ -60,7 +60,7 @@ go_gc_duration_seconds{quantile="0.75"} 0.000229911
 `kata-monitor` can be started on the cluster as follows:
 
 ```
-$ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/docs/how-to/data/kata-monitor-daemonset.yml
+$ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/2.0-dev/docs/how-to/data/kata-monitor-daemonset.yml
 ```
 
 This will create a new namespace `kata-system` and a `daemonset` in it.
@@ -73,7 +73,7 @@ Once the `daemonset` is running, Prometheus should discover `kata-monitor` as a
 Run this command to run Grafana in Kubernetes:
 
 ```
-$ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/docs/how-to/data/grafana.yml
+$ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/2.0-dev/docs/how-to/data/grafana.yml
 ```
 
 This will create deployment and service for Grafana under namespace `prometheus`.
@@ -99,7 +99,7 @@ You can import this dashboard using Grafana UI, or using `curl` command in conso
 $ curl -XPOST -i localhost:3000/api/dashboards/import \
     -u admin:admin \
     -H "Content-Type: application/json" \
-	-d "{\"dashboard\":$(curl -sL https://raw.githubusercontent.com/kata-containers/kata-containers/main/docs/how-to/data/dashboard.json )}"
+	-d "{\"dashboard\":$(curl -sL https://raw.githubusercontent.com/kata-containers/kata-containers/2.0-dev/docs/how-to/data/dashboard.json )}"
 ```
 
 ## References

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -3,11 +3,6 @@
 Kata Containers gives users freedom to customize at per-pod level, by setting
 a wide range of Kata specific annotations in the pod specification.
 
-Some annotations may be [restricted](#restricted-annotations) by the
-configuration file for security reasons, notably annotations that could lead the
-runtime to execute programs on the host. Such annotations are marked with _(R)_ in
-the tables below.
-
 # Kata Configuration Annotations
 There are several kinds of Kata configurations and they are listed below.
 
@@ -26,13 +21,11 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.runtime.disable_new_netns` | `boolean` | determines if a new netns is created for the hypervisor process |
 | `io.katacontainers.config.runtime.internetworking_model` | string| determines how the VM should be connected to the container network interface. Valid values are `macvtap`, `tcfilter` and `none` |
 | `io.katacontainers.config.runtime.sandbox_cgroup_only`| `boolean` | determines if Kata processes are managed only in sandbox cgroup |
-| `io.katacontainers.config.runtime.enable_pprof` | `boolean` | enables Golang `pprof` for `containerd-shim-kata-v2` process |
 
 ## Agent Options
 | Key | Value Type | Comments |
 |-------| ----- | ----- |
 | `io.katacontainers.config.agent.enable_tracing` | `boolean` | enable tracing for the agent |
-| `io.katacontainers.config.agent.container_pipe_size` | uint32 | specify the size of the std(in/out) pipes created for containers |
 | `io.katacontainers.config.agent.kernel_modules` | string | the list of kernel modules and their parameters that will be loaded in the guest kernel. Semicolon separated list of kernel modules and their parameters. These modules will be loaded in the guest kernel using `modprobe`(8). E.g., `e1000e InterruptThrottleRate=3000,3000,3000 EEE=1; i915 enable_ppgtt=0` |
 | `io.katacontainers.config.agent.trace_mode` | string | the trace mode for the agent |
 | `io.katacontainers.config.agent.trace_type` | string | the trace type for the agent |
@@ -45,24 +38,17 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.block_device_cache_noflush` | `boolean` | Denotes whether flush requests for the device are ignored |
 | `io.katacontainers.config.hypervisor.block_device_cache_set` | `boolean` | cache-related options will be set to block devices or not |
 | `io.katacontainers.config.hypervisor.block_device_driver` | string | the driver to be used for block device, valid values are `virtio-blk`, `virtio-scsi`, `nvdimm`|
-| `io.katacontainers.config.hypervisor.cpu_features` | `string` | Comma-separated list of CPU features to pass to the CPU (QEMU) |
-| `io.katacontainers.config.hypervisor.ctlpath` (R) | `string` | Path to the `acrnctl` binary for the ACRN hypervisor |
 | `io.katacontainers.config.hypervisor.default_max_vcpus` | uint32| the maximum number of vCPUs allocated for the VM by the hypervisor |
 | `io.katacontainers.config.hypervisor.default_memory` | uint32| the memory assigned for a VM by the hypervisor in `MiB` |
 | `io.katacontainers.config.hypervisor.default_vcpus` | uint32| the default vCPUs assigned for a VM by the hypervisor |
 | `io.katacontainers.config.hypervisor.disable_block_device_use` | `boolean` | disallow a block device from being used |
-| `io.katacontainers.config.hypervisor.disable_image_nvdimm` | `boolean` | specify if a `nvdimm` device should be used as rootfs for the guest (QEMU) |
 | `io.katacontainers.config.hypervisor.disable_vhost_net` | `boolean` | specify if `vhost-net` is not available on the host |
 | `io.katacontainers.config.hypervisor.enable_hugepages` | `boolean` | if the memory should be `pre-allocated` from huge pages |
-| `io.katacontainers.config.hypervisor.enable_iommu_platform` | `boolean` | enable `iommu` on CCW devices (QEMU s390x) |
-| `io.katacontainers.config.hypervisor.enable_iommu` | `boolean` | enable `iommu` on Q35 (QEMU x86_64) |
 | `io.katacontainers.config.hypervisor.enable_iothreads` | `boolean`| enable IO to be processed in a separate thread. Supported currently for virtio-`scsi` driver |
 | `io.katacontainers.config.hypervisor.enable_mem_prealloc` | `boolean` | the memory space used for `nvdimm` device by the hypervisor |
 | `io.katacontainers.config.hypervisor.enable_swap` | `boolean` | enable swap of VM memory |
-| `io.katacontainers.config.hypervisor.enable_vhost_user_store` | `boolean` | enable vhost-user storage device (QEMU) |
-| `io.katacontainers.config.hypervisor.enable_virtio_mem` | `boolean` | enable virtio-mem (QEMU) |
-| `io.katacontainers.config.hypervisor.entropy_source` (R) | string| the path to a host source of entropy (`/dev/random`, `/dev/urandom` or real hardware RNG device) |
-| `io.katacontainers.config.hypervisor.file_mem_backend` (R) | string | file based memory backend root directory |
+| `io.katacontainers.config.hypervisor.entropy_source` | string| the path to a host source of entropy (`/dev/random`, `/dev/urandom` or real hardware RNG device) |
+| `io.katacontainers.config.hypervisor.file_mem_backend` | string | file based memory backend root directory |
 | `io.katacontainers.config.hypervisor.firmware_hash` | string | container firmware SHA-512 hash value |
 | `io.katacontainers.config.hypervisor.firmware` | string | the guest firmware that will run the container VM |
 | `io.katacontainers.config.hypervisor.guest_hook_path` | string | the path within the VM that will be used for drop in hooks |
@@ -73,7 +59,7 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.initrd_hash` | string | container guest initrd SHA-512 hash value |
 | `io.katacontainers.config.hypervisor.initrd` | string | the guest initrd image that will run in the container VM |
 | `io.katacontainers.config.hypervisor.jailer_hash` | string | container jailer SHA-512 hash value |
-| `io.katacontainers.config.hypervisor.jailer_path` (R) | string | the jailer that will constrain the container VM |
+| `io.katacontainers.config.hypervisor.jailer_path` | string | the jailer that will constrain the container VM |
 | `io.katacontainers.config.hypervisor.kernel_hash` | string | container kernel image SHA-512 hash value |
 | `io.katacontainers.config.hypervisor.kernel_params` | string | additional guest kernel parameters |
 | `io.katacontainers.config.hypervisor.kernel` | string | the kernel used to boot the container VM |
@@ -83,21 +69,17 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.memory_slots` | uint32| the memory slots assigned to the VM by the hypervisor |
 | `io.katacontainers.config.hypervisor.msize_9p` | uint32 | the `msize` for 9p shares |
 | `io.katacontainers.config.hypervisor.path` | string | the hypervisor that will run the container VM |
-| `io.katacontainers.config.hypervisor.pcie_root_port` | specify the number of PCIe Root Port devices. The PCIe Root Port device is used to hot-plug a PCIe device (QEMU) |
 | `io.katacontainers.config.hypervisor.shared_fs` | string | the shared file system type, either `virtio-9p` or `virtio-fs` |
 | `io.katacontainers.config.hypervisor.use_vsock` | `boolean` | specify use of `vsock` for agent communication |
-| `io.katacontainers.config.hypervisor.vhost_user_store_path` (R) | `string` | specify the directory path where vhost-user devices related folders, sockets and device nodes should be (QEMU) |
 | `io.katacontainers.config.hypervisor.virtio_fs_cache_size` | uint32 | virtio-fs DAX cache size in `MiB` |
 | `io.katacontainers.config.hypervisor.virtio_fs_cache` | string | the cache mode for virtio-fs, valid values are `always`, `auto` and `none` |
 | `io.katacontainers.config.hypervisor.virtio_fs_daemon` | string | virtio-fs `vhost-user` daemon path |
 | `io.katacontainers.config.hypervisor.virtio_fs_extra_args` | string | extra options passed to `virtiofs` daemon |
 
-# CRI-O Configuration
+# CRI Configuration
 
 In case of CRI-O, all annotations specified in the pod spec are passed down to Kata.
 
-# containerd Configuration
-
 For containerd, annotations specified in the pod spec are passed down to Kata
 starting with version `1.3.0` of containerd. Additionally, extra configuration is
 needed for containerd, by providing a `pod_annotations` field in the containerd config
@@ -110,14 +92,16 @@ for passing annotations to Kata from containerd:
 $ cat /etc/containerd/config
 ....
 
-         [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
-           runtime_type = "io.containerd.kata.v2"
+[plugins.cri.containerd.runtimes.kata]
+           runtime_type = "io.containerd.runc.v1"
            pod_annotations = ["io.katacontainers.*"]
+           [plugins.cri.containerd.runtimes.kata.options]
+             BinaryName = "/usr/bin/kata-runtime"
 ....
 
 ```
 
-Additional documentation on the above configuration can be found in the
+Additional documentation on the above configuration can be found in the 
 [containerd docs](https://github.com/containerd/cri/blob/8d5a8355d07783ba2f8f451209f6bdcc7c412346/docs/config.md).
 
 # Example - Using annotations
@@ -175,32 +159,3 @@ spec:
     stdin: true
     tty: true
 ```
-
-# Restricted annotations
-
-Some annotations are _restricted_, meaning that the configuration file specifies
-the acceptable values. Currently, only hypervisor annotations are restricted,
-for security reason, with the intent to control which binaries the Kata
-Containers runtime will launch on your behalf.
-
-The configuration file validates the annotation _name_ as well as the annotation
-_value_.
-
-The acceptable annotation names are defined by the `enable_annotations` entry in
-the configuration file.
-
-For restricted annotations, an additional configuration entry provides a list of
-acceptable values. Since most restricted annotations are intended to control
-which binaries the runtime can execute, the valid value is generally provided by
-a shell pattern, as defined by `glob(3)`. The table below provides the name of
-the configuration entry:
-
-| Key | Config file entry | Comments |
-|-------| ----- | ----- |
-| `ctlpath`  | `valid_ctlpaths` | Valid paths for `acrnctl` binary |
-| `entropy_source` | `valid_entropy_sources` | Valid entropy sources, e.g. `/dev/random` |
-| `file_mem_backend`  | `valid_file_mem_backends` | Valid locations for the file-based memory backend root directory |
-| `jailer_path`  | `valid_jailer_paths`| Valid paths for the jailer constraining the container VM (Firecracker) |
-| `path`  | `valid_hypervisor_paths` | Valid hypervisors to run the container VM |
-| `vhost_user_store_path`  | `valid_vhost_user_store_paths` | Valid paths for vhost-user related files|
-| `virtio_fs_daemon`  | `valid_virtio_fs_daemon_paths` | Valid paths for the `virtiofsd` daemon |

docs/how-to/how-to-use-k8s-with-cri-containerd-and-kata.md

@@ -7,10 +7,9 @@
     * [Configure Kubelet to use containerd](#configure-kubelet-to-use-containerd)
     * [Configure HTTP proxy - OPTIONAL](#configure-http-proxy---optional)
 * [Start Kubernetes](#start-kubernetes)
-* [Configure Pod Network](#configure-pod-network)
+* [Install a Pod Network](#install-a-pod-network)
 * [Allow pods to run in the master node](#allow-pods-to-run-in-the-master-node)
-* [Create runtime class for Kata Containers](#create-runtime-class-for-kata-containers)
-* [Run pod in Kata Containers](#run-pod-in-kata-containers)
+* [Create an untrusted pod using Kata Containers](#create-an-untrusted-pod-using-kata-containers)
 * [Delete created pod](#delete-created-pod)
 
 This document describes how to set up a single-machine Kubernetes (k8s) cluster.
@@ -19,6 +18,9 @@ The Kubernetes cluster will use the
 [CRI containerd plugin](https://github.com/containerd/cri) and
 [Kata Containers](https://katacontainers.io) to launch untrusted workloads.
 
+For Kata Containers 1.5.0-rc2 and above, we will use `containerd-shim-kata-v2` (short as `shimv2` in this documentation)
+to launch Kata Containers. For the previous version of Kata Containers, the Pods are launched with `kata-runtime`.
+
 ## Requirements
 
 - Kubernetes, Kubelet, `kubeadm`
@@ -123,33 +125,43 @@ $ sudo systemctl daemon-reload
   $ sudo -E kubectl get pods
   ```
 
-## Configure Pod Network
+## Install a Pod Network
 
 A pod network plugin is needed to allow pods to communicate with each other.
-You can find more about CNI plugins from the [Creating a cluster with `kubeadm`](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#instructions) guide.
 
-By default the CNI plugin binaries is installed under `/opt/cni/bin` (in package `kubernetes-cni`), you only need to create a configuration file for CNI plugin.
+- Install the `flannel` plugin by following the
+  [Using `kubeadm` to Create a Cluster](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#instructions)
+  guide, starting from the **Installing a pod network** section.
+
+- Create a pod network using flannel
+
+  > **Note:** There is no known way to determine programmatically the best version (commit) to use.
+  > See https://github.com/coreos/flannel/issues/995.
 
   ```bash
-  $ sudo -E mkdir -p /etc/cni/net.d
+  $ sudo -E kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
+  ```
 
-  $ sudo -E cat > /etc/cni/net.d/10-mynet.conf <<EOF
-  {
-    "cniVersion": "0.2.0",
-    "name": "mynet",
-    "type": "bridge",
-    "bridge": "cni0",
-    "isGateway": true,
-    "ipMasq": true,
-    "ipam": {
-      "type": "host-local",
-      "subnet": "172.19.0.0/24",
-      "routes": [
-        { "dst": "0.0.0.0/0" }
-      ]
-    }
-  }
-  EOF
+- Wait for the pod network to become available
+
+  ```bash
+  # number of seconds to wait for pod network to become available
+  $ timeout_dns=420
+
+  $ while [ "$timeout_dns" -gt 0 ]; do
+      if sudo -E kubectl get pods --all-namespaces | grep dns | grep Running; then
+          break
+      fi
+
+      sleep 1s
+      ((timeout_dns--))
+   done
+  ```
+
+- Check the pod network is running
+
+  ```bash
+  $ sudo -E kubectl get pods --all-namespaces | grep dns | grep Running && echo "OK" || ( echo "FAIL" && false )
   ```
 
 ## Allow pods to run in the master node
@@ -160,38 +172,24 @@ By default, the cluster will not schedule pods in the master node. To enable mas
 $ sudo -E kubectl taint nodes --all node-role.kubernetes.io/master-
 ```
 
-## Create runtime class for Kata Containers
+## Create an untrusted pod using Kata Containers
 
 By default, all pods are created with the default runtime configured in CRI containerd plugin.
-From Kubernetes v1.12, users can use [`RuntimeClass`](https://kubernetes.io/docs/concepts/containers/runtime-class/#runtime-class) to specify a different runtime for Pods.
 
-```bash
-$ cat > runtime.yaml <<EOF
-apiVersion: node.k8s.io/v1beta1
-kind: RuntimeClass
-metadata:
-  name: kata
-handler: kata
-EOF
-
-$ sudo -E kubectl apply -f runtime.yaml
-```
-
-## Run pod in Kata Containers
-
-If a pod has the `runtimeClassName` set to `kata`, the CRI plugin runs the pod with the
+If a pod has the `io.kubernetes.cri.untrusted-workload` annotation set to `"true"`, the CRI plugin runs the pod with the
 [Kata Containers runtime](../../src/runtime/README.md).
 
-- Create an pod configuration that using Kata Containers runtime
+- Create an untrusted pod configuration
 
   ```bash
-  $ cat << EOT | tee nginx-kata.yaml
+  $ cat << EOT | tee nginx-untrusted.yaml
   apiVersion: v1
   kind: Pod
   metadata:
-    name: nginx-kata
+    name: nginx-untrusted
+    annotations:
+      io.kubernetes.cri.untrusted-workload: "true"
   spec:
-    runtimeClassName: kata
     containers:
     - name: nginx
       image: nginx
@@ -199,9 +197,9 @@ If a pod has the `runtimeClassName` set to `kata`, the CRI plugin runs the pod w
   EOT
   ```
 
-- Create the pod
+- Create an untrusted pod
   ```bash
-  $ sudo -E kubectl apply -f nginx-kata.yaml
+  $ sudo -E kubectl apply -f nginx-untrusted.yaml
   ```
 
 - Check pod is running
@@ -218,5 +216,5 @@ If a pod has the `runtimeClassName` set to `kata`, the CRI plugin runs the pod w
 ## Delete created pod
 
 ```bash
-$ sudo -E kubectl delete -f nginx-kata.yaml
+$ sudo -E kubectl delete -f nginx-untrusted.yaml
 ```

docs/how-to/how-to-use-kata-containers-with-acrn.md

@@ -91,7 +91,7 @@ To configure Kata Containers with ACRN, copy the generated `configuration-acrn.t
 The following command shows full paths to the `configuration.toml` files that the runtime loads. It will use the first path that exists. (Please make sure the kernel and image paths are set correctly in the `configuration.toml` file)
 
 ```bash
-$ sudo kata-runtime --show-default-config-paths
+$ sudo kata-runtime --kata-show-default-config-paths
 ```
 
 >**Warning:** Please offline CPUs using [this](offline_cpu.sh) script, else VM launches will fail.

docs/how-to/how-to-use-kata-containers-with-nemu.md

@@ -0,0 +1,115 @@
+
+# Kata Containers with NEMU
+
+* [Introduction](#introduction)
+* [Pre-requisites](#pre-requisites)
+* [NEMU](#nemu)
+    * [Download and build](#download-and-build)
+        * [x86_64](#x86_64)
+        * [aarch64](#aarch64)
+* [Configure Kata Containers](#configure-kata-containers)
+
+Kata Containers relies by default on the QEMU hypervisor in order to spawn the virtual machines running containers. [NEMU](https://github.com/intel/nemu) is a fork of QEMU that:
+- Reduces the number of lines of code.
+- Removes all legacy devices.
+- Reduces the emulation as far as possible.
+
+## Introduction
+
+This document describes how to run Kata Containers with NEMU, first by explaining how to download, build and install it. Then it walks through the steps needed to update your Kata Containers configuration in order to run with NEMU. 
+
+## Pre-requisites
+This document requires Kata Containers to be [installed](../install/README.md) on your system.
+
+Also, it's worth noting that NEMU only supports `x86_64` and `aarch64` architecture.
+
+## NEMU
+
+### Download and build
+
+```bash
+$ git clone https://github.com/intel/nemu.git
+$ cd nemu
+$ git fetch origin
+$ git checkout origin/experiment/automatic-removal
+```
+#### x86_64
+```
+$ SRCDIR=$PWD ./tools/build_x86_64_virt.sh
+```
+#### aarch64
+```
+$ SRCDIR=$PWD ./tools/build_aarch64.sh
+```
+
+> **Note:** The branch `experiment/automatic-removal` is a branch published by Jenkins after it has applied the automatic removal script to the `topic/virt-x86` branch. The purpose of this code removal being to reduce the source tree by removing files not being used by NEMU.
+
+After those commands have successfully returned, you will find the NEMU binary at `$HOME/build-x86_64_virt/x86_64_virt-softmmu/qemu-system-x86_64_virt` (__x86__), or `$HOME/build-aarch64/aarch64-softmmu/qemu-system-aarch64` (__ARM__).
+
+You also need the `OVMF` firmware in order to boot the virtual machine's kernel. It can currently be found at this [location](https://github.com/intel/ovmf-virt/releases).
+```bash
+$ sudo mkdir -p /usr/share/nemu
+$ OVMF_URL=$(curl -sL https://api.github.com/repos/intel/ovmf-virt/releases/latest | jq -S '.assets[0].browser_download_url')
+$ curl -o OVMF.fd -L $(sed -e 's/^"//' -e 's/"$//' <<<"$OVMF_URL")
+$ sudo install -o root -g root -m 0640 OVMF.fd /usr/share/nemu/
+```
+> **Note:** The OVMF firmware will be located at this temporary location until the changes can be pushed upstream.
+
+
+## Configure Kata Containers
+All you need from this section is to modify the configuration file `/usr/share/defaults/kata-containers/configuration.toml` to specify the options related to the hypervisor.
+
+
+```diff
+ [hypervisor.qemu]
+-path = "/usr/bin/qemu-lite-system-x86_64"
++path = "/home/foo/build-x86_64_virt/x86_64_virt-softmmu/qemu-system-x86_64_virt"
+ kernel = "/usr/share/kata-containers/vmlinuz.container"
+ initrd = "/usr/share/kata-containers/kata-containers-initrd.img"
+ image = "/usr/share/kata-containers/kata-containers.img"
+-machine_type = "pc"
++machine_type = "virt"
+ 
+ # Optional space-separated list of options to pass to the guest kernel.
+ # For example, use `kernel_params = "vsyscall=emulate"` if you are having
+@@ -31,7 +31,7 @@
+ 
+ # Path to the firmware.
+ # If you want that qemu uses the default firmware leave this option empty
+-firmware = ""
++firmware = "/usr/share/nemu/OVMF.fd"
+ 
+ # Machine accelerators
+ # comma-separated list of machine accelerators to pass to the hypervisor.
+```
+
+As you can see from this snippet above, all you need to change is:
+- The path to the hypervisor binary, `/home/foo/build-x86_64_virt/x86_64_virt-softmmu/qemu-system-x86_64_virt` in this example.
+- The machine type from `pc` to `virt`.
+- The path to the firmware binary, `/usr/share/nemu/OVMF.fd` in this example.
+
+Once you have saved those modifications, you can start a new container:
+```bash
+$ docker run --runtime=kata-runtime -it busybox
+```
+And you will be able to verify this new container is running with the NEMU hypervisor by looking for the hypervisor path and the machine type from the `qemu` process running on your system:
+```bash
+$ ps -aux | grep qemu
+root ... /home/foo/build-x86_64_virt/x86_64_virt-softmmu/qemu-system-x86_64_virt
+...  -machine virt,accel=kvm,kernel_irqchip,nvdimm ...
+```
+
+Also relying on `kata-runtime kata-env` is a reliable way to validate you are using the expected hypervisor:
+```bash
+$ kata-runtime kata-env | awk -v RS= '/\[Hypervisor\]/'
+[Hypervisor]
+  MachineType = "virt"
+  Version = "NEMU (like QEMU) version 3.0.0 (v3.0.0-179-gaf9a791)\nCopyright (c) 2003-2017 Fabrice Bellard and the QEMU Project developers"
+  Path = "/home/foo/build-x86_64_virt/x86_64_virt-softmmu/qemu-system-x86_64_virt"
+  BlockDeviceDriver = "virtio-scsi"
+  EntropySource = "/dev/urandom"
+  Msize9p = 8192
+  MemorySlots = 10
+  Debug = true
+  UseVSock = false
+```

docs/how-to/how-to-use-virtio-fs-with-kata.md

@@ -1,12 +1,61 @@
 # Kata Containers with virtio-fs
 
-- [Kata Containers with virtio-fs](#kata-containers-with-virtio-fs)
-  - [Introduction](#introduction)
+- [Introduction](#introduction)
+- [Pre-requisites](#pre-requisites)
+- [Install Kata Containers with virtio-fs support](#install-kata-containers-with-virtio-fs-support)
+- [Run a Kata Container utilizing virtio-fs](#run-a-kata-container-utilizing-virtio-fs)
 
 ## Introduction
 
 Container deployments utilize explicit or implicit file sharing between host filesystem and containers. From a trust perspective, avoiding a shared file-system between the trusted host and untrusted container is recommended. This is not always feasible. In Kata Containers, block-based volumes are preferred as they allow usage of either device pass through or `virtio-blk` for access within the virtual machine.
 
-As of the 2.0 release of Kata Containers, [virtio-fs](https://virtio-fs.gitlab.io/) is the default filesystem sharing mechanism.
+As of the 1.7 release of Kata Containers, [9pfs](https://www.kernel.org/doc/Documentation/filesystems/9p.txt) is the default filesystem sharing mechanism. While this does allow for workload compatibility, it does so with degraded performance and potential for POSIX compliance limitations.
 
-virtio-fs support works out of the box for `cloud-hypervisor` and `qemu`, when Kata Containers is deployed using `kata-deploy`. Learn more about `kata-deploy` and how to use `kata-deploy` in Kubernetes [here](https://github.com/kata-containers/packaging/tree/master/kata-deploy#kubernetes-quick-start).
+To help address these limitations, [virtio-fs](https://virtio-fs.gitlab.io/) has been developed. virtio-fs is a shared file system that lets virtual machines access a directory tree on the host. In Kata Containers, virtio-fs can be used to share container volumes, secrets, config-maps, configuration files (hostname, hosts, `resolv.conf`) and the container rootfs on the host with the guest.  virtio-fs provides significant performance and POSIX compliance improvements compared to 9pfs.
+
+Enabling of virtio-fs requires changes in the guest kernel as well as the VMM. For Kata Containers, experimental virtio-fs support is enabled through `qemu` and `cloud-hypervisor` VMMs.
+
+**Note: virtio-fs support is experimental in the 1.7 release of Kata Containers. Work is underway to improve stability, performance and upstream integration. This is available for early preview - use at your own risk**
+
+This document describes how to get Kata Containers to work with virtio-fs.
+
+## Pre-requisites
+
+Before Kata 1.8 this feature required the host to have hugepages support enabled. Enable this with the `sysctl vm.nr_hugepages=1024` command on the host.In later versions of Kata, virtio-fs leverages `/dev/shm` as the shared memory backend. The default size of `/dev/shm` on a system is typically half of the total system memory. This can pose a physical limit to the maximum number of pods that can be launched with virtio-fs. This can be overcome by increasing the size of `/dev/shm` as shown below:
+
+```bash
+$ mount -o remount,size=${desired_shm_size} /dev/shm
+```
+ 
+## Install Kata Containers with virtio-fs support
+
+The Kata Containers `qemu` configuration with virtio-fs and the `virtiofs` daemon are available in the [Kata Container release](https://github.com/kata-containers/runtime/releases) artifacts starting with the 1.9 release. Installation is available through [distribution packages](https://github.com/kata-containers/documentation/blob/master/install/README.md#supported-distributions) as well through [`kata-deploy`](https://github.com/kata-containers/packaging/tree/master/kata-deploy).
+
+**Note: Support for virtio-fs was first introduced in `NEMU` hypervisor in Kata 1.8 release. This hypervisor has been deprecated.**
+
+Install the latest release of Kata with `kata-deploy` as follows:
+```
+docker run --runtime=runc -v /opt/kata:/opt/kata -v /var/run/dbus:/var/run/dbus -v /run/systemd:/run/systemd -v /etc/docker:/etc/docker -it katadocker/kata-deploy kata-deploy-docker install
+```
+
+This will place the Kata release artifacts in `/opt/kata`, and update Docker's configuration to include a runtime target, `kata-qemu-virtiofs`. Learn more about `kata-deploy` and how to use `kata-deploy` in Kubernetes [here](https://github.com/kata-containers/packaging/tree/master/kata-deploy#kubernetes-quick-start).
+
+## Run a Kata Container utilizing virtio-fs
+
+Once installed, start a new container, utilizing `qemu` + `virtiofs`:
+```bash
+$ docker run --runtime=kata-qemu-virtiofs -it busybox
+```
+
+Verify the new container is running with the `qemu` hypervisor as well as using `virtiofsd`. To do this look for the hypervisor path and the `virtiofs` daemon process on the host:
+```bash
+$ ps -aux | grep virtiofs
+root ... /home/foo/build-x86_64_virt/x86_64_virt-softmmu/qemu-system-x86_64_virt
+...  -machine virt,accel=kvm,kernel_irqchip,nvdimm ...
+root ... /home/foo/build-x86_64_virt/virtiofsd-x86_64 ...
+```
+
+You can also try out virtio-fs using `cloud-hypervisor` VMM:
+```bash
+$ docker run --runtime=kata-clh -it busybox
+```

docs/how-to/what-is-vm-templating-and-how-do-I-use-it.md

@@ -46,7 +46,6 @@ overridden by `/etc/kata-containers/configuration.toml` if provided) such that:
   - `enable_template = true`
   - `initrd =` is set
   - `image =` option is commented out or removed
-  - `shared_fs` should not be `virtio-fs`
 
 Then you can create a VM templating for later usage by calling
 ```

docs/hypervisors.md

@@ -1,68 +0,0 @@
-# Hypervisors
-
-* [Hypervisors](#hypervisors)
-    * [Introduction](#introduction)
-    * [Types](#types)
-    * [Determine currently configured hypervisor](#determine-currently-configured-hypervisor)
-    * [Choose a Hypervisor](#choose-a-hypervisor)
-
-## Introduction
-
-Kata Containers supports multiple hypervisors. This document provides a very
-high level overview of the available hypervisors, giving suggestions as to
-which hypervisors you may wish to investigate further.
-
-> **Note:**
->
-> This document is not prescriptive or authoritative:
->
-> - It is up to you to decide which hypervisors may be most appropriate for
->   your use-case.
-> - Refer to the official documentation for each hypervisor for further details.
-
-## Types
-
-Since each hypervisor offers different features and options, Kata Containers
-provides a separate
-[configuration file](/src/runtime/README.md#configuration)
-for each. The configuration files contain comments explaining which options
-are available, their default values and how each setting can be used.
-
-> **Note:**
->
-> The simplest way to switch between hypervisors is to create a symbolic link
-> to the appropriate hypervisor-specific configuration file.
-
-| Hypervisor | Written in | Architectures | Type | Configuration file |
-|-|-|-|-|-|
-[ACRN] | C | `x86_64` | Type 1 (bare metal) | `configuration-acrn.toml` |
-[Cloud Hypervisor] | rust | `aarch64`, `x86_64` | Type 2 ([KVM]) | `configuration-clh.toml` |
-[Firecracker] | rust | `aarch64`, `x86_64` | Type 2 ([KVM]) | `configuration-fc.toml` |
-[QEMU] | C | all | Type 2 ([KVM]) | `configuration-qemu.toml` |
-
-## Determine currently configured hypervisor
-
-```bash
-$ kata-runtime kata-env | awk -v RS= '/\[Hypervisor\]/' | grep Path
-```
-
-## Choose a Hypervisor
-
-The table below provides a brief summary of some of the differences between
-the hypervisors:
-
-
-| Hypervisor | Summary | Features | Limitations | Container Creation speed | Memory density | Use cases | Comment |
-|-|-|-|-|-|-|-|-|
-[ACRN] | Safety critical and real-time workloads | | | excellent | excellent | Embedded and IOT systems | For advanced users |
-[Cloud Hypervisor] | Low latency, small memory footprint, small attack surface | Minimal | | excellent | excellent | High performance modern cloud workloads | |
-[Firecracker] | Very slimline | Extremely minimal | Doesn't support all device types | excellent | excellent | Serverless / FaaS | |
-[QEMU] | Lots of features | Lots | | good | good | Good option for most users | | All users |
-
-For further details, see the [Virtualization in Kata Containers](design/virtualization.md) document and the official documentation for each hypervisor.
-
-[ACRN]: https://projectacrn.org
-[Cloud Hypervisor]: https://github.com/cloud-hypervisor/cloud-hypervisor
-[Firecracker]: https://github.com/firecracker-microvm/firecracker
-[KVM]: https://en.wikipedia.org/wiki/Kernel-based_Virtual_Machine
-[QEMU]: http://www.qemu-project.org

docs/install/README.md

@@ -50,7 +50,9 @@ Kata packages are provided by official distribution repositories for:
 | Distribution (link to installation guide)                | Minimum versions                                                               |
 |----------------------------------------------------------|--------------------------------------------------------------------------------|
 | [CentOS](centos-installation-guide.md)                   | 8                                                                              |
-| [Fedora](fedora-installation-guide.md)                   | 34                                                                             |
+| [Fedora](fedora-installation-guide.md)                   | 32, Rawhide                                                                    |
+| [openSUSE](opensuse-installation-guide.md)               | [Leap 15.1](opensuse-leap-15.1-installation-guide.md)<br>Leap 15.2, Tumbleweed |
+| [SUSE Linux Enterprise (SLE)](sle-installation-guide.md) | SLE 15 SP1, 15 SP2                                                             |
 
 > **Note::**
 >

docs/install/centos-installation-guide.md

@@ -3,9 +3,15 @@
 1. Install the Kata Containers components with the following commands:
 
    ```bash
-   $ sudo -E dnf install -y centos-release-advanced-virtualization
-   $ sudo -E dnf module disable -y virt:rhel
    $ source /etc/os-release
+   $ cat <<EOF | sudo -E tee /etc/yum.repos.d/advanced-virt.repo
+     [advanced-virt]
+     name=Advanced Virtualization
+     baseurl=http://mirror.centos.org/\$contentdir/\$releasever/virt/\$basearch/advanced-virtualization
+     enabled=1
+     gpgcheck=1
+     skip_if_unavailable=1
+     EOF
    $ cat <<EOF | sudo -E tee /etc/yum.repos.d/kata-containers.repo
      [kata-containers]
      name=Kata Containers
@@ -14,7 +20,8 @@
      gpgcheck=1
      skip_if_unavailable=1
      EOF
-   $ sudo -E dnf install -y kata-containers
+   $ sudo -E dnf module disable -y virt:rhel
+   $ sudo -E dnf install -y kata-runtime
    ```
 
 2. Decide which container manager to use and select the corresponding link that follows:

docs/install/container-manager/containerd/containerd-install.md

@@ -18,7 +18,7 @@
 >
 > - If you decide to proceed and install a Kata Containers release, you can
 >   still check for the latest version of Kata Containers by running
->   `kata-runtime check --only-list-releases`.
+>   `kata-runtime kata-check --only-list-releases`.
 >
 > - These instructions will not work for Fedora 31 and higher since those
 >   distribution versions only support cgroups version 2 by default. However,

docs/install/fedora-installation-guide.md

@@ -3,7 +3,7 @@
 1. Install the Kata Containers components with the following commands:
 
    ```bash
-   $ sudo -E dnf -y install kata-containers
+   $ sudo -E dnf -y install kata-runtime
    ```
 
 2. Decide which container manager to use and select the corresponding link that follows:

docs/install/gce-installation-guide.md

@@ -6,7 +6,7 @@
 * [Install Kata](#install-kata)
 * [Create a Kata-enabled Image](#create-a-kata-enabled-image)
 
-Kata Containers on Google Compute Engine (GCE) makes use of [nested virtualization](https://cloud.google.com/compute/docs/instances/enable-nested-virtualization-vm-instances). Most of the installation procedure is identical to that for Kata on your preferred distribution, but enabling nested virtualization currently requires extra steps on GCE. This guide walks you through creating an image and instance with nested virtualization enabled. Note that `kata-runtime check` checks for nested virtualization, but does not fail if support is not found.
+Kata Containers on Google Compute Engine (GCE) makes use of [nested virtualization](https://cloud.google.com/compute/docs/instances/enable-nested-virtualization-vm-instances). Most of the installation procedure is identical to that for Kata on your preferred distribution, but enabling nested virtualization currently requires extra steps on GCE. This guide walks you through creating an image and instance with nested virtualization enabled. Note that `kata-runtime kata-check` checks for nested virtualization, but does not fail if support is not found.
 
 As a pre-requisite this guide assumes an installed and configured instance of the [Google Cloud SDK](https://cloud.google.com/sdk/downloads). For a zero-configuration option, all of the commands below were been tested under [Google Cloud Shell](https://cloud.google.com/shell/) (as of Jun 2018). Verify your `gcloud` installation and configuration:
 

docs/install/minikube-installation-guide.md

@@ -54,7 +54,7 @@ to enable nested virtualization can be found on the
 [KVM Nested Guests page](https://www.linux-kvm.org/page/Nested_Guests)
 
 Alternatively, and for other architectures, the Kata Containers built in
-[`check`](../../src/runtime/README.md#hardware-requirements)
+[`kata-check`](../../src/runtime/README.md#hardware-requirements)
 command can be used *inside Minikube* once Kata has been installed, to check for compatibility.
 
 ## Setting up Minikube

docs/install/opensuse-installation-guide.md

@@ -0,0 +1,10 @@
+# Install Kata Containers on openSUSE
+
+1. Install the Kata Containers components with the following commands:
+
+   ```bash
+   $ sudo -E zypper -n install katacontainers
+   ```
+
+2. Decide which container manager to use and select the corresponding link that follows:
+   - [Kubernetes](../Developer-Guide.md#run-kata-containers-with-kubernetes)

docs/install/opensuse-leap-15.1-installation-guide.md

@@ -0,0 +1,11 @@
+# Install Kata Containers on openSUSE Leap 15.1
+
+1. Install the Kata Containers components with the following commands:
+
+   ```bash
+   $ sudo -E zypper addrepo --refresh "https://download.opensuse.org/repositories/devel:/kubic/openSUSE_Leap_15.1/devel:kubic.repo"
+   $ sudo -E zypper -n --gpg-auto-import-keys install katacontainers
+   ```
+
+2. Decide which container manager to use and select the corresponding link that follows:
+   - [Kubernetes](../Developer-Guide.md#run-kata-containers-with-kubernetes)

docs/install/sle-installation-guide.md

@@ -0,0 +1,13 @@
+# Install Kata Containers on SLE
+
+1. Install the Kata Containers components with the following commands:
+
+   ```bash
+   $ source /etc/os-release
+   $ DISTRO_VERSION=$(sed "s/-/_/g" <<< "$VERSION")
+   $ sudo -E zypper addrepo --refresh "https://download.opensuse.org/repositories/devel:/kubic/SLE_${DISTRO_VERSION}_Backports/devel:kubic.repo"
+   $ sudo -E zypper -n --gpg-auto-import-keys install katacontainers
+   ```
+
+2. Decide which container manager to use and select the corresponding link that follows:
+   - [Kubernetes](../Developer-Guide.md#run-kata-containers-with-kubernetes)

docs/install/snap-installation-guide.md

@@ -1,58 +1,13 @@
-# Kata Containers snap package
-
-* [Install Kata Containers](#install-kata-containers)
-* [Configure Kata Containers](#configure-kata-containers)
-* [Integration with shim v2 Container Engines](#integration-with-shim-v2-container-engines)
-* [Remove Kata Containers snap package](#remove-kata-containers-snap-package)
-
-
-## Install Kata Containers
+# Install Kata Containers from `snapcraft.io`
 
 Kata Containers can be installed in any Linux distribution that supports
 [snapd](https://docs.snapcraft.io/installing-snapd).
 
-Run the following command to install **Kata Containers**:
+Run the following command to install Kata Containers:
 
-```sh
-$ sudo snap install kata-containers --candidate --classic
-```
+   ```bash
+   $ sudo snap install kata-containers --classic
+   ```
 
-## Configure Kata Containers
-
-By default Kata Containers snap image is mounted at `/snap/kata-containers` as a
-read-only file system, therefore default configuration file can not be edited.
-Fortunately Kata Containers supports loading a configuration file from another
-path than the default.
-
-```sh
-$ sudo mkdir -p /etc/kata-containers
-$ sudo cp /snap/kata-containers/current/usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers/
-$ $EDITOR /etc/kata-containers/configuration.toml
-```
-
-## Integration with shim v2 Container Engines
-
-The Container engine daemon (`cri-o`, `containerd`, etc) needs to be able to find the
-`containerd-shim-kata-v2` binary to allow Kata Containers to be created.
-Run the following command to create a symbolic link to the shim v2 binary.
-
-```sh
-$ sudo ln -sf /snap/kata-containers/current/usr/bin/containerd-shim-kata-v2 /usr/local/bin/containerd-shim-kata-v2
-```
-
-Once the symbolic link has been created and the engine daemon configured, `io.containerd.kata.v2`
-can be used as runtime.
-
-Read the following documents to know how to run Kata Containers 2.x with `containerd`.
-
-* [How to use Kata Containers and Containerd](https://github.com/kata-containers/kata-containers/blob/main/docs/how-to/containerd-kata.md)
-* [Install Kata Containers with containerd](https://github.com/kata-containers/kata-containers/blob/main/docs/install/container-manager/containerd/containerd-install.md)
-
-
-## Remove Kata Containers snap package
-
-Run the following command to remove the Kata Containers snap:
-
-```sh
-$ sudo snap remove kata-containers
-```
+For further information on integrating and configuring the `snap` Kata Containers install,
+refer to the [Kata Containers packaging `snap` documentation](https://github.com/kata-containers/packaging/blob/master/snap/README.md#configure-kata-containers).

docs/install/ubuntu-installation-guide.md

@@ -0,0 +1,15 @@
+# Install Kata Containers on Ubuntu
+
+1. Install the Kata Containers components with the following commands:
+
+   ```bash
+   $ ARCH=$(arch)
+   $ BRANCH="${BRANCH:-master}"
+   $ sudo sh -c "echo 'deb http://download.opensuse.org/repositories/home:/katacontainers:/releases:/${ARCH}:/${BRANCH}/xUbuntu_$(lsb_release -rs)/ /' > /etc/apt/sources.list.d/kata-containers.list"
+   $ curl -sL  http://download.opensuse.org/repositories/home:/katacontainers:/releases:/${ARCH}:/${BRANCH}/xUbuntu_$(lsb_release -rs)/Release.key | sudo apt-key add -
+   $ sudo -E apt-get update
+   $ sudo -E apt-get -y install kata-runtime kata-proxy kata-shim
+   ```
+
+2. Decide which container manager to use and select the corresponding link that follows:
+   - [Kubernetes](../Developer-Guide.md#run-kata-containers-with-kubernetes)

docs/use-cases/using-Intel-QAT-and-kata.md

@@ -1,62 +1,56 @@
 # Table of Contents
 
-- [Table of Contents](#table-of-contents)
-- [Introduction](#introduction)
-  - [Helpful Links before starting](#helpful-links-before-starting)
-  - [Steps to enable Intel® QAT in Kata Containers](#steps-to-enable-intel-qat-in-kata-containers)
-  - [Script variables](#script-variables)
-    - [Set environment variables (Every Reboot)](#set-environment-variables-every-reboot)
-  - [Prepare the Ubuntu Host](#prepare-the-ubuntu-host)
-    - [Identify which PCI Bus the Intel® QAT card is on](#identify-which-pci-bus-the-intel-qat-card-is-on)
-    - [Install necessary packages for Ubuntu](#install-necessary-packages-for-ubuntu)
-    - [Download Intel® QAT drivers](#download-intel-qat-drivers)
-    - [Copy Intel® QAT configuration files and enable virtual functions](#copy-intel-qat-configuration-files-and-enable-virtual-functions)
-    - [Expose and Bind Intel® QAT virtual functions to VFIO-PCI (Every reboot)](#expose-and-bind-intel-qat-virtual-functions-to-vfio-pci-every-reboot)
-    - [Check Intel® QAT virtual functions are enabled](#check-intel-qat-virtual-functions-are-enabled)
-  - [Prepare Kata Containers](#prepare-kata-containers)
-    - [Download Kata kernel Source](#download-kata-kernel-source)
-    - [Build Kata kernel](#build-kata-kernel)
-    - [Copy Kata kernel](#copy-kata-kernel)
-    - [Prepare Kata root filesystem](#prepare-kata-root-filesystem)
-    - [Compile Intel® QAT drivers for Kata Containers kernel and add to Kata Containers rootfs](#compile-intel-qat-drivers-for-kata-containers-kernel-and-add-to-kata-containers-rootfs)
-    - [Copy Kata rootfs](#copy-kata-rootfs)
-  - [Verify Intel® QAT works in a container](#verify-intel-qat-works-in-a-container)
-    - [Build OpenSSL Intel® QAT engine container](#build-openssl-intel-qat-engine-container)
-    - [Test Intel® QAT with the ctr tool](#test-intel-qat-with-the-ctr-tool)
-    - [Test Intel® QAT in Kubernetes](#test-intel-qat-in-kubernetes)
-    - [Troubleshooting](#troubleshooting)
-  - [Optional Scripts](#optional-scripts)
-    - [Verify Intel® QAT card counters are incremented](#verify-intel-qat-card-counters-are-incremented)
+* [Table of Contents](#table-of-contents)
+* [Introduction](#introduction)
+    * [Helpful Links before starting](#helpful-links-before-starting)
+    * [Steps to enable Intel QAT in Kata Containers](#steps-to-enable-intel-qat-in-kata-containers)
+    * [Script variables](#script-variables)
+        * [Set environment variables (Every Reboot)](#set-environment-variables-every-reboot)
+    * [Prepare the Clear Linux Host](#prepare-the-clear-linux-host)
+        * [Identify which PCI Bus the Intel QAT card is on](#identify-which-pci-bus-the-intel-qat-card-is-on)
+        * [Install necessary bundles for Clear Linux](#install-necessary-bundles-for-clear-linux)
+        * [Download Intel QAT drivers](#download-intel-qat-drivers)
+        * [Copy Intel QAT configuration files and enable Virtual Functions](#copy-intel-qat-configuration-files-and-enable-virtual-functions)
+        * [Expose and Bind Intel QAT virtual functions to VFIO-PCI (Every reboot)](#expose-and-bind-intel-qat-virtual-functions-to-vfio-pci-every-reboot)
+        * [Check Intel QAT virtual functions are enabled](#check-intel-qat-virtual-functions-are-enabled)
+    * [Prepare Kata Containers](#prepare-kata-containers)
+        * [Download Kata kernel Source](#download-kata-kernel-source)
+        * [Build Kata kernel](#build-kata-kernel)
+        * [Copy Kata kernel](#copy-kata-kernel)
+        * [Prepare Kata root filesystem](#prepare-kata-root-filesystem)
+        * [Compile Intel QAT drivers for Kata Containers kernel and add to Kata Containers rootfs](#compile-intel-qat-drivers-for-kata-containers-kernel-and-add-to-kata-containers-rootfs)
+        * [Copy Kata rootfs](#copy-kata-rootfs)
+        * [Update Kata configuration to point to custom kernel and rootfs](#update-kata-configuration-to-point-to-custom-kernel-and-rootfs)
+    * [Verify Intel QAT works in a Docker Kata Containers container](#verify-intel-qat-works-in-a-docker-kata-containers-container)
+    * [Build OpenSSL Intel QAT engine container](#build-openssl-intel-qat-engine-container)
+        * [Test Intel QAT in Docker](#test-intel-qat-in-docker)
+        * [Troubleshooting](#troubleshooting)
+    * [Optional Scripts](#optional-scripts)
+        * [Verify Intel QAT card counters are incremented](#verify-intel-qat-card-counters-are-incremented)
 
 # Introduction
 
-Intel® QuickAssist Technology (QAT) provides hardware acceleration 
+Intel QuickAssist Technology (Intel QAT) provides hardware acceleration 
 for security (cryptography) and compression. These instructions cover the 
-steps for the latest [Ubuntu LTS release](https://ubuntu.com/download/desktop) 
-which already include the QAT host driver. These instructions can be adapted to 
-any Linux distribution. These instructions guide the user on how to download 
-the kernel sources, compile kernel driver modules against those sources, and 
-load them onto the host as well as preparing a specially built Kata Containers 
-kernel and custom Kata Containers rootfs.
-
-* Download kernel sources
-* Compile Kata kernel
-* Compile kernel driver modules against those sources
-* Download rootfs
-* Add driver modules to rootfs
-* Build rootfs image 
+steps for [Clear Linux](https://clearlinux.org) but can be adapted to any 
+Linux distribution. Your distribution may already have the Intel QAT 
+drivers, but it is likely they do not contain the necessary user space 
+components. These instructions guide the user on how to download the kernel 
+sources, compile kernel driver modules against those sources, and load them 
+onto the host as well as preparing a specially built Kata Containers kernel 
+and custom Kata Containers rootfs.  
 
 ## Helpful Links before starting
 
-[Intel® QuickAssist Technology at `01.org`](https://01.org/intel-quickassist-technology)
+[Intel QAT Engine](https://github.com/intel/QAT_Engine)
 
-[Intel® QuickAssist Technology Engine for OpenSSL](https://github.com/intel/QAT_Engine)
+[Intel QuickAssist Technology at `01.org`](https://01.org/intel-quickassist-technology)
 
 [Intel Device Plugin for Kubernetes](https://github.com/intel/intel-device-plugins-for-kubernetes)
 
-[Intel® QuickAssist Technology for Crypto Poll Mode Driver](https://dpdk-docs.readthedocs.io/en/latest/cryptodevs/qat.html)
+[Intel QuickAssist Crypto Poll Mode Driver](https://dpdk-docs.readthedocs.io/en/latest/cryptodevs/qat.html)
 
-## Steps to enable Intel® QAT in Kata Containers
+## Steps to enable Intel QAT in Kata Containers
 
 There are some steps to complete only once, some steps to complete with every
 reboot, and some steps to complete when the host kernel changes.
@@ -73,95 +67,91 @@ needed to point to updated drivers or different install locations.
 Make sure to check [`01.org`](https://01.org/intel-quickassist-technology) for 
 the latest driver.
 
-```bash
-$ export QAT_DRIVER_VER=qat1.7.l.4.12.0-00011.tar.gz
-$ export QAT_DRIVER_URL=https://downloadmirror.intel.com/30178/eng/${QAT_DRIVER_VER}
+```sh
+$ export QAT_DRIVER_VER=qat1.7.l.4.8.0-00005.tar.gz 
+$ export QAT_DRIVER_URL=https://01.org/sites/default/files/downloads/${QAT_DRIVER_VER}
 $ export QAT_CONF_LOCATION=~/QAT_conf
 $ export QAT_DOCKERFILE=https://raw.githubusercontent.com/intel/intel-device-plugins-for-kubernetes/master/demo/openssl-qat-engine/Dockerfile
 $ export QAT_SRC=~/src/QAT
 $ export GOPATH=~/src/go
+$ export OSBUILDER=~/src/osbuilder
 $ export KATA_KERNEL_LOCATION=~/kata
 $ export KATA_ROOTFS_LOCATION=~/kata
 ```
 
-## Prepare the Ubuntu Host
+## Prepare the Clear Linux Host
 
 The host could be a bare metal instance or a virtual machine. If using a 
 virtual machine, make sure that KVM nesting is enabled. The following 
-instructions reference an Intel® C62X chipset. Some of the instructions must be 
-modified if using a different Intel® QAT device. The Intel® QAT chipset can be
-identified by executing the following.
+instructions reference an Intel QAT. Some of the instructions must be 
+modified if using a different Intel QAT device. You can identify the Intel QAT
+chipset by executing the following.
 
-### Identify which PCI Bus the Intel® QAT card is on
+### Identify which PCI Bus the Intel QAT card is on
 
-```bash
+```sh
 $ for i in 0434 0435 37c8 1f18 1f19; do lspci -d 8086:$i; done
 ```
 
-### Install necessary packages for Ubuntu
+### Install necessary bundles for Clear Linux
 
-These packages are necessary to compile the Kata kernel, Intel® QAT driver, and to
-prepare the rootfs for Kata. [Docker](https://docs.docker.com/engine/install/ubuntu/)
-also needs to be installed to be able to build the rootfs. To test that 
-everything works a Kubernetes pod is started requesting Intel® QAT resources. For the
-pass through of the virtual functions the kernel boot parameter needs to have
-`INTEL_IOMMU=on`.
+Clear Linux version 30780 (Released August 13, 2019) includes a 
+`linux-firmware-qat` bundle that has the necessary QAT firmware along with a
+functional QAT host driver that works with Kata Containers. 
 
-```bash
-$ sudo apt update
-$ sudo apt install -y golang-go build-essential python pkg-config zlib1g-dev libudev-dev bison libelf-dev flex libtool automake autotools-dev autoconf bc libpixman-1-dev coreutils libssl-dev
-$ sudo sed -i 's/GRUB_CMDLINE_LINUX_DEFAULT=""/GRUB_CMDLINE_LINUX_DEFAULT="intel_iommu=on"/' /etc/default/grub
-$ sudo update-grub
+```sh
+$ sudo swupd bundle-add network-basic linux-firmware-qat make c-basic go-basic containers-virt dev-utils devpkg-elfutils devpkg-systemd devpkg-ssl
+$ sudo clr-boot-manager update
+$ sudo systemctl enable --now docker
 $ sudo reboot
 ```
 
-### Download Intel® QAT drivers
+### Download Intel QAT drivers
 
-This will download the [Intel® QAT drivers](https://01.org/intel-quickassist-technology). 
+This will download the Intel QAT drivers from [`01.org`](https://01.org/intel-quickassist-technology). 
 Make sure to check the website for the latest version.
 
-```bash
+```sh
 $ mkdir -p $QAT_SRC
 $ cd $QAT_SRC
 $ curl -L $QAT_DRIVER_URL | tar zx
 ```
 
-### Copy Intel® QAT configuration files and enable virtual functions
+### Copy Intel QAT configuration files and enable Virtual Functions
 
-Modify the instructions below as necessary if using a different Intel® QAT hardware 
+Modify the instructions below as necessary if using a different QAT hardware 
 platform. You can learn more about customizing configuration files at the 
-[Intel® QAT Engine repository](https://github.com/intel/QAT_Engine/#copy-the-correct-intel-quickassist-technology-driver-config-files)
+[Intel QAT Engine repository](https://github.com/intel/QAT_Engine/#copy-the-correct-intel-quickassist-technology-driver-config-files)
 This section starts from a base config file and changes the `SSL` section to 
 `SHIM` to support the OpenSSL engine. There are more tweaks that you can make
-depending on the use case and how many Intel® QAT engines should be run. You
+depending on the use case and how many Intel QAT engines should be run. You
 can find more information about how to customize in the 
 [Intel® QuickAssist Technology Software for Linux* - Programmer's Guide.](https://01.org/sites/default/files/downloads/336210qatswprogrammersguiderev006.pdf) 
 
-> **Note: This section assumes that a Intel® QAT `c6xx` platform is used.**
+> **Note: This section assumes that a QAT `c6xx` platform is used.**
 
-```bash
+```sh
 $ mkdir -p $QAT_CONF_LOCATION
 $ cp $QAT_SRC/quickassist/utilities/adf_ctl/conf_files/c6xxvf_dev0.conf.vm $QAT_CONF_LOCATION/c6xxvf_dev0.conf
 $ sed -i 's/\[SSL\]/\[SHIM\]/g' $QAT_CONF_LOCATION/c6xxvf_dev0.conf
 ```
 
-### Expose and Bind Intel® QAT virtual functions to VFIO-PCI (Every reboot)
+### Expose and Bind Intel QAT virtual functions to VFIO-PCI (Every reboot)
 
 To enable virtual functions, the host OS should have IOMMU groups enabled. In 
-the UEFI Firmware Intel® Virtualization Technology for Directed I/O 
-(Intel® VT-d) must be enabled. Also, the kernel boot parameter should be 
-`intel_iommu=on` or `intel_iommu=ifgx_off`. This should have been set from
-the instructions above. Check the output of `/proc/cmdline` to confirm. The 
-following commands assume you installed an Intel® QAT card, IOMMU is on, and
+the UEFI Firmware Intel Virtualization Technology for Directed I/O 
+(Intel VT-d) must be enabled. Also, the kernel boot parameter should be 
+`intel_iommu=on` or `intel_iommu=ifgx_off`. The default in Clear Linux currently 
+is `intel_iommu=igfx_off` which should work with the Intel QAT device. The 
+following commands assume you installed an Intel QAT card, IOMMU is on, and
 VT-d is enabled. The vendor and device ID add to the `VFIO-PCI` driver so that
 each exposed virtual function can be bound to the `VFIO-PCI` driver. Once
 complete, each virtual function passes into a Kata Containers container using
-the PCIe device passthrough feature. For Kubernetes, the 
-[Intel device plugin](https://github.com/intel/intel-device-plugins-for-kubernetes)
-for Kubernetes handles the binding of the driver, but the VF’s still must be
+the PCIe device passthrough feature. For Kubernetes, the Intel device plugin
+for Kubernetes handles the binding of the driver but the VF’s still must be
 enabled.
 
-```bash
+```sh
 $ sudo modprobe vfio-pci
 $ QAT_PCI_BUS_PF_NUMBERS=$((lspci -d :435 && lspci -d :37c8 && lspci -d :19e2 && lspci -d :6f54) | cut -d ' ' -f 1)
 $ QAT_PCI_BUS_PF_1=$(echo $QAT_PCI_BUS_PF_NUMBERS | cut -d ' ' -f 1)
@@ -170,10 +160,8 @@ $ QAT_PCI_ID_VF=$(cat /sys/bus/pci/devices/0000:${QAT_PCI_BUS_PF_1}/virtfn0/ueve
 $ QAT_VENDOR_AND_ID_VF=$(echo ${QAT_PCI_ID_VF/PCI_ID=} | sed 's/:/ /')
 $ echo $QAT_VENDOR_AND_ID_VF | sudo tee --append /sys/bus/pci/drivers/vfio-pci/new_id
 ```
-
 Loop through all the virtual functions and bind to the VFIO driver
-
-```bash
+```sh
 $ for f in /sys/bus/pci/devices/0000:$QAT_PCI_BUS_PF_1/virtfn*
   do QAT_PCI_BUS_VF=$(basename $(readlink $f))
    echo $QAT_PCI_BUS_VF | sudo tee --append /sys/bus/pci/drivers/c6xxvf/unbind
@@ -181,23 +169,22 @@ $ for f in /sys/bus/pci/devices/0000:$QAT_PCI_BUS_PF_1/virtfn*
   done
 ```
 
-### Check Intel® QAT virtual functions are enabled
+### Check Intel QAT virtual functions are enabled
 
 If the following command returns empty, then the virtual functions are not 
 properly enabled. This command checks the enumerated device IDs for just the 
-virtual functions. Using the Intel® QAT as an example, the physical device ID 
+virtual functions. Using the Intel QAT as an example, the physical device ID 
 is `37c8` and virtual function device ID is `37c9`. The following command checks 
-if VF's are enabled for any of the currently known Intel® QAT device ID's. The
+if VF's are enabled for any of the currently known Intel QAT device ID's. The
 following `ls` command should show the 16 VF's bound to `VFIO-PCI`.
 
-```bash
+```sh
 $ for i in 0442 0443 37c9 19e3; do lspci -d 8086:$i; done
 ```
 
 Another way to check is to see what PCI devices that `VFIO-PCI` is mapped to.
 It should match the device ID's of the VF's.
-
-```bash
+```sh
 $ ls -la /sys/bus/pci/drivers/vfio-pci
 ```
 
@@ -214,16 +201,16 @@ There are some patches that must be installed as well, which the
 `build-kernel.sh` script should automatically apply. If you are using a
 different kernel version, then you might need to manually apply them. Since
 the Kata Containers kernel has a minimal set of kernel flags set, you must
-create a Intel® QAT kernel fragment with the necessary `CONFIG_CRYPTO_*` options set.
+create a QAT kernel fragment with the necessary `CONFIG_CRYPTO_*` options set.
 Update the config to set some of the `CRYPTO` flags to enabled. This might
-change with different kernel versions. The following instructions were tested
-with kernel `v5.4.0-64-generic`.
+change with different kernel versions. We tested the following instructions
+with kernel `v4.19.28-41`.
 
-```bash
+```sh
 $ mkdir -p $GOPATH
 $ cd $GOPATH
-$ go get -v github.com/kata-containers/kata-containers
-$ cat << EOF > $GOPATH/src/github.com/kata-containers/kata-containers/tools/packaging/kernel/configs/fragments/common/qat.conf
+$ go get -v github.com/kata-containers/packaging
+$ cat << EOF > $GOPATH/src/github.com/kata-containers/packaging/kernel/configs/fragments/common/qat.conf
 CONFIG_PCIEAER=y
 CONFIG_UIO=y
 CONFIG_CRYPTO_HW=y
@@ -234,70 +221,61 @@ CONFIG_MODULE_SIG=y
 CONFIG_CRYPTO_AUTHENC=y
 CONFIG_CRYPTO_DH=y
 EOF
-$ $GOPATH/src/github.com/kata-containers/kata-containers/tools/packaging/kernel/build-kernel.sh setup
+$ $GOPATH/src/github.com/kata-containers/packaging/kernel/build-kernel.sh setup
 ```
 
 ### Build Kata kernel
 
-```bash
-$ cd $GOPATH
-$ export LINUX_VER=$(ls -d kata-linux-*)
+```sh
+$ export LINUX_VER=$(ls -d kata*)
 $ sed -i 's/EXTRAVERSION =/EXTRAVERSION = .qat.container/' $LINUX_VER/Makefile
-$ $GOPATH/src/github.com/kata-containers/kata-containers/tools/packaging/kernel/build-kernel.sh build
+$ $GOPATH/src/github.com/kata-containers/packaging/kernel/build-kernel.sh build
 ```
 
+
 ### Copy Kata kernel
 
-```bash
-$ export KATA_KERNEL_NAME=vmlinux-${LINUX_VER}_qat
+```sh
 $ mkdir -p $KATA_KERNEL_LOCATION
-$ cp ${GOPATH}/${LINUX_VER}/vmlinux ${KATA_KERNEL_LOCATION}/${KATA_KERNEL_NAME}
+$ cp $LINUX_VER/arch/x86/boot/bzImage $KATA_KERNEL_LOCATION/vmlinuz-${LINUX_VER}_qat
 ```
 
 ### Prepare Kata root filesystem
 
 These instructions build upon the OS builder instructions located in the 
-[Developer Guide](../Developer-Guide.md). At this point it is recommended that
-[Docker](https://docs.docker.com/engine/install/ubuntu/) is installed first, and
-then [Kata-deploy](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy)
-is use to install Kata. This will make sure that the correct `agent` version 
-is installed into the rootfs in the steps below.
+[Developer Guide](../Developer-Guide.md). The following instructions use Clear
+Linux (Kata Containers default) as the root filesystem with systemd as the 
+init and will add in the `kmod` binary, which is not a standard binary in a 
+Kata rootfs image. The `kmod` binary is necessary to load the QAT kernel 
+modules when the virtual machine rootfs boots. You should install Docker on
+your system before running the following commands. If you need to use a custom 
+`kata-agent`, then refer to the previous link on how to add it in.
 
-The following instructions use Debian as the root filesystem with systemd as 
-the init and will add in the `kmod` binary, which is not a standard binary in 
-a Kata rootfs image. The `kmod` binary is necessary to load the Intel® QAT 
-kernel modules when the virtual machine rootfs boots. 
-
-```bash
-$ export OSBUILDER=$GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder
-$ export ROOTFS_DIR=${OSBUILDER}/rootfs-builder/rootfs
+```sh
+$ mkdir -p $OSBUILDER
+$ cd $OSBUILDER
+$ git clone https://github.com/kata-containers/osbuilder.git
+$ export ROOTFS_DIR=${OSBUILDER}/osbuilder/rootfs-builder/rootfs
 $ export EXTRA_PKGS='kmod'
 ```
-
 Make sure that the `kata-agent` version matches the installed `kata-runtime`
-version. Also make sure the `kata-runtime` install location is in your `PATH` 
-variable. The following `AGENT_VERSION` can be set manually to match
-the `kata-runtime` version if the following commands don't work.
-
-```bash
-$ export PATH=$PATH:/opt/kata/bin
-$ cd $GOPATH
+version.
+```sh
 $ export AGENT_VERSION=$(kata-runtime version | head -n 1 | grep -o "[0-9.]\+")
-$ cd ${OSBUILDER}/rootfs-builder
+$ cd ${OSBUILDER}/osbuilder/rootfs-builder
 $ sudo rm -rf ${ROOTFS_DIR}
-$ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true SECCOMP=no ./rootfs.sh debian'
+$ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true SECCOMP=no ./rootfs.sh clearlinux'
 ```
 
-### Compile Intel® QAT drivers for Kata Containers kernel and add to Kata Containers rootfs
+### Compile Intel QAT drivers for Kata Containers kernel and add to Kata Containers rootfs
 
 After the Kata Containers kernel builds with the proper configuration flags, 
-you must build the Intel® QAT drivers against that Kata Containers kernel
+you must build the Intel QAT drivers against that Kata Containers kernel
 version in a similar way they were previously built for the host OS. You must 
 set the `KERNEL_SOURCE_ROOT` variable to the Kata Containers kernel source 
-directory and build the Intel® QAT drivers again. The  `make` command will
-install the Intel® QAT modules into the Kata rootfs.
+directory and build the Intel QAT drivers again.
 
-```bash
+```sh
 $ cd $GOPATH
 $ export LINUX_VER=$(ls -d kata*)
 $ export KERNEL_MAJOR_VERSION=$(awk '/^VERSION =/{print $NF}' $GOPATH/$LINUX_VER/Makefile)
@@ -306,18 +284,16 @@ $ export KERNEL_SUBLEVEL=$(awk '/^SUBLEVEL =/{print $NF}' $GOPATH/$LINUX_VER/Mak
 $ export KERNEL_EXTRAVERSION=$(awk '/^EXTRAVERSION =/{print $NF}' $GOPATH/$LINUX_VER/Makefile)
 $ export KERNEL_ROOTFS_DIR=${KERNEL_MAJOR_VERSION}.${KERNEL_PATHLEVEL}.${KERNEL_SUBLEVEL}${KERNEL_EXTRAVERSION}
 $ cd $QAT_SRC
-$ KERNEL_SOURCE_ROOT=$GOPATH/$LINUX_VER ./configure --enable-icp-sriov=guest
+$ KERNEL_SOURCE_ROOT=$GOPATH/$LINUX_VER ./configure --disable-qat-lkcf --enable-icp-sriov=guest
 $ sudo -E make all -j$(nproc)
 $ sudo -E make INSTALL_MOD_PATH=$ROOTFS_DIR qat-driver-install -j$(nproc)
 ```
-
 The `usdm_drv` module also needs to be copied into the rootfs modules path and
 `depmod` should be run. 
-
-```bash
-$ sudo cp $QAT_SRC/build/usdm_drv.ko $ROOTFS_DIR/lib/modules/${KERNEL_ROOTFS_DIR}/updates/drivers  
+```sh
+$ sudo cp $QAT_SRC/build/usdm_drv.ko $ROOTFS_DIR/usr/lib/modules/${KERNEL_ROOTFS_DIR}/updates/drivers  
 $ sudo depmod -a -b ${ROOTFS_DIR} ${KERNEL_ROOTFS_DIR}
-$ cd ${OSBUILDER}/image-builder
+$ cd ${OSBUILDER}/osbuilder/image-builder
 $ script -fec 'sudo -E USE_DOCKER=true ./image_builder.sh ${ROOTFS_DIR}'
 ```
 
@@ -326,225 +302,84 @@ $ script -fec 'sudo -E USE_DOCKER=true ./image_builder.sh ${ROOTFS_DIR}'
 
 ### Copy Kata rootfs
 
-```bash
+```sh
 $ mkdir -p $KATA_ROOTFS_LOCATION
-$ cp ${OSBUILDER}/image-builder/kata-containers.img $KATA_ROOTFS_LOCATION
+$ cp ${OSBUILDER}/osbuilder/image-builder/kata-containers.img $KATA_ROOTFS_LOCATION
 ```
 
-## Verify Intel® QAT works in a container
+### Update Kata configuration to point to custom kernel and rootfs
 
-The following instructions uses a OpenSSL Dockerfile that builds the 
-Intel® QAT engine to allow OpenSSL to offload crypto functions. It is a 
-convenient way to test that VFIO device passthrough for the Intel® QAT VF’s are
+You must update the `configuration.toml` for Kata Containers to point to the 
+custom kernel, custom rootfs, and to specify which modules to load when the 
+virtual machine is booted when a container is run. The following example
+assumes you installed an Intel QAT, and you need to load those modules.
+
+```sh
+$ sudo mkdir -p /etc/kata-containers
+$ sudo cp /usr/share/defaults/kata-containers/configuration-qemu.toml /etc/kata-containers/configuration.toml
+$ sudo sed -i "s|kernel_params = \"\"|kernel_params = \"modules-load=usdm_drv,qat_c62xvf\"|g" /etc/kata-containers/configuration.toml
+$ sudo sed -i "s|\/usr\/share\/kata-containers\/kata-containers.img|${KATA_KERNEL_LOCATION}\/kata-containers.img|g" /etc/kata-containers/configuration.toml
+$ sudo sed -i "s|\/usr\/share\/kata-containers\/vmlinuz.container|${KATA_ROOTFS_LOCATION}\/vmlinuz-${LINUX_VER}_qat|g" /etc/kata-containers/configuration.toml
+```
+
+## Verify Intel QAT works in a Docker Kata Containers container
+
+The following instructions leverage an OpenSSL Dockerfile that builds the 
+Intel QAT engine to allow OpenSSL to offload crypto functions. It is a 
+convenient way to test that VFIO device passthrough for the Intel QAT VF’s are
 working properly with the Kata Containers VM.
 
-### Build OpenSSL Intel® QAT engine container
+## Build OpenSSL Intel QAT engine container
 
-Use the OpenSSL Intel® QAT [Dockerfile](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/master/demo/openssl-qat-engine) 
+Use the OpenSSL Intel QAT [Dockerfile](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/master/demo/openssl-qat-engine) 
 to build a container image with an optimized OpenSSL engine for 
-Intel® QAT. Using `docker build` with the Kata Containers runtime can sometimes
-have issues. Therefore, make sure that `runc` is the default Docker container 
-runtime.
+Intel QAT. Using `docker build` with the Kata Containers runtime can sometimes
+have issues. Therefore, we recommended you change the default runtime to
+`runc` before doing a build. Instructions for this are below.
 
-```bash
+```sh
 $ cd $QAT_SRC
 $ curl -O $QAT_DOCKERFILE
+$ sudo sed -i 's/kata-runtime/runc/g' /etc/systemd/system/docker.service.d/50-runtime.conf
+$ sudo systemctl daemon-reload && sudo systemctl restart docker
 $ sudo docker build -t openssl-qat-engine .
 ```
 
-> **Note: The Intel® QAT driver version in this container might not match the 
-> Intel® QAT driver compiled and loaded on the host when compiling.**
+> **Note: The Intel QAT driver version in this container might not match the 
+> Intel QAT driver compiled and loaded on the host when compiling.**
 
-### Test Intel® QAT with the ctr tool
+### Test Intel QAT in Docker
 
-The `ctr` tool can be used to interact with the containerd daemon. It may be 
-more convenient to use this tool to verify the kernel and image instead of
-setting up a Kubernetes cluster. The correct Kata runtimes need to be added
-to the containerd `config.toml`. Below is a sample snippet that can be added
-to allow QEMU and Cloud Hypervisor (CLH) to work with `ctr`.
+The host should already be setup with 16 virtual functions of the Intel QAT 
+card bound to `VFIO-PCI`. Verify this by looking in `/dev/vfio` for a listing
+of devices. Replace the number 90 with one of the VF’s exposed in `/dev/vfio`.
+It might require you to add an `IPC_LOCK` capability to your Docker runtime
+depending on which rootfs you use.
 
-```
-[plugins.cri.containerd.runtimes.kata-qemu]
-  runtime_type = "io.containerd.kata-qemu.v2"
-  privileged_without_host_devices = true
-  pod_annotations = ["io.katacontainers.*"]
-  [plugins.cri.containerd.runtimes.kata-qemu.options]
-    ConfigPath = "/opt/kata/share/defaults/kata-containers/configuration-qemu.toml"
-[plugins.cri.containerd.runtimes.kata-clh]
-  runtime_type = "io.containerd.kata-clh.v2"
-  privileged_without_host_devices = true
-  pod_annotations = ["io.katacontainers.*"]
-  [plugins.cri.containerd.runtimes.kata-clh.options]
-    ConfigPath = "/opt/kata/share/defaults/kata-containers/configuration-clh.toml"
+```sh
+$ sudo docker run -it --runtime=kata-runtime --cap-add=IPC_LOCK --cap-add=SYS_ADMIN --device=/dev/vfio/90 -v /dev:/dev -v ${QAT_CONF_LOCATION}:/etc openssl-qat-engine bash
 ```
 
-In addition, containerd expects the binary to be in `/usr/local/bin` so add 
-this small script so that it redirects to be able to use either QEMU or
-Cloud Hypervisor with Kata.
-
-```bash
-$ echo '#!/bin/bash' | sudo tee /usr/local/bin/containerd-shim-kata-qemu-v2
-$ echo 'KATA_CONF_FILE=/opt/kata/share/defaults/kata-containers/configuration-qemu.toml /opt/kata/bin/containerd-shim-kata-v2 $@' | sudo tee -a /usr/local/bin/containerd-shim-kata-qemu-v2
-$ sudo chmod +x /usr/local/bin/containerd-shim-kata-qemu-v2
-$ echo '#!/bin/bash' | sudo tee /usr/local/bin/containerd-shim-kata-clh-v2
-$ echo 'KATA_CONF_FILE=/opt/kata/share/defaults/kata-containers/configuration-clh.toml /opt/kata/bin/containerd-shim-kata-v2 $@' | sudo tee -a /usr/local/bin/containerd-shim-kata-clh-v2
-$ sudo chmod +x /usr/local/bin/containerd-shim-kata-clh-v2
-```
-
-After the OpenSSL image is built and imported into containerd, a Intel® QAT 
-virtual function exposed in the step above can be added to the `ctr` command. 
-Make sure to change the `/dev/vfio` number to one that actually exists on the 
-host system. When using the `ctr` tool, the`configuration.toml` for Kata needs 
-to point to the custom Kata kernel and rootfs built above and the Intel® QAT 
-modules in the Kata rootfs need to load at boot. The following steps assume that 
-`kata-deploy` was used to install Kata and QEMU is being tested. If using a 
-different hypervisor, different install method for Kata, or a different 
-Intel® QAT chipset then the command will need to be modified. 
-
-> **Note: The following was tested with 
-[containerd v1.3.9](https://github.com/containerd/containerd/releases/tag/v1.3.9).**
-
-```bash
-$ config_file="/opt/kata/share/defaults/kata-containers/configuration-qemu.toml"
-$ sudo sed -i "/kernel =/c kernel = "\"${KATA_ROOTFS_LOCATION}/${KATA_KERNEL_NAME}\""" $config_file
-$ sudo sed -i "/image =/c image = "\"${KATA_KERNEL_LOCATION}/kata-containers.img\""" $config_file
-$ sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 modules-load=usdm_drv,qat_c62xvf"/g' $config_file 
-$ sudo docker save -o openssl-qat-engine.tar openssl-qat-engine:latest
-$ sudo ctr images import openssl-qat-engine.tar
-$ sudo ctr run --runtime io.containerd.run.kata-qemu.v2 --privileged -t --rm --device=/dev/vfio/180 --mount type=bind,src=/dev,dst=/dev,options=rbind:rw --mount type=bind,src=${QAT_CONF_LOCATION}/c6xxvf_dev0.conf,dst=/etc/c6xxvf_dev0.conf,options=rbind:rw  docker.io/library/openssl-qat-engine:latest bash
-```
-
-Below are some commands to run in the container image to verify Intel® QAT is 
+Below are some commands to run in the container image to verify Intel QAT is 
 working
 
 ```sh
-root@67561dc2757a/ # cat /proc/modules
-qat_c62xvf 16384 - - Live 0xffffffffc00d9000 (OE)
-usdm_drv 86016 - - Live 0xffffffffc00e8000 (OE)
-intel_qat 249856 - - Live 0xffffffffc009b000 (OE)
-
-root@67561dc2757a/ # adf_ctl restart
-Restarting all devices.
-Processing /etc/c6xxvf_dev0.conf
-
-root@67561dc2757a/ # adf_ctl status
-Checking status of all devices.
-There is 1 QAT acceleration device(s) in the system:
- qat_dev0 - type: c6xxvf,  inst_id: 0,  node_id: 0,  bsf: 0000:01:01.0,  #accel: 1 #engines: 1 state: up
-
-root@67561dc2757a/ # openssl engine -c -t qat-hw
-(qat-hw) Reference implementation of QAT crypto engine v0.6.1
- [RSA, DSA, DH, AES-128-CBC-HMAC-SHA1, AES-128-CBC-HMAC-SHA256, AES-256-CBC-HMAC-SHA1, AES-256-CBC-HMAC-SHA256, TLS1-PRF, HKDF, X25519, X448]
-     [ available ]
+bash-5.0# cat /proc/modules
+bash-5.0# adf_ctl restart
+bash-5.0# adf_ctl status
+bash-5.0# openssl engine -c -t qat
 ```
 
-### Test Intel® QAT in Kubernetes
-
-Start a Kubernetes cluster with containerd as the CRI. The host should 
-already be setup with 16 virtual functions of the Intel® QAT card bound to 
-`VFIO-PCI`. Verify this by looking in `/dev/vfio` for a listing of devices. 
-You might need to disable Docker before initializing Kubernetes. Be aware 
-that the OpenSSL container image built above will need to be exported from
-Docker and imported into containerd.
-
-If Kata is installed through [`kata-deploy`](https://github.com/kata-containers/kata-containers/blob/stable-2.0/tools/packaging/kata-deploy/README.md)
-there will be multiple `configuration.toml` files associated with different 
-hypervisors. Rather than add in the custom Kata kernel, Kata rootfs, and 
-kernel modules to each `configuration.toml` as the default, instead use
-[annotations](https://github.com/kata-containers/kata-containers/blob/stable-2.0/docs/how-to/how-to-load-kernel-modules-with-kata.md)
-in the Kubernetes YAML file to tell Kata which kernel and rootfs to use. The 
-easy way to do this is to use `kata-deploy` which will install the Kata binaries
-to `/opt` and properly configure the `/etc/containerd/config.toml` with annotation 
-support. However, the `configuration.toml` needs to enable support for
-annotations as well. The following configures both QEMU and Cloud Hypervisor
-`configuration.toml` files that are currently available with Kata Container 
-versions 2.0 and higher.
-
-```bash
-$ sudo sed -i 's/enable_annotations\s=\s\[\]/enable_annotations = [".*"]/' /opt/kata/share/defaults/kata-containers/configuration-qemu.toml
-$ sudo sed -i 's/enable_annotations\s=\s\[\]/enable_annotations = [".*"]/' /opt/kata/share/defaults/kata-containers/configuration-clh.toml
-```
-
-Export the OpenSSL image from Docker and import into containerd.
-
-```bash
-$ sudo docker save -o openssl-qat-engine.tar openssl-qat-engine:latest
-$ sudo ctr -n=k8s.io images import openssl-qat-engine.tar
-```
-
-The [Intel® QAT Plugin](https://github.com/intel/intel-device-plugins-for-kubernetes/blob/master/cmd/qat_plugin/README.md)
-needs to be started so that the virtual functions can be discovered and
-used by Kubernetes. 
-
-The following YAML file can be used to start a Kata container with Intel® QAT
-support. If Kata is installed with `kata-deploy`, then the containerd 
-`configuration.toml` should have all of the Kata runtime classes already 
-populated and annotations supported. To use a Intel® QAT virtual function, the 
-Intel® QAT plugin needs to be started after the VF's are bound to `VFIO-PCI` as 
-described [above](#expose-and-bind-intel-qat-virtual-functions-to-vfio-pci-every-reboot). 
-Edit the following to point to the correct Kata kernel and rootfs location 
-built with Intel® QAT support.
-
-```bash
-$ cat << EOF > kata-openssl-qat.yaml
-apiVersion: v1
-kind: Pod
-metadata:
-  name: kata-openssl-qat
-  labels:
-    app: kata-openssl-qat
-  annotations:
-    io.katacontainers.config.hypervisor.kernel: "$KATA_KERNEL_LOCATION/$KATA_KERNEL_NAME"
-    io.katacontainers.config.hypervisor.image: "$KATA_ROOTFS_LOCATION/kata-containers.img"
-    io.katacontainers.config.hypervisor.kernel_params: "modules-load=usdm_drv,qat_c62xvf"
-spec:
-  runtimeClassName: kata-qemu
-  containers:
-  - name: kata-openssl-qat
-    image: docker.io/library/openssl-qat-engine:latest
-    imagePullPolicy: IfNotPresent
-    resources:
-      limits:
-        qat.intel.com/generic: 1
-        cpu: 1
-    securityContext:
-      capabilities:
-        add: ["IPC_LOCK", "SYS_ADMIN"]
-    volumeMounts:
-      - mountPath: /etc/c6xxvf_dev0.conf
-        name: etc-mount
-      - mountPath: /dev
-        name: dev-mount
-  volumes:
-    - name: dev-mount
-      hostPath:
-        path: /dev
-    - name: etc-mount
-      hostPath:
-        path: $QAT_CONF_LOCATION/c6xxvf_dev0.conf
-EOF
-```
-
-Use `kubectl` to start the pod. Verify that Intel® QAT card acceleration is 
-working with the Intel® QAT engine.
-```bash
-$ kubectl apply -f kata-openssl-qat.yaml
-```
+Test with Intel QAT card acceleration
 
 ```sh
-$ kubectl exec -it kata-openssl-qat -- adf_ctl restart
-Restarting all devices.
-Processing /etc/c6xxvf_dev0.conf
+bash-5.0# openssl speed -engine qat -elapsed -async_jobs 72 rsa2048 
+```
 
-$ kubectl exec -it kata-openssl-qat -- adf_ctl status
-Checking status of all devices.
-There is 1 QAT acceleration device(s) in the system:
- qat_dev0 - type: c6xxvf,  inst_id: 0,  node_id: 0,  bsf: 0000:01:01.0,  #accel: 1 #engines: 1 state: up
+Test with CPU acceleration
 
-$ kubectl exec -it kata-openssl-qat -- openssl engine -c -t qat-hw
-(qat-hw) Reference implementation of QAT crypto engine v0.6.1
- [RSA, DSA, DH, AES-128-CBC-HMAC-SHA1, AES-128-CBC-HMAC-SHA256, AES-256-CBC-HMAC-SHA1, AES-256-CBC-HMAC-SHA256, TLS1-PRF, HKDF, X25519, X448]
-     [ available ]
+```sh
+bash-5.0# openssl speed -elapsed rsa2048
 ```
 
 ### Troubleshooting
@@ -577,9 +412,9 @@ c6xxvf_dev10.conf  c6xxvf_dev13.conf  c6xxvf_dev2.conf   c6xxvf_dev5.conf c6xxvf
 ```
 
 * Check `dmesg` inside the container to see if there are any issues with the 
-Intel® QAT driver.
+Intel QAT driver.
 
-* If there are issues building the OpenSSL Intel® QAT container image, then 
+* If there are issues building the OpenSSL Intel QAT container image, then 
 check to make sure that runc is the default runtime for building container.
 
 ```sh
@@ -590,18 +425,17 @@ Environment="DOCKER_DEFAULT_RUNTIME=--default-runtime runc"
 
 ## Optional Scripts
 
-### Verify Intel® QAT card counters are incremented
+### Verify Intel QAT card counters are incremented
 
-To check the built in firmware counters, the Intel® QAT driver has to be compiled 
-and installed to the host and can't rely on the built in host driver. The 
-counters will increase when the accelerator is actively being used. To verify 
-Intel® QAT is actively accelerating the containerized application, use the 
-following instructions to check if any of the counters increment. Make 
-sure to change the PCI Device ID to match whats in the system.
+Use the `lspci` command to figure out which PCI bus the Intel QAT accelerators
+are on. The counters will increase when the accelerator is actively being
+used. To verify QAT is actively accelerating the containerized application,
+use the following instructions to check if any of the counters are
+incrementing. You will have to change the PCI device ID to match your system.
 
-```bash
+```sh
 $ for i in 0434 0435 37c8 1f18 1f19; do lspci -d 8086:$i; done
 $ sudo watch cat /sys/kernel/debug/qat_c6xx_0000\:b1\:00.0/fw_counters
 $ sudo watch cat /sys/kernel/debug/qat_c6xx_0000\:b3\:00.0/fw_counters
 $ sudo watch cat /sys/kernel/debug/qat_c6xx_0000\:b5\:00.0/fw_counters
-```
+```

docs/use-cases/using-Intel-SGX-and-kata.md

@@ -1,112 +0,0 @@
-# Kata Containers with SGX
-
-- [Check if SGX is enabled](#check-if-sgx-is-enabled)
-- [Install Host kernel with SGX support](#install-host-kernel-with-sgx-support)
-- [Install Guest kernel with SGX support](#install-guest-kernel-with-sgx-support)
-- [Run Kata Containers with SGX enabled](#run-kata-containers-with-sgx-enabled)
-
-Intel® Software Guard Extensions (SGX) is a set of instructions that increases the security
-of applications code and data, giving them more protections from disclosure or modification.
-
-> **Note:** At the time of writing this document, SGX patches have not landed on the Linux kernel
-> project, so specific versions for guest and host kernels must be installed to enable SGX.
-
-## Check if SGX is enabled
-
-Run the following command to check if your host supports SGX.
-
-```sh
-$ grep -o sgx /proc/cpuinfo
-```
-
-Continue to the following section if the output of the above command is empty,
-otherwise continue to section [Install Guest kernel with SGX support](#install-guest-kernel-with-sgx-support)
-
-## Install Host kernel with SGX support
-
-The following commands were tested on Fedora 32, they might work on other distros too.
-
-```sh
-$ git clone --depth=1 https://github.com/intel/kvm-sgx
-$ pushd kvm-sgx
-$ cp /boot/config-$(uname -r) .config
-$ yes "" | make oldconfig
-$ # In the following step, enable: INTEL_SGX and INTEL_SGX_VIRTUALIZATION
-$ make menuconfig
-$ make -j$(($(nproc)-1)) bzImage
-$ make -j$(($(nproc)-1)) modules
-$ sudo make modules_install
-$ sudo make install
-$ popd
-$ sudo reboot
-```
-
-> **Notes:**
-> * Run: `mokutil --sb-state` to check whether secure boot is enabled, if so, you will need to sign the kernel.
-> * You'll lose SGX support when a new distro kernel is installed and the system rebooted.
-
-Once you have restarted your system with the new brand Linux Kernel with SGX support, run
-the following command to make sure it's enabled. If the output is empty, go to the BIOS
-setup and enable SGX manually.
-
-```sh
-$ grep -o sgx /proc/cpuinfo
-```
-
-## Install Guest kernel with SGX support
-
-Install the guest kernel in the Kata Containers directory, this way it can be used to run
-Kata Containers.
-
-```sh
-$ curl -LOk https://github.com/devimc/kvm-sgx/releases/download/v0.0.1/kata-virtiofs-sgx.tar.gz
-$ sudo tar -xf kata-virtiofs-sgx.tar.gz -C /usr/share/kata-containers/
-$ sudo sed -i 's|kernel =|kernel = "/usr/share/kata-containers/vmlinux-virtiofs-sgx.container"|g' \
-  /usr/share/defaults/kata-containers/configuration.toml
-```
-
-## Run Kata Containers with SGX enabled
-
-Before running a Kata Container make sure that your version of `crio` or `containerd`
-supports annotations.
-For `containerd` check in `/etc/containerd/config.toml` that the list of `pod_annotations` passed
-to the `sandbox` are: `["io.katacontainers.*", "sgx.intel.com/epc"]`.
-
-> `sgx.yaml`
-```yaml
-apiVersion: v1
-kind: Pod
-metadata:
-  name: sgx
-  annotations:
-    sgx.intel.com/epc: "32Mi"
-spec:
-  terminationGracePeriodSeconds: 0
-  runtimeClassName: kata
-  containers:
-  - name: c1
-    image: busybox
-    command:
-        - sh
-    stdin: true
-    tty: true
-    volumeMounts:
-    - mountPath: /dev/sgx/
-      name: test-volume
-  volumes:
-  - name: test-volume
-    hostPath:
-      path: /dev/sgx/
-      type: Directory
-```
-
-```sh
-$ kubectl apply -f sgx.yaml
-$ kubectl exec -ti sgx ls /dev/sgx/
-enclave    provision
-```
-
-The output of the latest command shouldn't be empty, otherwise check
-your system environment to make sure SGX is fully supported.
-
-[1]: github.com/cloud-hypervisor/cloud-hypervisor/

docs/use-cases/zun_kata.md

@@ -10,6 +10,9 @@ Currently, the instructions are based on the following links:
 
 - https://docs.openstack.org/zun/latest/admin/clear-containers.html
 
+- ../install/ubuntu-installation-guide.md
+
+
 ## Install Git to use with DevStack
 
 ```sh
@@ -51,7 +54,7 @@ $ zun delete test
 
 ## Install Kata Containers
 
-Follow [these instructions](../install/README.md)
+Follow [these instructions](../install/ubuntu-installation-guide.md)
 to install the Kata Containers components.
 
 ## Update Docker with new Kata Containers runtime

pkg/logging/src/lib.rs

@@ -21,12 +21,7 @@ const LOG_LEVELS: &[(&str, slog::Level)] = &[
 ];
 
 // XXX: 'writer' param used to make testing possible.
-pub fn create_logger<W>(
-    name: &str,
-    source: &str,
-    level: slog::Level,
-    writer: W,
-) -> (slog::Logger, slog_async::AsyncGuard)
+pub fn create_logger<W>(name: &str, source: &str, level: slog::Level, writer: W) -> slog::Logger
 where
     W: Write + Send + Sync + 'static,
 {
@@ -42,21 +37,17 @@ where
     let filter_drain = RuntimeLevelFilter::new(unique_drain, level).fuse();
 
     // Ensure the logger is thread-safe
-    let (async_drain, guard) = slog_async::Async::new(filter_drain)
-        .thread_name("slog-async-logger".into())
-        .build_with_guard();
+    let async_drain = slog_async::Async::new(filter_drain).build().fuse();
 
     // Add some "standard" fields
-    let logger = slog::Logger::root(
+    slog::Logger::root(
         async_drain.fuse(),
         o!("version" => env!("CARGO_PKG_VERSION"),
             "subsystem" => "root",
             "pid" => process::id().to_string(),
             "name" => name.to_string(),
             "source" => source.to_string()),
-    );
-
-    (logger, guard)
+    )
 }
 
 pub fn get_log_levels() -> Vec<&'static str> {
@@ -102,7 +93,9 @@ impl HashSerializer {
         // Take care to only add the first instance of a key. This matters for loggers (but not
         // Records) since a child loggers have parents and the loggers are serialised child first
         // meaning the *newest* fields are serialised first.
-        self.fields.entry(key).or_insert(value);
+        if !self.fields.contains_key(&key) {
+            self.fields.insert(key, value);
+        }
     }
 
     fn remove_field(&mut self, key: &str) {

snap/snapcraft.yaml

@@ -21,10 +21,20 @@ parts:
       version="9999"
       kata_url="https://github.com/kata-containers/kata-containers"
 
-      if echo "${GITHUB_REF}" | grep -q -E "^refs/tags"; then
-        version=$(echo ${GITHUB_REF} | cut -d/ -f3)
-        git checkout ${version}
-      fi
+      image_info="${SNAPCRAFT_IMAGE_INFO:-}"
+      snap_env="$(echo "${image_info}" | egrep -o "build_url.*" | egrep -o "snap.*build" | cut -d/ -f2)"
+
+      case "${snap_env}" in
+        stable)
+          # Get the latest stable version
+          version=$(git ls-remote --tags ${kata_url}  | egrep -o "refs.*" | egrep -v "\-alpha|\-rc|{}" | egrep -o "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+" | sort -V -r | head -1)
+          git checkout ${version}
+        ;;
+
+        *-dev)
+          version="${snap_env}"
+        ;;
+      esac
 
       snapcraftctl set-grade "stable"
       snapcraftctl set-version "${version}"
@@ -57,10 +67,15 @@ parts:
         *) echo "unsupported architecture: $(uname -m)"; exit 1;;
       esac
 
-      yq_version=3.4.1
+      # Workaround to get latest release from github (to not use github token).
+      # Get the redirection to latest release on github.
+      yq_latest_url=$(curl -Ls -o /dev/null -w %{url_effective} "https://${yq_pkg}/releases/latest")
+      # The redirected url should include the latest release version
+      # https://github.com/mikefarah/yq/releases/tag/<VERSION-HERE>
+      yq_version=$(basename "${yq_latest_url}")
       yq_url="https://${yq_pkg}/releases/download/${yq_version}/yq_${goos}_${goarch}"
-      curl -o "${yq_path}" -LSsf "${yq_url}"
-      chmod +x "${yq_path}"
+      curl -o "${yq_path}" -LSsf ${yq_url}
+      chmod +x ${yq_path}
 
       kata_dir=gopath/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
       version="$(${yq_path} r ${kata_dir}/versions.yaml languages.golang.meta.newest-version)"
@@ -69,7 +84,7 @@ parts:
       tar -xf ${tarfile} --strip-components=1
 
   image:
-    after: [godeps, qemu, kernel]
+    after: [godeps]
     plugin: nil
     build-packages:
       - docker.io
@@ -89,8 +104,6 @@ parts:
       export GOROOT=${SNAPCRAFT_STAGE}
       export PATH="${GOROOT}/bin:${PATH}"
 
-      http_proxy=${http_proxy:-""}
-      https_proxy=${https_proxy:-""}
       if [ -n "$http_proxy" ]; then
         echo "Setting proxy $http_proxy"
         sudo -E systemctl set-environment http_proxy=$http_proxy || true
@@ -171,7 +184,7 @@ parts:
       fi
 
   kernel:
-    after: [godeps]
+    after: [godeps, image]
     plugin: nil
     build-packages:
       - libelf-dev
@@ -185,8 +198,8 @@ parts:
 
       cd ${kata_dir}/tools/packaging/kernel
 
-      # Setup and build kernel
-      ./build-kernel.sh -d setup
+      # Say 'no' to everithing, fix issues with incomplete .config files
+      yes "n" | ./build-kernel.sh setup
       kernel_dir_prefix="kata-linux-"
       cd ${kernel_dir_prefix}*
       version=$(basename ${PWD} | sed 's|'"${kernel_dir_prefix}"'||' | cut -d- -f1)
@@ -208,7 +221,7 @@ parts:
 
   qemu:
     plugin: make
-    after: [godeps]
+    after: [godeps, runtime]
     build-packages:
       - gcc
       - python3
@@ -228,7 +241,6 @@ parts:
       - libffi-dev
       - libmount-dev
       - libselinux1-dev
-      - ninja-build
     override-build: |
       yq=${SNAPCRAFT_STAGE}/yq
       export GOPATH=${SNAPCRAFT_STAGE}/gopath
@@ -245,11 +257,10 @@ parts:
         ;;
 
         *)
-          branch="$(${yq} r ${versions_file} assets.hypervisor.qemu.version)"
+          branch="$(${yq} r ${versions_file} assets.hypervisor.qemu.tag)"
           url="$(${yq} r ${versions_file} assets.hypervisor.qemu.url)"
           commit=""
           patches_dir="${kata_dir}/tools/packaging/qemu/patches/$(echo ${branch} | sed -e 's/.[[:digit:]]*$//' -e 's/^v//').x"
-          patches_version_dir="${kata_dir}/tools/packaging/qemu/patches/tag_patches/${branch}"
         ;;
       esac
 
@@ -262,23 +273,31 @@ parts:
       [ -n "$(ls -A ui/keycodemapdb)" ] || git clone https://github.com/qemu/keycodemapdb ui/keycodemapdb/
       [ -n "$(ls -A capstone)" ] || git clone https://github.com/qemu/capstone capstone
 
-      # Apply branch patches
-      ${kata_dir}/tools/packaging/scripts/apply_patches.sh "${patches_dir}"
-      ${kata_dir}/tools/packaging/scripts/apply_patches.sh "${patches_version_dir}"
+      # Apply patches
+      for patch in ${patches_dir}/*.patch; do
+        echo "Applying $(basename "$patch") ..."
+        patch \
+          --batch \
+          --forward \
+          --strip 1 \
+          --input "$patch"
+      done
 
       # Only x86_64 supports libpmem
       [ "$(uname -m)" = "x86_64" ] && sudo apt-get --no-install-recommends install -y apt-utils ca-certificates libpmem-dev libseccomp-dev
 
       configure_hypervisor=${kata_dir}/tools/packaging/scripts/configure-hypervisor.sh
       chmod +x ${configure_hypervisor}
-      # static build. The --prefix, --libdir, --libexecdir, --datadir arguments are
-      # based on PREFIX and set by configure-hypervisor.sh
-      echo "$(PREFIX=/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr ${configure_hypervisor} -s kata-qemu) \
-        --disable-rbd " \
+      # static build
+      echo "$(${configure_hypervisor} -s qemu) \
+        --disable-rbd
+        --prefix=/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr \
+        --datadir=/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr/share \
+        --libexecdir=/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr/libexec/qemu" \
         | xargs ./configure
 
       # Copy QEMU configurations (Kconfigs)
-      cp -a ${kata_dir}/tools/packaging/qemu/default-configs/* default-configs/devices/
+      cp -a ${kata_dir}/tools/packaging/qemu/default-configs/* default-configs/
 
       # build and install
       make -j $(($(nproc)-1))
@@ -289,6 +308,7 @@ parts:
       - -usr/bin/qemu-pr-helper
       - -usr/bin/virtfs-proxy-helper
       - -usr/include/
+      - -usr/libexec/
       - -usr/share/applications/
       - -usr/share/icons/
       - -usr/var/
@@ -300,8 +320,4 @@ parts:
 
 apps:
   runtime:
-    command: usr/bin/kata-runtime
-  shim:
     command: usr/bin/containerd-shim-kata-v2
-  collect-data:
-    command: usr/bin/kata-collect-data.sh

src/agent/.gitignore

@@ -1 +0,0 @@
-tarpaulin-report.html

src/agent/.rustfmt.toml

@@ -1 +0,0 @@
-edition = "2018"

src/agent/Cargo.toml

@@ -9,50 +9,35 @@ oci = { path = "oci" }
 logging = { path = "../../pkg/logging" }
 rustjail = { path = "rustjail" }
 protocols = { path = "protocols" }
+netlink = { path = "netlink", features = ["with-log", "with-agent-handler"] }
 lazy_static = "1.3.0"
-ttrpc = { version = "0.5.0", features = ["async", "protobuf-codec"], default-features = false }
+ttrpc = { git = "https://github.com/containerd/ttrpc-rust.git", branch="0.3.0" }
 protobuf = "=2.14.0"
 libc = "0.2.58"
 nix = "0.17.0"
 prctl = "1.0.0"
 serde_json = "1.0.39"
+signal-hook = "0.1.9"
 scan_fmt = "0.2.3"
 scopeguard = "1.0.0"
 regex = "1"
-
-async-trait = "0.1.42"
-tokio = { version = "1.2.0", features = ["rt", "rt-multi-thread", "sync", "macros", "io-util", "time", "signal", "io-std", "process", "fs"] }
-futures = "0.3.12"
-netlink-sys = { version = "0.6.0", features = ["tokio_socket",]}
-tokio-vsock = "0.3.0"
-rtnetlink = "0.7.0"
-netlink-packet-utils = "0.4.0"
-ipnetwork = "0.17.0"
-
 # slog:
 # - Dynamic keys required to allow HashMap keys to be slog::Serialized.
 # - The 'max_*' features allow changing the log level at runtime
 #   (by stopping the compiler from removing log calls).
 slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_info"] }
 slog-scope = "4.1.2"
-
-# Redirect ttrpc log calls
-slog-stdlog = "4.0.0"
-log = "0.4.11"
-
 # for testing
 tempfile = "3.1.0"
 prometheus = { version = "0.9.0", features = ["process"] }
 procfs = "0.7.9"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.2.5" }
+cgroups = { git = "https://github.com/kata-containers/cgroups-rs", branch = "stable-0.1.1"}
 
 [workspace]
 members = [
+    "netlink",
     "oci",
     "protocols",
     "rustjail",
 ]
-
-[profile.release]
-lto = true

src/agent/Makefile

@@ -3,11 +3,6 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-# To show variables or targets help on `make help`
-# Use the following format:
-# '##VAR VARIABLE_NAME: help about variable'
-# '##TARGET TARGET_NAME: help about target'
-
 PROJECT_NAME = Kata Containers
 PROJECT_URL = https://github.com/kata-containers
 PROJECT_COMPONENT = kata-agent
@@ -21,18 +16,21 @@ SOURCES := \
 VERSION_FILE := ./VERSION
 VERSION := $(shell grep -v ^\# $(VERSION_FILE))
 COMMIT_NO := $(shell git rev-parse HEAD 2>/dev/null || true)
+COMMIT_NO_SHORT := $(shell git rev-parse --short HEAD 2>/dev/null || true)
 COMMIT := $(if $(shell git status --porcelain --untracked-files=no 2>/dev/null || true),${COMMIT_NO}-dirty,${COMMIT_NO})
 COMMIT_MSG = $(if $(COMMIT),$(COMMIT),unknown)
 
 # Exported to allow cargo to see it
 export VERSION_COMMIT := $(if $(COMMIT),$(VERSION)-$(COMMIT),$(VERSION))
 
-##VAR BUILD_TYPE=release|debug type of rust build
 BUILD_TYPE = release
 
-##VAR ARCH=arch target to build (format: uname -m)
+# set proto file to generate
+ifdef proto
+  PROTO_FILE=${proto}
+endif
+
 ARCH = $(shell uname -m)
-##VAR LIBC=musl|gnu
 LIBC ?= musl
 ifneq ($(LIBC),musl)
     ifeq ($(LIBC),gnu)
@@ -48,11 +46,6 @@ ifeq ($(ARCH), ppc64le)
     $(warning "WARNING: powerpc64le-unknown-linux-musl target is unavailable")
 endif
 
-ifeq ($(ARCH), s390x)
-    override LIBC = gnu
-    $(warning "WARNING: s390x-unknown-linux-musl target is unavailable")
-endif
-
 
 EXTRA_RUSTFLAGS :=
 ifeq ($(ARCH), aarch64)
@@ -64,12 +57,10 @@ TRIPLE = $(ARCH)-unknown-linux-$(LIBC)
 
 TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET)
 
-##VAR DESTDIR=<path> is a directory prepended to each installed target file
 DESTDIR :=
-##VAR BINDIR=<path> is a directory for installing executable programs
 BINDIR := /usr/bin
 
-##VAR INIT=yes|no define if agent will be installed as init
+# Define if agent will be installed as init
 INIT := no
 
 # Path to systemd unit directory if installed as not init.
@@ -117,7 +108,6 @@ define INSTALL_FILE
 	install -D -m 644 $1 $(DESTDIR)$2/$1 || exit 1;
 endef
 
-##TARGET default: build code
 default: $(TARGET) show-header
 
 $(TARGET): $(GENERATED_CODE) $(TARGET_PATH)
@@ -125,55 +115,36 @@ $(TARGET): $(GENERATED_CODE) $(TARGET_PATH)
 $(TARGET_PATH): $(SOURCES) | show-summary
 	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE)
 
-$(GENERATED_FILES): %: %.in
-	@sed $(foreach r,$(GENERATED_REPLACEMENTS),-e 's|@$r@|$($r)|g') "$<" > "$@"
-
-##TARGET optimize: optimized  build
 optimize: $(SOURCES) | show-summary show-header
 	@RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny-warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE)
 
+show-header:
+	@printf "%s - version %s (commit %s)\n\n" "$(TARGET)" "$(VERSION)" "$(COMMIT_MSG)"
 
-##TARGET clippy: run clippy linter
-clippy: $(GENERATED_CODE)
-	cargo clippy --all-targets --all-features --release \
-		-- \
-		-Aclippy::redundant_allocation \
-		-D warnings
+$(GENERATED_FILES): %: %.in
+	@sed $(foreach r,$(GENERATED_REPLACEMENTS),-e 's|@$r@|$($r)|g') "$<" > "$@"
 
-format:
-	cargo fmt -- --check
-
-
-##TARGET install: install agent
-install: install-services
+install: build-service
 	@install -D $(TARGET_PATH) $(DESTDIR)/$(BINDIR)/$(TARGET)
 
-##TARGET clean: clean build
 clean:
 	@cargo clean
 	@rm -f $(GENERATED_FILES)
-	@rm -f tarpaulin-report.html
 
-#TARGET test: run cargo tests
 test:
 	@cargo test --all --target $(TRIPLE)
 
-##TARGET check: run test
-check: clippy format
+check: test
 
-##TARGET run: build and run agent
 run:
 	@cargo run --target $(TRIPLE)
 
-install-services: $(GENERATED_FILES)
+build-service: $(GENERATED_FILES)
 ifeq ($(INIT),no)
 	@echo "Installing systemd unit files..."
 	$(foreach f,$(UNIT_FILES),$(call INSTALL_FILE,$f,$(UNIT_DIR)))
 endif
 
-show-header:
-	@printf "%s - version %s (commit %s)\n\n" "$(TARGET)" "$(VERSION)" "$(COMMIT_MSG)"
-
 show-summary: show-header
 	@printf "project:\n"
 	@printf "  name: $(PROJECT_NAME)\n"
@@ -189,35 +160,7 @@ show-summary: show-header
 	@printf "  %s\n" "$(call get_toolchain_version)"
 	@printf "\n"
 
-## help: Show help comments that start with `##VAR` and `##TARGET`
-help: Makefile show-summary
-	@echo "==========================Help============================="
-	@echo "Variables:"
-	@sed -n 's/^##VAR//p' $< | sort
-	@echo ""
-	@echo "Targets:"
-	@sed -n 's/^##TARGET//p' $< | sort
-
-TARPAULIN_ARGS:=-v --workspace
-install-tarpaulin:
-	cargo install cargo-tarpaulin
-
-# Check if cargo tarpaulin is installed
-HAS_TARPAULIN:= $(shell cargo --list | grep tarpaulin 2>/dev/null)
-check_tarpaulin:
-ifndef  HAS_TARPAULIN
-	$(error "tarpaulin is not available please: run make install-tarpaulin ")
-else
-	$(info OK: tarpaulin installed)
-endif
-
-##TARGET codecov: Generate code coverage report
-codecov: check_tarpaulin
-	cargo tarpaulin $(TARPAULIN_ARGS)
-
-##TARGET codecov-html: Generate code coverage html report
-codecov-html: check_tarpaulin
-	cargo tarpaulin $(TARPAULIN_ARGS) -o Html
+help: show-summary
 
 .PHONY: \
 	help \
@@ -225,6 +168,5 @@ codecov-html: check_tarpaulin
 	show-summary \
 	optimize
 
-##TARGET generate-protocols: generate/update grpc agent protocols
 generate-protocols:
-	protocols/hack/update-generated-proto.sh all
+	protocols/hack/update-generated-proto.sh "${PROTO_FILE}"

src/agent/README.md

@@ -39,27 +39,11 @@ After that, we drafted the initial code here, and any contributions are welcome.
 ## Getting Started
 
 ### Build from Source
-The rust-agent needs to be built statically and linked with `musl`
-
-> **Note:** skip this step for ppc64le, the build scripts explicitly use gnu for ppc64le.
-
+The rust-agent need to be built with rust newer than 1.37, and static linked with `musl`.
 ```bash
-$ arch=$(uname -m)
-$ rustup target add "${arch}-unknown-linux-musl"
-$ sudo ln -s /usr/bin/g++ /bin/musl-g++
-```
-
-ppc64le-only: Manually install `protoc`, e.g.
-```bash
-$ sudo dnf install protobuf-compiler
-```
-
-Download the source files in the Kata containers repository and build the agent:
-```bash
-$ GOPATH="${GOPATH:-$HOME/go}"
-$ dir="$GOPATH/src/github.com/kata-containers"
-$ git -C ${dir} clone --depth 1 https://github.com/kata-containers/kata-containers
-$ make -C ${dir}/kata-containers/src/agent
+rustup target add x86_64-unknown-linux-musl
+sudo ln -s /usr/bin/g++ /bin/musl-g++  
+cargo build --target x86_64-unknown-linux-musl --release
 ```
 
 ## Run Kata CI with rust-agent

src/agent/kata-agent.service.in

@@ -15,10 +15,8 @@ Wants=kata-containers.target
 StandardOutput=tty
 Type=simple
 ExecStart=@BINDIR@/@AGENT_NAME@
-LimitNOFILE=1048576
+LimitNOFILE=infinity
 # ExecStop is required for static agent tracing; in all other scenarios
 # the runtime handles shutting down the VM.
 ExecStop=/bin/sync ; /usr/bin/systemctl --force poweroff
 FailureAction=poweroff
-# Discourage OOM-killer from touching the agent
-OOMScoreAdjust=-997

src/agent/netlink/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "netlink"
+version = "0.1.0"
+authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+libc = "0.2.58"
+nix = "0.17.0"
+
+protobuf = { version = "=2.14.0", optional = true }
+protocols = { path = "../protocols", optional = true }
+slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_info"], optional = true }
+slog-scope = { version = "4.1.2", optional = true }
+
+[features]
+with-log = ["slog", "slog-scope"]
+with-agent-handler = ["protobuf", "protocols"]

src/agent/netlink/src/agent_handler.rs

@@ -0,0 +1,572 @@
+// Copyright (c) 2020 Ant Financial
+// Copyright (C) 2020 Alibaba Cloud. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+//! Dedicated Netlink interfaces for Kata agent protocol handler.
+
+use std::convert::TryFrom;
+
+use protobuf::RepeatedField;
+use protocols::types::{ARPNeighbor, IPAddress, IPFamily, Interface, Route};
+
+use super::*;
+
+#[cfg(feature = "with-log")]
+// Convenience macro to obtain the scope logger
+macro_rules! sl {
+    () => {
+        slog_scope::logger().new(o!("subsystem" => "netlink"))
+    };
+}
+
+impl super::RtnlHandle {
+    pub fn update_interface(&mut self, iface: &Interface) -> Result<Interface> {
+        // the reliable way to find link is using hardware address
+        // as filter. However, hardware filter might not be supported
+        // by netlink, we may have to dump link list and the find the
+        // target link. filter using name or family is supported, but
+        // we cannot use that to find target link.
+        // let's try if hardware address filter works. -_-
+
+        let ifinfo = self.find_link_by_hwaddr(iface.hwAddr.as_str())?;
+
+        // bring down interface if it is up
+        if ifinfo.ifi_flags & libc::IFF_UP as u32 != 0 {
+            self.set_link_status(&ifinfo, false)?;
+        }
+
+        // delete all addresses associated with the link
+        let del_addrs: Vec<RtIPAddr> = self.get_link_addresses(&ifinfo)?;
+        self.delete_all_addrs(&ifinfo, del_addrs.as_ref())?;
+
+        // add new ip addresses in request
+        for grpc_addr in &iface.IPAddresses {
+            let rtip = RtIPAddr::try_from(grpc_addr.clone())?;
+            self.add_one_address(&ifinfo, &rtip)?;
+        }
+
+        let mut v: Vec<u8> = vec![0; DEFAULT_NETLINK_BUF_SIZE];
+        // Safe because we have allocated enough buffer space.
+        let nlh = unsafe { &mut *(v.as_mut_ptr() as *mut nlmsghdr) };
+        let ifi = unsafe { &mut *(NLMSG_DATA!(nlh) as *mut ifinfomsg) };
+
+        // set name, set mtu, IFF_NOARP. in one rtnl_talk.
+        nlh.nlmsg_len = NLMSG_LENGTH!(mem::size_of::<ifinfomsg>() as u32) as __u32;
+        nlh.nlmsg_type = RTM_NEWLINK;
+        nlh.nlmsg_flags = NLM_F_REQUEST;
+        self.assign_seqnum(nlh);
+
+        ifi.ifi_family = ifinfo.ifi_family;
+        ifi.ifi_type = ifinfo.ifi_type;
+        ifi.ifi_index = ifinfo.ifi_index;
+        if iface.raw_flags & libc::IFF_NOARP as u32 != 0 {
+            ifi.ifi_change |= libc::IFF_NOARP as u32;
+            ifi.ifi_flags |= libc::IFF_NOARP as u32;
+        }
+
+        // Safe because we have allocated enough buffer space.
+        unsafe {
+            nlh.addattr32(IFLA_MTU, iface.mtu as u32);
+
+            // if str is null terminated, use addattr_var.
+            // otherwise, use addattr_str
+            nlh.addattr_var(IFLA_IFNAME, iface.name.as_ref());
+        }
+
+        self.rtnl_talk(v.as_mut_slice(), false)?;
+
+        // TODO: why the result is ignored here?
+        let _ = self.set_link_status(&ifinfo, true);
+
+        Ok(iface.clone())
+    }
+
+    /// Delete this interface/link per request
+    pub fn remove_interface(&mut self, iface: &Interface) -> Result<Interface> {
+        let ifinfo = self.find_link_by_hwaddr(iface.hwAddr.as_str())?;
+
+        self.set_link_status(&ifinfo, false)?;
+
+        let mut v: Vec<u8> = vec![0; DEFAULT_NETLINK_BUF_SIZE];
+        // Safe because we have allocated enough buffer space.
+        let nlh = unsafe { &mut *(v.as_mut_ptr() as *mut nlmsghdr) };
+        let ifi = unsafe { &mut *(NLMSG_DATA!(nlh) as *mut ifinfomsg) };
+
+        // No attributes needed?
+        nlh.nlmsg_len = NLMSG_LENGTH!(mem::size_of::<ifinfomsg>()) as __u32;
+        nlh.nlmsg_type = RTM_DELLINK;
+        nlh.nlmsg_flags = NLM_F_REQUEST;
+        self.assign_seqnum(nlh);
+
+        ifi.ifi_family = ifinfo.ifi_family;
+        ifi.ifi_index = ifinfo.ifi_index;
+        ifi.ifi_type = ifinfo.ifi_type;
+
+        self.rtnl_talk(v.as_mut_slice(), false)?;
+
+        Ok(iface.clone())
+    }
+
+    pub fn list_interfaces(&mut self) -> Result<Vec<Interface>> {
+        let mut ifaces: Vec<Interface> = Vec::new();
+        let (_slv, lv) = self.dump_all_links()?;
+        let (_sav, av) = self.dump_all_addresses(0)?;
+
+        for link in &lv {
+            // Safe because dump_all_links() returns valid pointers.
+            let nlh = unsafe { &**link };
+            if nlh.nlmsg_type != RTM_NEWLINK && nlh.nlmsg_type != RTM_DELLINK {
+                continue;
+            }
+
+            if nlh.nlmsg_len < NLMSG_SPACE!(mem::size_of::<ifinfomsg>()) {
+                info!(
+                    sl!(),
+                    "invalid nlmsg! nlmsg_len: {}, nlmsg_space: {}",
+                    nlh.nlmsg_len,
+                    NLMSG_SPACE!(mem::size_of::<ifinfomsg>())
+                );
+                break;
+            }
+
+            // Safe because we have just validated available buffer space above.
+            let ifi = unsafe { &*(NLMSG_DATA!(nlh) as *const ifinfomsg) };
+            let rta: *mut rtattr = IFLA_RTA!(ifi as *const ifinfomsg) as *mut rtattr;
+            let rtalen = IFLA_PAYLOAD!(nlh) as u32;
+            let attrs = unsafe { parse_attrs(rta, rtalen, (IFLA_MAX + 1) as usize)? };
+
+            // fill out some fields of Interface,
+            let mut iface: Interface = Interface::default();
+
+            // Safe because parse_attrs() returns valid pointers.
+            unsafe {
+                if !attrs[IFLA_IFNAME as usize].is_null() {
+                    let t = attrs[IFLA_IFNAME as usize];
+                    iface.name = String::from_utf8(getattr_var(t as *const rtattr))?;
+                }
+
+                if !attrs[IFLA_MTU as usize].is_null() {
+                    let t = attrs[IFLA_MTU as usize];
+                    iface.mtu = getattr32(t) as u64;
+                }
+
+                if !attrs[IFLA_ADDRESS as usize].is_null() {
+                    let alen = RTA_PAYLOAD!(attrs[IFLA_ADDRESS as usize]);
+                    let a: *const u8 = RTA_DATA!(attrs[IFLA_ADDRESS as usize]) as *const u8;
+                    iface.hwAddr = parser::format_address(a, alen as u32)?;
+                }
+            }
+
+            // get ip address info from av
+            let mut ads: Vec<IPAddress> = Vec::new();
+            for address in &av {
+                // Safe because dump_all_addresses() returns valid pointers.
+                let alh = unsafe { &**address };
+                if alh.nlmsg_type != RTM_NEWADDR {
+                    continue;
+                }
+
+                let tlen = NLMSG_SPACE!(mem::size_of::<ifaddrmsg>());
+                if alh.nlmsg_len < tlen {
+                    info!(
+                        sl!(),
+                        "invalid nlmsg! nlmsg_len: {}, nlmsg_space: {}", alh.nlmsg_len, tlen
+                    );
+                    break;
+                }
+
+                // Safe becahse we have checked avialable buffer space by NLMSG_SPACE above.
+                let ifa = unsafe { &*(NLMSG_DATA!(alh) as *const ifaddrmsg) };
+                let arta: *mut rtattr = IFA_RTA!(ifa) as *mut rtattr;
+                let artalen = IFA_PAYLOAD!(alh) as u32;
+
+                if ifa.ifa_index as u32 == ifi.ifi_index as u32 {
+                    // found target addresses, parse attributes and fill out Interface
+                    let addrs = unsafe { parse_attrs(arta, artalen, (IFA_MAX + 1) as usize)? };
+
+                    // fill address field of Interface
+                    let mut one: IPAddress = IPAddress::default();
+                    let tattr: *const rtattr = if !addrs[IFA_ADDRESS as usize].is_null() {
+                        addrs[IFA_ADDRESS as usize]
+                    } else {
+                        addrs[IFA_LOCAL as usize]
+                    };
+
+                    one.mask = format!("{}", ifa.ifa_prefixlen);
+                    one.family = IPFamily::v4;
+                    if ifa.ifa_family == libc::AF_INET6 as u8 {
+                        one.family = IPFamily::v6;
+                    }
+
+                    // Safe because parse_attrs() returns valid pointers.
+                    unsafe {
+                        let a: *const u8 = RTA_DATA!(tattr) as *const u8;
+                        let alen = RTA_PAYLOAD!(tattr);
+                        one.address = parser::format_address(a, alen as u32)?;
+                    }
+
+                    ads.push(one);
+                }
+            }
+
+            iface.IPAddresses = RepeatedField::from_vec(ads);
+            ifaces.push(iface);
+        }
+
+        Ok(ifaces)
+    }
+
+    pub fn update_routes(&mut self, rt: &[Route]) -> Result<Vec<Route>> {
+        let rs = self.get_all_routes()?;
+        self.delete_all_routes(&rs)?;
+
+        for grpcroute in rt {
+            if grpcroute.gateway.as_str() == "" {
+                let r = RtRoute::try_from(grpcroute.clone())?;
+                if r.index == -1 {
+                    continue;
+                }
+                self.add_one_route(&r)?;
+            }
+        }
+
+        for grpcroute in rt {
+            if grpcroute.gateway.as_str() != "" {
+                let r = RtRoute::try_from(grpcroute.clone())?;
+                if r.index == -1 {
+                    continue;
+                }
+                self.add_one_route(&r)?;
+            }
+        }
+
+        Ok(rt.to_owned())
+    }
+
+    pub fn list_routes(&mut self) -> Result<Vec<Route>> {
+        // currently, only dump routes from main table for ipv4
+        // ie, rtmsg.rtmsg_family = AF_INET, set RT_TABLE_MAIN
+        // attribute in dump request
+        // Fix Me: think about othe tables, ipv6..
+        let mut rs: Vec<Route> = Vec::new();
+        let (_srv, rv) = self.dump_all_routes()?;
+
+        // parse out routes and store in rs
+        for r in &rv {
+            // Safe because dump_all_routes() returns valid pointers.
+            let nlh = unsafe { &**r };
+            if nlh.nlmsg_type != RTM_NEWROUTE && nlh.nlmsg_type != RTM_DELROUTE {
+                info!(sl!(), "not route message!");
+                continue;
+            }
+            let tlen = NLMSG_SPACE!(mem::size_of::<rtmsg>());
+            if nlh.nlmsg_len < tlen {
+                info!(
+                    sl!(),
+                    "invalid nlmsg! nlmsg_len: {}, nlmsg_spae: {}", nlh.nlmsg_len, tlen
+                );
+                break;
+            }
+
+            // Safe because we have just validated available buffer space above.
+            let rtm = unsafe { &mut *(NLMSG_DATA!(nlh) as *mut rtmsg) };
+            if rtm.rtm_table != RT_TABLE_MAIN as u8 {
+                continue;
+            }
+            let rta: *mut rtattr = RTM_RTA!(rtm) as *mut rtattr;
+            let rtalen = RTM_PAYLOAD!(nlh) as u32;
+            let attrs = unsafe { parse_attrs(rta, rtalen, (RTA_MAX + 1) as usize)? };
+
+            let t = attrs[RTA_TABLE as usize];
+            if !t.is_null() {
+                // Safe because parse_attrs() returns valid pointers
+                let table = unsafe { getattr32(t) };
+                if table != RT_TABLE_MAIN {
+                    continue;
+                }
+            }
+
+            // find source, destination, gateway, scope, and and device name
+            let mut t = attrs[RTA_DST as usize];
+            let mut rte: Route = Route::default();
+
+            // Safe because parse_attrs() returns valid pointers
+            unsafe {
+                // destination
+                if !t.is_null() {
+                    let data: *const u8 = RTA_DATA!(t) as *const u8;
+                    let len = RTA_PAYLOAD!(t) as u32;
+                    rte.dest =
+                        format!("{}/{}", parser::format_address(data, len)?, rtm.rtm_dst_len);
+                }
+
+                // gateway
+                t = attrs[RTA_GATEWAY as usize];
+                if !t.is_null() {
+                    let data: *const u8 = RTA_DATA!(t) as *const u8;
+                    let len = RTA_PAYLOAD!(t) as u32;
+                    rte.gateway = parser::format_address(data, len)?;
+
+                    // for gateway, destination is 0.0.0.0
+                    rte.dest = "0.0.0.0".to_string();
+                }
+
+                // source
+                t = attrs[RTA_SRC as usize];
+                if t.is_null() {
+                    t = attrs[RTA_PREFSRC as usize];
+                }
+                if !t.is_null() {
+                    let data: *const u8 = RTA_DATA!(t) as *const u8;
+                    let len = RTA_PAYLOAD!(t) as u32;
+                    rte.source = parser::format_address(data, len)?;
+
+                    if rtm.rtm_src_len != 0 {
+                        rte.source = format!("{}/{}", rte.source.as_str(), rtm.rtm_src_len);
+                    }
+                }
+
+                // scope
+                rte.scope = rtm.rtm_scope as u32;
+
+                // oif
+                t = attrs[RTA_OIF as usize];
+                if !t.is_null() {
+                    let data = &*(RTA_DATA!(t) as *const i32);
+                    assert_eq!(RTA_PAYLOAD!(t), 4);
+
+                    rte.device = self
+                        .get_name_by_index(*data)
+                        .unwrap_or_else(|_| "unknown".to_string());
+                }
+            }
+
+            rs.push(rte);
+        }
+
+        Ok(rs)
+    }
+
+    pub fn add_arp_neighbors(&mut self, neighs: &[ARPNeighbor]) -> Result<()> {
+        for neigh in neighs {
+            self.add_one_arp_neighbor(&neigh)?;
+        }
+
+        Ok(())
+    }
+
+    pub fn add_one_arp_neighbor(&mut self, neigh: &ARPNeighbor) -> Result<()> {
+        let to_ip = match neigh.toIPAddress.as_ref() {
+            None => return nix_errno(Errno::EINVAL),
+            Some(v) => {
+                if v.address.is_empty() {
+                    return nix_errno(Errno::EINVAL);
+                }
+                v.address.as_ref()
+            }
+        };
+
+        let dev = self.find_link_by_name(&neigh.device)?;
+
+        let mut v: Vec<u8> = vec![0; DEFAULT_NETLINK_BUF_SIZE];
+        // Safe because we have allocated enough buffer space.
+        let nlh = unsafe { &mut *(v.as_mut_ptr() as *mut nlmsghdr) };
+        let ndm = unsafe { &mut *(NLMSG_DATA!(nlh) as *mut ndmsg) };
+
+        nlh.nlmsg_len = NLMSG_LENGTH!(std::mem::size_of::<ndmsg>()) as u32;
+        nlh.nlmsg_type = RTM_NEWNEIGH;
+        nlh.nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL;
+        self.assign_seqnum(nlh);
+
+        ndm.ndm_family = libc::AF_UNSPEC as __u8;
+        ndm.ndm_state = IFA_F_PERMANENT as __u16;
+        // process lladdr
+        if neigh.lladdr != "" {
+            let llabuf = parser::parse_mac_addr(&neigh.lladdr)?;
+
+            // Safe because we have allocated enough buffer space.
+            unsafe { nlh.addattr_var(NDA_LLADDR, llabuf.as_ref()) };
+        }
+
+        let (family, ip_data) = parser::parse_ip_addr_with_family(&to_ip)?;
+        ndm.ndm_family = family;
+        // Safe because we have allocated enough buffer space.
+        unsafe { nlh.addattr_var(NDA_DST, ip_data.as_ref()) };
+
+        // process state
+        if neigh.state != 0 {
+            ndm.ndm_state = neigh.state as __u16;
+        }
+
+        // process flags
+        ndm.ndm_flags = (*ndm).ndm_flags | neigh.flags as __u8;
+
+        // process dev
+        ndm.ndm_ifindex = dev.ifi_index;
+
+        // send
+        self.rtnl_talk(v.as_mut_slice(), false)?;
+
+        Ok(())
+    }
+}
+
+impl TryFrom<IPAddress> for RtIPAddr {
+    type Error = nix::Error;
+
+    fn try_from(ipi: IPAddress) -> std::result::Result<Self, Self::Error> {
+        let ip_family = if ipi.family == IPFamily::v4 {
+            libc::AF_INET
+        } else {
+            libc::AF_INET6
+        } as __u8;
+
+        let ip_mask = parser::parse_u8(ipi.mask.as_str(), 10)?;
+        let addr = parser::parse_ip_addr(ipi.address.as_ref())?;
+
+        Ok(Self {
+            ip_family,
+            ip_mask,
+            addr,
+        })
+    }
+}
+
+impl TryFrom<Route> for RtRoute {
+    type Error = nix::Error;
+
+    fn try_from(r: Route) -> std::result::Result<Self, Self::Error> {
+        // only handle ipv4
+
+        let index = {
+            let mut rh = RtnlHandle::new(NETLINK_ROUTE, 0)?;
+            match rh.find_link_by_name(r.device.as_str()) {
+                Ok(ifi) => ifi.ifi_index,
+                Err(_) => -1,
+            }
+        };
+
+        let (dest, dst_len) = if r.dest.is_empty() {
+            (Some(vec![0 as u8; 4]), 0)
+        } else {
+            let (dst, mask) = parser::parse_cidr(r.dest.as_str())?;
+            (Some(dst), mask)
+        };
+
+        let (source, src_len) = if r.source.is_empty() {
+            (None, 0)
+        } else {
+            let (src, mask) = parser::parse_cidr(r.source.as_str())?;
+            (Some(src), mask)
+        };
+
+        let gateway = if r.gateway.is_empty() {
+            None
+        } else {
+            Some(parser::parse_ip_addr(r.gateway.as_str())?)
+        };
+
+        Ok(Self {
+            dest,
+            source,
+            src_len,
+            dst_len,
+            index,
+            gateway,
+            scope: r.scope as u8,
+            protocol: RTPROTO_UNSPEC,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{RtnlHandle, NETLINK_ROUTE};
+    use protocols::types::IPAddress;
+    use std::process::Command;
+
+    fn clean_env_for_test_add_one_arp_neighbor(dummy_name: &str, ip: &str) {
+        // ip link delete dummy
+        Command::new("ip")
+            .args(&["link", "delete", dummy_name])
+            .output()
+            .expect("prepare: failed to delete dummy");
+
+        // ip neigh del dev dummy ip
+        Command::new("ip")
+            .args(&["neigh", "del", dummy_name, ip])
+            .output()
+            .expect("prepare: failed to delete neigh");
+    }
+
+    fn prepare_env_for_test_add_one_arp_neighbor(dummy_name: &str, ip: &str) {
+        clean_env_for_test_add_one_arp_neighbor(dummy_name, ip);
+        // modprobe dummy
+        Command::new("modprobe")
+            .arg("dummy")
+            .output()
+            .expect("failed to run modprobe dummy");
+
+        // ip link add dummy type dummy
+        Command::new("ip")
+            .args(&["link", "add", dummy_name, "type", "dummy"])
+            .output()
+            .expect("failed to add dummy interface");
+
+        // ip addr add 192.168.0.2/16 dev dummy
+        Command::new("ip")
+            .args(&["addr", "add", "192.168.0.2/16", "dev", dummy_name])
+            .output()
+            .expect("failed to add ip for dummy");
+
+        // ip link set dummy up;
+        Command::new("ip")
+            .args(&["link", "set", dummy_name, "up"])
+            .output()
+            .expect("failed to up dummy");
+    }
+
+    #[test]
+    fn test_add_one_arp_neighbor() {
+        // skip_if_not_root
+        if !nix::unistd::Uid::effective().is_root() {
+            println!("INFO: skipping {} which needs root", module_path!());
+            return;
+        }
+
+        let mac = "6a:92:3a:59:70:aa";
+        let to_ip = "169.254.1.1";
+        let dummy_name = "dummy_for_arp";
+
+        prepare_env_for_test_add_one_arp_neighbor(dummy_name, to_ip);
+
+        let mut ip_address = IPAddress::new();
+        ip_address.set_address(to_ip.to_string());
+
+        let mut neigh = ARPNeighbor::new();
+        neigh.set_toIPAddress(ip_address);
+        neigh.set_device(dummy_name.to_string());
+        neigh.set_lladdr(mac.to_string());
+        neigh.set_state(0x80);
+
+        let mut rtnl = RtnlHandle::new(NETLINK_ROUTE, 0).unwrap();
+
+        rtnl.add_one_arp_neighbor(&neigh).unwrap();
+
+        // ip neigh show dev dummy ip
+        let stdout = Command::new("ip")
+            .args(&["neigh", "show", "dev", dummy_name, to_ip])
+            .output()
+            .expect("failed to show neigh")
+            .stdout;
+
+        let stdout = std::str::from_utf8(&stdout).expect("failed to conveert stdout");
+
+        assert_eq!(stdout, format!("{} lladdr {} PERMANENT\n", to_ip, mac));
+
+        clean_env_for_test_add_one_arp_neighbor(dummy_name, to_ip);
+    }
+}

src/agent/netlink/src/parser.rs

@@ -0,0 +1,201 @@
+// Copyright (c) 2019 Ant Financial
+//
+// SPDX-License-Identifier: Apache-2.0
+
+//! Parser for IPv4/IPv6/MAC addresses.
+
+use std::net::{Ipv4Addr, Ipv6Addr};
+use std::str::FromStr;
+
+use super::{Errno, Result, __u8, nix_errno};
+
+#[inline]
+pub(crate) fn parse_u8(s: &str, radix: u32) -> Result<u8> {
+    if radix >= 2 && radix <= 36 {
+        u8::from_str_radix(s, radix).map_err(|_| nix::Error::Sys(Errno::EINVAL))
+    } else {
+        u8::from_str(s).map_err(|_| nix::Error::Sys(Errno::EINVAL))
+    }
+}
+
+pub fn parse_ipv4_addr(s: &str) -> Result<Vec<u8>> {
+    match Ipv4Addr::from_str(s) {
+        Ok(v) => Ok(Vec::from(v.octets().as_ref())),
+        Err(_e) => nix_errno(Errno::EINVAL),
+    }
+}
+
+pub fn parse_ip_addr(s: &str) -> Result<Vec<u8>> {
+    if let Ok(v6) = Ipv6Addr::from_str(s) {
+        Ok(Vec::from(v6.octets().as_ref()))
+    } else {
+        parse_ipv4_addr(s)
+    }
+}
+
+pub fn parse_ip_addr_with_family(ip_address: &str) -> Result<(__u8, Vec<u8>)> {
+    if let Ok(v6) = Ipv6Addr::from_str(ip_address) {
+        Ok((libc::AF_INET6 as __u8, Vec::from(v6.octets().as_ref())))
+    } else {
+        parse_ipv4_addr(ip_address).map(|v| (libc::AF_INET as __u8, v))
+    }
+}
+
+pub fn parse_ipv4_cidr(s: &str) -> Result<(Vec<u8>, u8)> {
+    let fields: Vec<&str> = s.split('/').collect();
+
+    if fields.len() != 2 {
+        nix_errno(Errno::EINVAL)
+    } else {
+        Ok((parse_ipv4_addr(fields[0])?, parse_u8(fields[1], 10)?))
+    }
+}
+
+pub fn parse_cidr(s: &str) -> Result<(Vec<u8>, u8)> {
+    let fields: Vec<&str> = s.split('/').collect();
+
+    if fields.len() != 2 {
+        nix_errno(Errno::EINVAL)
+    } else {
+        Ok((parse_ip_addr(fields[0])?, parse_u8(fields[1], 10)?))
+    }
+}
+
+pub fn parse_mac_addr(hwaddr: &str) -> Result<Vec<u8>> {
+    let fields: Vec<&str> = hwaddr.split(':').collect();
+
+    if fields.len() != 6 {
+        nix_errno(Errno::EINVAL)
+    } else {
+        Ok(vec![
+            parse_u8(fields[0], 16)?,
+            parse_u8(fields[1], 16)?,
+            parse_u8(fields[2], 16)?,
+            parse_u8(fields[3], 16)?,
+            parse_u8(fields[4], 16)?,
+            parse_u8(fields[5], 16)?,
+        ])
+    }
+}
+
+/// Format an IPv4/IPv6/MAC address.
+///
+/// # Safety
+/// Caller needs to ensure that addr and len are valid.
+pub unsafe fn format_address(addr: *const u8, len: u32) -> Result<String> {
+    let mut a: String;
+    if len == 4 {
+        // ipv4
+        let mut i = 1;
+        let mut p = addr as i64;
+
+        a = format!("{}", *(p as *const u8));
+        while i < len {
+            p += 1;
+            i += 1;
+            a.push_str(format!(".{}", *(p as *const u8)).as_str());
+        }
+
+        return Ok(a);
+    }
+
+    if len == 6 {
+        // hwaddr
+        let mut i = 1;
+        let mut p = addr as i64;
+
+        a = format!("{:0>2X}", *(p as *const u8));
+        while i < len {
+            p += 1;
+            i += 1;
+            a.push_str(format!(":{:0>2X}", *(p as *const u8)).as_str());
+        }
+
+        return Ok(a);
+    }
+
+    if len == 16 {
+        // ipv6
+        let p = addr as *const u8 as *const libc::c_void;
+        let mut ar: [u8; 16] = [0; 16];
+        let mut v: Vec<u8> = vec![0; 16];
+        let dp: *mut libc::c_void = v.as_mut_ptr() as *mut libc::c_void;
+        libc::memcpy(dp, p, 16);
+
+        ar.copy_from_slice(v.as_slice());
+
+        return Ok(Ipv6Addr::from(ar).to_string());
+    }
+
+    nix_errno(Errno::EINVAL)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use libc;
+
+    #[test]
+    fn test_ip_addr() {
+        let ip = parse_ipv4_addr("1.2.3.4").unwrap();
+        assert_eq!(ip, vec![0x1u8, 0x2u8, 0x3u8, 0x4u8]);
+        parse_ipv4_addr("1.2.3.4.5").unwrap_err();
+        parse_ipv4_addr("1.2.3-4").unwrap_err();
+        parse_ipv4_addr("1.2.3.a").unwrap_err();
+        parse_ipv4_addr("1.2.3.x").unwrap_err();
+        parse_ipv4_addr("-1.2.3.4").unwrap_err();
+        parse_ipv4_addr("+1.2.3.4").unwrap_err();
+
+        let (family, _) = parse_ip_addr_with_family("192.168.1.1").unwrap();
+        assert_eq!(family, libc::AF_INET as __u8);
+
+        let (family, ip) =
+            parse_ip_addr_with_family("2001:0db8:85a3:0000:0000:8a2e:0370:7334").unwrap();
+        assert_eq!(family, libc::AF_INET6 as __u8);
+        assert_eq!(ip.len(), 16);
+        parse_ip_addr_with_family("2001:0db8:85a3:0000:0000:8a2e:0370:73345").unwrap_err();
+
+        let ip = parse_ip_addr("::1").unwrap();
+        assert_eq!(ip[0], 0x0);
+        assert_eq!(ip[15], 0x1);
+    }
+
+    #[test]
+    fn test_parse_cidr() {
+        let (_, mask) = parse_ipv4_cidr("1.2.3.4/31").unwrap();
+        assert_eq!(mask, 31);
+
+        parse_ipv4_cidr("1.2.3/4/31").unwrap_err();
+        parse_ipv4_cidr("1.2.3.4/f").unwrap_err();
+        parse_ipv4_cidr("1.2.3/8").unwrap_err();
+        parse_ipv4_cidr("1.2.3.4.8").unwrap_err();
+
+        let (ip, mask) = parse_cidr("2001:db8:a::123/64").unwrap();
+        assert_eq!(mask, 64);
+        assert_eq!(ip[0], 0x20);
+        assert_eq!(ip[15], 0x23);
+    }
+
+    #[test]
+    fn test_parse_mac_addr() {
+        let mac = parse_mac_addr("FF:FF:FF:FF:FF:FE").unwrap();
+        assert_eq!(mac.len(), 6);
+        assert_eq!(mac[0], 0xff);
+        assert_eq!(mac[5], 0xfe);
+
+        parse_mac_addr("FF:FF:FF:FF:FF:FE:A0").unwrap_err();
+        parse_mac_addr("FF:FF:FF:FF:FF:FX").unwrap_err();
+        parse_mac_addr("FF:FF:FF:FF:FF").unwrap_err();
+    }
+
+    #[test]
+    fn test_format_address() {
+        let buf = [1u8, 2u8, 3u8, 4u8];
+        let addr = unsafe { format_address(&buf as *const u8, 4).unwrap() };
+        assert_eq!(addr, "1.2.3.4");
+
+        let buf = [1u8, 2u8, 3u8, 4u8, 5u8, 6u8];
+        let addr = unsafe { format_address(&buf as *const u8, 6).unwrap() };
+        assert_eq!(addr, "01:02:03:04:05:06");
+    }
+}

src/agent/oci/src/lib.rs

@@ -8,7 +8,7 @@ extern crate serde;
 extern crate serde_derive;
 extern crate serde_json;
 
-use libc::{self, mode_t};
+use libc::mode_t;
 use std::collections::HashMap;
 
 mod serialize;
@@ -27,10 +27,6 @@ where
     *d == T::default()
 }
 
-fn default_seccomp_errno() -> u32 {
-    libc::EPERM as u32
-}
-
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
 pub struct Spec {
     #[serde(
@@ -58,7 +54,7 @@ pub struct Spec {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub windows: Option<Windows<String>>,
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub vm: Option<Vm>,
+    pub vm: Option<VM>,
 }
 
 impl Spec {
@@ -71,7 +67,7 @@ impl Spec {
     }
 }
 
-pub type LinuxRlimit = PosixRlimit;
+pub type LinuxRlimit = POSIXRlimit;
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
 pub struct Process {
@@ -93,7 +89,7 @@ pub struct Process {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub capabilities: Option<LinuxCapabilities>,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
-    pub rlimits: Vec<PosixRlimit>,
+    pub rlimits: Vec<POSIXRlimit>,
     #[serde(default, rename = "noNewPrivileges")]
     pub no_new_privileges: bool,
     #[serde(
@@ -146,7 +142,7 @@ pub struct User {
     pub gid: u32,
     #[serde(
         default,
-        rename = "additionalGids",
+        rename = "addtionalGids",
         skip_serializing_if = "Vec::is_empty"
     )]
     pub additional_gids: Vec<u32>,
@@ -199,9 +195,9 @@ pub struct Hooks {
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
 pub struct Linux {
     #[serde(default, rename = "uidMappings", skip_serializing_if = "Vec::is_empty")]
-    pub uid_mappings: Vec<LinuxIdMapping>,
+    pub uid_mappings: Vec<LinuxIDMapping>,
     #[serde(default, rename = "gidMappings", skip_serializing_if = "Vec::is_empty")]
-    pub gid_mappings: Vec<LinuxIdMapping>,
+    pub gid_mappings: Vec<LinuxIDMapping>,
     #[serde(default, skip_serializing_if = "HashMap::is_empty")]
     pub sysctl: HashMap<String, String>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -261,7 +257,7 @@ pub const UTSNAMESPACE: &str = "uts";
 pub const CGROUPNAMESPACE: &str = "cgroup";
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
-pub struct LinuxIdMapping {
+pub struct LinuxIDMapping {
     #[serde(default, rename = "containerID")]
     pub container_id: u32,
     #[serde(default, rename = "hostID")]
@@ -271,7 +267,7 @@ pub struct LinuxIdMapping {
 }
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
-pub struct PosixRlimit {
+pub struct POSIXRlimit {
     #[serde(default)]
     pub r#type: String,
     #[serde(default)]
@@ -297,7 +293,7 @@ pub struct LinuxInterfacePriority {
 }
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
-pub struct LinuxBlockIoDevice {
+pub struct LinuxBlockIODevice {
     #[serde(default)]
     pub major: i64,
     #[serde(default)]
@@ -306,8 +302,7 @@ pub struct LinuxBlockIoDevice {
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
 pub struct LinuxWeightDevice {
-    #[serde(flatten)]
-    pub blk: LinuxBlockIoDevice,
+    pub blk: LinuxBlockIODevice,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub weight: Option<u16>,
     #[serde(
@@ -320,14 +315,13 @@ pub struct LinuxWeightDevice {
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
 pub struct LinuxThrottleDevice {
-    #[serde(flatten)]
-    pub blk: LinuxBlockIoDevice,
+    pub blk: LinuxBlockIODevice,
     #[serde(default)]
     pub rate: u64,
 }
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
-pub struct LinuxBlockIo {
+pub struct LinuxBlockIO {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub weight: Option<u16>,
     #[serde(
@@ -381,7 +375,7 @@ pub struct LinuxMemory {
     #[serde(default, skip_serializing_if = "Option::is_none", rename = "kernelTCP")]
     pub kernel_tcp: Option<i64>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub swappiness: Option<i64>,
+    pub swapiness: Option<i64>,
     #[serde(
         default,
         skip_serializing_if = "Option::is_none",
@@ -391,7 +385,7 @@ pub struct LinuxMemory {
 }
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
-pub struct LinuxCpu {
+pub struct LinuxCPU {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub shares: Option<u64>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -453,11 +447,11 @@ pub struct LinuxResources {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub memory: Option<LinuxMemory>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub cpu: Option<LinuxCpu>,
+    pub cpu: Option<LinuxCPU>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub pids: Option<LinuxPids>,
     #[serde(skip_serializing_if = "Option::is_none", rename = "blockIO")]
-    pub block_io: Option<LinuxBlockIo>,
+    pub block_io: Option<LinuxBlockIO>,
     #[serde(
         default,
         skip_serializing_if = "Vec::is_empty",
@@ -517,7 +511,7 @@ pub struct Solaris {
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub anet: Vec<SolarisAnet>,
     #[serde(default, skip_serializing_if = "Option::is_none", rename = "cappedCPU")]
-    pub capped_cpu: Option<SolarisCappedCpu>,
+    pub capped_cpu: Option<SolarisCappedCPU>,
     #[serde(
         default,
         skip_serializing_if = "Option::is_none",
@@ -527,7 +521,7 @@ pub struct Solaris {
 }
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
-pub struct SolarisCappedCpu {
+pub struct SolarisCappedCPU {
     #[serde(default, skip_serializing_if = "String::is_empty")]
     pub ncpus: String,
 }
@@ -605,7 +599,7 @@ pub struct WindowsResources {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub memory: Option<WindowsMemoryResources>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub cpu: Option<WindowsCpuResources>,
+    pub cpu: Option<WindowsCPUResources>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub storage: Option<WindowsStorageResources>,
 }
@@ -617,7 +611,7 @@ pub struct WindowsMemoryResources {
 }
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
-pub struct WindowsCpuResources {
+pub struct WindowsCPUResources {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub count: Option<u64>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -675,14 +669,14 @@ pub struct WindowsHyperV {
 }
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
-pub struct Vm {
-    pub hypervisor: VmHypervisor,
-    pub kernel: VmKernel,
-    pub image: VmImage,
+pub struct VM {
+    pub hypervisor: VMHypervisor,
+    pub kernel: VMKernel,
+    pub image: VMImage,
 }
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
-pub struct VmHypervisor {
+pub struct VMHypervisor {
     #[serde(default)]
     pub path: String,
     #[serde(default, skip_serializing_if = "String::is_empty")]
@@ -690,7 +684,7 @@ pub struct VmHypervisor {
 }
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
-pub struct VmKernel {
+pub struct VMKernel {
     #[serde(default)]
     pub path: String,
     #[serde(default, skip_serializing_if = "String::is_empty")]
@@ -700,7 +694,7 @@ pub struct VmKernel {
 }
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
-pub struct VmImage {
+pub struct VMImage {
     #[serde(default)]
     pub path: String,
     #[serde(default)]
@@ -714,8 +708,6 @@ pub struct LinuxSeccomp {
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub architectures: Vec<Arch>,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
-    pub flags: Vec<LinuxSeccompFlag>,
-    #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub syscalls: Vec<LinuxSyscall>,
 }
 
@@ -739,20 +731,14 @@ pub const ARCHS390: &str = "SCMP_ARCH_S390";
 pub const ARCHS390X: &str = "SCMP_ARCH_S390X";
 pub const ARCHPARISC: &str = "SCMP_ARCH_PARISC";
 pub const ARCHPARISC64: &str = "SCMP_ARCH_PARISC64";
-pub const ARCHRISCV64: &str = "SCMP_ARCH_RISCV64";
-
-pub type LinuxSeccompFlag = String;
 
 pub type LinuxSeccompAction = String;
 
 pub const ACTKILL: &str = "SCMP_ACT_KILL";
-pub const ACTKILLPROCESS: &str = "SCMP_ACT_KILL_PROCESS";
-pub const ACTKILLTHREAD: &str = "SCMP_ACT_KILL_THREAD";
 pub const ACTTRAP: &str = "SCMP_ACT_TRAP";
 pub const ACTERRNO: &str = "SCMP_ACT_ERRNO";
 pub const ACTTRACE: &str = "SCMP_ACT_TRACE";
 pub const ACTALLOW: &str = "SCMP_ACT_ALLOW";
-pub const ACTLOG: &str = "SCMP_ACT_LOG";
 
 pub type LinuxSeccompOperator = String;
 
@@ -782,8 +768,6 @@ pub struct LinuxSyscall {
     pub names: Vec<String>,
     #[serde(default, skip_serializing_if = "String::is_empty")]
     pub action: LinuxSeccompAction,
-    #[serde(default = "default_seccomp_errno", rename = "errnoRet")]
-    pub errno_ret: u32,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub args: Vec<LinuxSeccompArg>,
 }
@@ -798,17 +782,7 @@ pub struct LinuxIntelRdt {
     pub l3_cache_schema: String,
 }
 
-#[derive(Debug, Serialize, Deserialize, Copy, Clone, PartialEq)]
-#[serde(rename_all = "lowercase")]
-pub enum ContainerState {
-    Creating,
-    Created,
-    Running,
-    Stopped,
-    Paused,
-}
-
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
+#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
 pub struct State {
     #[serde(
         default,
@@ -818,7 +792,8 @@ pub struct State {
     pub version: String,
     #[serde(default, skip_serializing_if = "String::is_empty")]
     pub id: String,
-    pub status: ContainerState,
+    #[serde(default, skip_serializing_if = "String::is_empty")]
+    pub status: String,
     #[serde(default)]
     pub pid: i32,
     #[serde(default, skip_serializing_if = "String::is_empty")]
@@ -829,8 +804,6 @@ pub struct State {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
-
     #[test]
     fn test_deserialize_state() {
         let data = r#"{
@@ -843,10 +816,10 @@ mod tests {
                 "myKey": "myValue"
             }
         }"#;
-        let expected = State {
+        let expected = crate::State {
             version: "0.2.0".to_string(),
             id: "oci-container1".to_string(),
-            status: ContainerState::Running,
+            status: "running".to_string(),
             pid: 4422,
             bundle: "/containers/redis".to_string(),
             annotations: [("myKey".to_string(), "myValue".to_string())]
@@ -860,7 +833,7 @@ mod tests {
     }
 
     #[test]
-    fn test_deserialize_spec() {
+    fn test_deserialize_sepc() {
         let data = r#"{
             "ociVersion": "1.0.1",
             "process": {
@@ -1145,28 +1118,36 @@ mod tests {
                         "leafWeight": 10,
                         "weightDevice": [
                             {
-                                "major": 8,
-                                "minor": 0,
+                                "blk": {
+                                    "major": 8,
+                                    "minor": 0
+                                },
                                 "weight": 500,
                                 "leafWeight": 300
                             },
                             {
-                                "major": 8,
-                                "minor": 16,
+                                "blk":{
+                                    "major": 8,
+                                    "minor": 16
+                                },
                                 "weight": 500
                             }
                         ],
                         "throttleReadBpsDevice": [
                             {
-                                "major": 8,
-                                "minor": 0,
+                                "blk":{
+                                    "major": 8,
+                                    "minor": 0
+                                },
                                 "rate": 600
                             }
                         ],
                         "throttleWriteIOPSDevice": [
                             {
-                                "major": 8,
-                                "minor": 16,
+                                "blk":{
+                                    "major": 8,
+                                    "minor": 16
+                                },
                                 "rate": 300
                             }
                         ]
@@ -1242,7 +1223,8 @@ mod tests {
                     uid: 1,
                     gid: 1,
                     // incompatible with oci
-                    additional_gids: vec![5, 6],
+                    // additional_gids: vec![5, 6],
+                    additional_gids: vec![],
                     username: "".to_string(),
                 },
                 args: vec!["sh".to_string()],
@@ -1271,12 +1253,12 @@ mod tests {
                     ambient: vec!["CAP_NET_BIND_SERVICE".to_string()],
                 }),
                 rlimits: vec![
-                    crate::PosixRlimit {
+                    crate::POSIXRlimit {
                         r#type: "RLIMIT_CORE".to_string(),
                         hard: 1024,
                         soft: 1024,
                     },
-                    crate::PosixRlimit {
+                    crate::POSIXRlimit {
                         r#type: "RLIMIT_NOFILE".to_string(),
                         hard: 1024,
                         soft: 1024,
@@ -1408,12 +1390,12 @@ mod tests {
             .cloned()
             .collect(),
             linux: Some(crate::Linux {
-                uid_mappings: vec![crate::LinuxIdMapping {
+                uid_mappings: vec![crate::LinuxIDMapping {
                     container_id: 0,
                     host_id: 1000,
                     size: 32000,
                 }],
-                gid_mappings: vec![crate::LinuxIdMapping {
+                gid_mappings: vec![crate::LinuxIDMapping {
                     container_id: 0,
                     host_id: 1000,
                     size: 32000,
@@ -1455,10 +1437,11 @@ mod tests {
                         swap: Some(536870912),
                         kernel: Some(-1),
                         kernel_tcp: Some(-1),
-                        swappiness: Some(0),
+                        // incompatible with oci
+                        swapiness: None,
                         disable_oom_killer: Some(false),
                     }),
-                    cpu: Some(crate::LinuxCpu {
+                    cpu: Some(crate::LinuxCPU {
                         shares: Some(1024),
                         quota: Some(1000000),
                         period: Some(500000),
@@ -1468,17 +1451,17 @@ mod tests {
                         mems: "0-7".to_string(),
                     }),
                     pids: Some(crate::LinuxPids { limit: 32771 }),
-                    block_io: Some(crate::LinuxBlockIo {
+                    block_io: Some(crate::LinuxBlockIO {
                         weight: Some(10),
                         leaf_weight: Some(10),
                         weight_device: vec![
                             crate::LinuxWeightDevice {
-                                blk: crate::LinuxBlockIoDevice { major: 8, minor: 0 },
+                                blk: crate::LinuxBlockIODevice { major: 8, minor: 0 },
                                 weight: Some(500),
                                 leaf_weight: Some(300),
                             },
                             crate::LinuxWeightDevice {
-                                blk: crate::LinuxBlockIoDevice {
+                                blk: crate::LinuxBlockIODevice {
                                     major: 8,
                                     minor: 16,
                                 },
@@ -1487,13 +1470,13 @@ mod tests {
                             },
                         ],
                         throttle_read_bps_device: vec![crate::LinuxThrottleDevice {
-                            blk: crate::LinuxBlockIoDevice { major: 8, minor: 0 },
+                            blk: crate::LinuxBlockIODevice { major: 8, minor: 0 },
                             rate: 600,
                         }],
                         throttle_write_bps_device: vec![],
                         throttle_read_iops_device: vec![],
                         throttle_write_iops_device: vec![crate::LinuxThrottleDevice {
-                            blk: crate::LinuxBlockIoDevice {
+                            blk: crate::LinuxBlockIODevice {
                                 major: 8,
                                 minor: 16,
                             },
@@ -1579,11 +1562,9 @@ mod tests {
                 seccomp: Some(crate::LinuxSeccomp {
                     default_action: "SCMP_ACT_ALLOW".to_string(),
                     architectures: vec!["SCMP_ARCH_X86".to_string(), "SCMP_ARCH_X32".to_string()],
-                    flags: vec![],
                     syscalls: vec![crate::LinuxSyscall {
                         names: vec!["getcwd".to_string(), "chmod".to_string()],
                         action: "SCMP_ACT_ERRNO".to_string(),
-                        errno_ret: crate::default_seccomp_errno(),
                         args: vec![],
                     }],
                 }),
@@ -1610,6 +1591,25 @@ mod tests {
             vm: None,
         };
 
+        // warning : incompatible with oci : https://github.com/opencontainers/runtime-spec/blob/master/config.md
+        // 1. User use addtionalGids while oci use additionalGids
+        // 2. LinuxMemory use swapiness while oci use swappiness
+        // 3. LinuxWeightDevice with blk
+        //    {
+        //        "blk": {
+        //            "major": 8,
+        //            "minor": 0
+        //        },
+        //        "weight": 500,
+        //        "leafWeight": 300
+        //    }
+        //    oci without blk
+        //    {
+        //        "major": 8,
+        //        "minor": 0,
+        //        "weight": 500,
+        //        "leafWeight": 300
+        //    }
         let current: crate::Spec = serde_json::from_str(data).unwrap();
         assert_eq!(expected, current);
     }

src/agent/oci/src/serialize.rs

@@ -4,6 +4,7 @@
 //
 
 use serde::{Deserialize, Serialize};
+use serde_json;
 
 use std::error;
 use std::fmt::{Display, Formatter, Result as FmtResult};

src/agent/protocols/Cargo.toml

@@ -5,9 +5,6 @@ authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
 edition = "2018"
 
 [dependencies]
-ttrpc = { version = "0.5.0", features = ["async"] }
-async-trait = "0.1.42"
+ttrpc = { git = "https://github.com/containerd/ttrpc-rust.git", branch="0.3.0" }
 protobuf = "=2.14.0"
-
-[build-dependencies]
-ttrpc-codegen = "0.2.0"
+futures = "0.1.27"

src/agent/protocols/build.rs

@@ -1,44 +0,0 @@
-// Copyright (c) 2020 Ant Group
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-use std::fs;
-use ttrpc_codegen::{Codegen, Customize};
-
-fn main() {
-    let protos = vec![
-        "protos/types.proto",
-        "protos/agent.proto",
-        "protos/health.proto",
-        "protos/google/protobuf/empty.proto",
-        "protos/oci.proto",
-    ];
-
-    Codegen::new()
-        .out_dir("src")
-        .inputs(&protos)
-        .include("protos")
-        .rust_protobuf()
-        .customize(Customize {
-            async_server: true,
-            ..Default::default()
-        })
-        .run()
-        .expect("Gen codes failed.");
-
-    // There is a message named 'Box' in oci.proto
-    // so there is a struct named 'Box', we should replace Box<Self> to ::std::boxed::Box<Self>
-    // to avoid the conflict.
-    replace_text_in_file(
-        "src/oci.rs",
-        "self: Box<Self>",
-        "self: ::std::boxed::Box<Self>",
-    )
-    .unwrap();
-}
-
-fn replace_text_in_file(file_name: &str, from: &str, to: &str) -> Result<(), std::io::Error> {
-    let new_contents = fs::read_to_string(file_name)?.replace(from, to);
-    fs::write(&file_name, new_contents.as_bytes())
-}

src/agent/protocols/hack/update-generated-proto.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # //
-# // Copyright (c) 2020 Ant Group
+# // Copyright 2020 Ant Financial
 # //
 # // SPDX-License-Identifier: Apache-2.0
 # //
@@ -47,11 +47,11 @@ show_usage() {
 }
 
 generate_go_sources() {
-    local cmd="protoc -I$GOPATH/src:$GOPATH/src/github.com/kata-containers/kata-containers/src/agent/protocols/protos \
+    local cmd="protoc -I$GOPATH/src/github.com/kata-containers/agent/vendor/github.com/gogo/protobuf:$GOPATH/src/github.com/kata-containers/agent/vendor:$GOPATH/src/github.com/gogo/protobuf:$GOPATH/src/github.com/gogo/googleapis:$GOPATH/src:$GOPATH/src/github.com/kata-containers/kata-containers/src/agent/protocols/protos \
 --gogottrpc_out=plugins=ttrpc+fieldpath,\
 import_path=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc,\
 \
-Mgithub.com/kata-containers/kata-containers/src/agent/protocols/protos/types.proto=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols,\
+Mgithub.com/kata-containers/kata-containers/src/agent/protocols/protos/github.com/kata-containers/agent/pkg/types/types.proto=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols,\
 \
 Mgithub.com/kata-containers/kata-containers/src/agent/protocols/protos/oci.proto=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc,\
 \
@@ -64,12 +64,31 @@ $GOPATH/src/github.com/kata-containers/kata-containers/src/agent/protocols/proto
     [ $? -eq 0 ] || die "Failed to generate golang file from $1"
 }
 
+generate_rust_sources() {
+    local cmd="protoc --rust_out=./protocols/src/ \
+--ttrpc_out=./protocols/src/,plugins=ttrpc:./protocols/src/ \
+--plugin=protoc-gen-ttrpc=`which ttrpc_rust_plugin` \
+-I $GOPATH/src/github.com/kata-containers/agent/vendor/github.com/gogo/protobuf:$GOPATH/src/github.com/kata-containers/agent/vendor:$GOPATH/src/github.com/gogo/protobuf:$GOPATH/src/github.com/gogo/googleapis:$GOPATH/src:$GOPATH/src/github.com/kata-containers/kata-containers/src/agent/protocols/protos \
+$GOPATH/src/github.com/kata-containers/kata-containers/src/agent/protocols/protos/$1"
+
+    echo $cmd
+    $cmd
+    [ $? -eq 0 ] || die "Failed to generate rust file from $1"
+
+    if [ "$1" = "oci.proto" ]; then
+        # Need change Box<Self> to ::std::boxed::Box<Self> because there is another struct Box
+        sed 's/fn into_any(self: Box<Self>) -> ::std::boxed::Box<::std::any::Any> {/fn into_any(self: ::std::boxed::Box<Self>) -> ::std::boxed::Box<::std::any::Any> {/g' ./protocols/src/oci.rs > ./protocols/src/new_oci.rs
+        sed 's/fn into_any(self: Box<Self>) -> ::std::boxed::Box<dyn (::std::any::Any)> {/fn into_any(self: ::std::boxed::Box<Self>) -> ::std::boxed::Box<dyn (::std::any::Any)> {/g' ./protocols/src/oci.rs > ./protocols/src/new_oci.rs
+        mv ./protocols/src/new_oci.rs ./protocols/src/oci.rs
+    fi;
+}
+
 if [ "$(basename $(pwd))" != "agent" ]; then
-	die "Please go to root directory of agent before execute this shell"
+	die "Please go to directory of protocols before execute this shell"
 fi
 
 # Protocol buffer files required to generate golang/rust bindings.
-proto_files_list=(agent.proto health.proto oci.proto types.proto)
+proto_files_list=(agent.proto health.proto oci.proto github.com/kata-containers/agent/pkg/types/types.proto)
 
 if [ "$1" = "" ]; then
     show_usage "${proto_files_list[@]}"
@@ -80,6 +99,12 @@ fi;
 which protoc
 [ $? -eq 0 ] || die "Please install protoc from github.com/protocolbuffers/protobuf"
 
+which protoc-gen-rust
+[ $? -eq 0 ] || die "Please install protobuf-codegen from github.com/pingcap/grpc-rs"
+
+which ttrpc_rust_plugin
+[ $? -eq 0 ] || die "Please install ttrpc_rust_plugin from https://github.com/containerd/ttrpc-rust"
+
 which protoc-gen-gogottrpc
 [ $? -eq 0 ] || die "Please install protoc-gen-gogottrpc from https://github.com/containerd/ttrpc"
 
@@ -93,6 +118,10 @@ if [ "$target" = "all" ]; then
         echo -e "\n   [golang] compiling ${f} ..."
         generate_go_sources $f
         echo -e "   [golang] ${f} compiled\n"
+
+        echo -e "\n   [rust] compiling ${f} ..."
+        generate_rust_sources $f
+        echo -e "   [rust] ${f} compiled\n"
     done
 else
     # compile individual proto file
@@ -101,6 +130,10 @@ else
             echo -e "\n   [golang] compiling ${target} ..."
             generate_go_sources $target
             echo -e "   [golang] ${target} compiled\n"
+
+            echo -e "\n   [rust] compiling ${target} ..."
+            generate_rust_sources $target
+            echo -e "   [rust] ${target} compiled\n"
         fi
     done
 fi;

src/agent/protocols/protos/agent.proto

@@ -1,6 +1,6 @@
 //
 // Copyright 2017 HyperHQ Inc.
-// Copyright (c) 2019-2020 Ant Group
+// Copyright 2019 Ant Financial
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -11,8 +11,8 @@ option go_package = "github.com/kata-containers/kata-containers/src/runtime/virt
 
 package grpc;
 
-import "oci.proto";
-import "types.proto";
+import "github.com/kata-containers/kata-containers/src/agent/protocols/protos/oci.proto";
+import "github.com/kata-containers/kata-containers/src/agent/protocols/protos/github.com/kata-containers/agent/pkg/types/types.proto";
 
 import "google/protobuf/empty.proto";
 
@@ -32,6 +32,7 @@ service AgentService {
 	rpc ExecProcess(ExecProcessRequest) returns (google.protobuf.Empty);
 	rpc SignalProcess(SignalProcessRequest) returns (google.protobuf.Empty);
 	rpc WaitProcess(WaitProcessRequest) returns (WaitProcessResponse); // wait & reap like waitpid(2)
+	rpc ListProcesses(ListProcessesRequest) returns (ListProcessesResponse);
 	rpc UpdateContainer(UpdateContainerRequest) returns (google.protobuf.Empty);
 	rpc StatsContainer(StatsContainerRequest) returns (StatsContainerResponse);
 	rpc PauseContainer(PauseContainerRequest) returns (google.protobuf.Empty);
@@ -125,6 +126,18 @@ message WaitProcessResponse {
 	int32 status = 1;
 }
 
+// ListProcessesRequest contains the options used to list running processes inside the container
+message ListProcessesRequest {
+	string container_id = 1;
+	string format = 2;
+	repeated string args = 3;
+}
+
+// ListProcessesResponse represents the list of running processes inside the container
+message ListProcessesResponse {
+	bytes process_list = 1;
+}
+
 message UpdateContainerRequest {
 	string container_id = 1;
 	LinuxResources resources = 2;

src/agent/protocols/protos/github.com/kata-containers/agent/pkg/types/types.proto

@@ -1,6 +1,6 @@
 //
 // Copyright 2018 Intel Corporation.
-// Copyright (c) 2019-2020 Ant Group
+// Copyright (c) 2019 Ant Financial
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -29,8 +29,10 @@ message Interface {
 	uint64 mtu = 4;
 	string hwAddr = 5;
 
-	// PCI path for the device (see the pci::Path (Rust) or types.PciPath (Go) type for format details)
-	string pciPath = 6;
+	// pciAddr is the PCI address in the format  "bridgeAddr/deviceAddr".
+	// Here, bridgeAddr is the address at which the bridge is attached on the root bus,
+	// while deviceAddr is the address at which the network device is attached on the bridge.
+	string pciAddr = 6;
 
 	// Type defines the type of interface described by this structure.
 	// The expected values are the one that are defined by the netlink

src/agent/protocols/protos/health.proto

@@ -1,6 +1,6 @@
 //
-// Copyright (c) 2017 HyperHQ Inc.
-// Copyright (c) 2019-2020 Ant Group
+// Copyright 2017 HyperHQ Inc.
+// Copyright (c) 2019 Ant Financial
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -11,7 +11,7 @@ option go_package = "github.com/kata-containers/kata-containers/src/runtime/virt
 
 package grpc;
 
-import "gogo/protobuf/gogoproto/gogo.proto";
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
 
 option (gogoproto.equal_all) = true;
 option (gogoproto.populate_all) = true;

src/agent/protocols/protos/oci.proto

@@ -1,6 +1,6 @@
 //
 // Copyright (c) 2017 Intel Corporation
-// Copyright (c) 2019-2020 Ant Group
+// Copyright (c) 2019 Ant Financial
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -11,7 +11,8 @@ option go_package = "github.com/kata-containers/kata-containers/src/runtime/virt
 
 package grpc;
 
-import "gogo/protobuf/gogoproto/gogo.proto";
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+import "google/protobuf/wrappers.proto";
 
 option (gogoproto.equal_all) = true;
 option (gogoproto.populate_all) = true;
@@ -441,8 +442,7 @@ message LinuxInterfacePriority {
 message LinuxSeccomp {
 	string DefaultAction = 1;
 	repeated string Architectures = 2;
-	repeated string Flags = 3;
-	repeated LinuxSyscall Syscalls = 4  [(gogoproto.nullable) = false];
+	repeated LinuxSyscall Syscalls = 3  [(gogoproto.nullable) = false];
 }
 
 message LinuxSeccompArg {
@@ -455,10 +455,7 @@ message LinuxSeccompArg {
 message LinuxSyscall {
 	repeated string Names = 1;
 	string Action = 2;
-	oneof ErrnoRet {
-		uint32 errnoret = 3;
-	}
-	repeated LinuxSeccompArg Args = 4  [(gogoproto.nullable) = false];
+	repeated LinuxSeccompArg Args = 3  [(gogoproto.nullable) = false];
 }
 
 message LinuxIntelRdt {

src/agent/protocols/src/agent_ttrpc.rs

@@ -0,0 +1,808 @@
+// Copyright (c) 2020 Ant Financial
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+// This file is generated by ttrpc-compiler 0.3.0. Do not edit
+// @generated
+
+// https://github.com/Manishearth/rust-clippy/issues/702
+#![allow(unknown_lints)]
+#![allow(clipto_camel_casepy)]
+
+#![cfg_attr(rustfmt, rustfmt_skip)]
+
+#![allow(box_pointers)]
+#![allow(dead_code)]
+#![allow(missing_docs)]
+#![allow(non_camel_case_types)]
+#![allow(non_snake_case)]
+#![allow(non_upper_case_globals)]
+#![allow(trivial_casts)]
+#![allow(unsafe_code)]
+#![allow(unused_imports)]
+#![allow(unused_results)]
+use protobuf::{CodedInputStream, CodedOutputStream, Message};
+use std::collections::HashMap;
+use std::sync::Arc;
+
+#[derive(Clone)]
+pub struct AgentServiceClient {
+    client: ::ttrpc::Client,
+}
+
+impl AgentServiceClient {
+    pub fn new(client: ::ttrpc::Client) -> Self {
+        AgentServiceClient {
+            client: client,
+        }
+    }
+
+    pub fn create_container(&self, req: &super::agent::CreateContainerRequest, timeout_nano: i64) -> ::ttrpc::Result<super::empty::Empty> {
+        let mut cres = super::empty::Empty::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "CreateContainer", cres);
+        Ok(cres)
+    }
+
+    pub fn start_container(&self, req: &super::agent::StartContainerRequest, timeout_nano: i64) -> ::ttrpc::Result<super::empty::Empty> {
+        let mut cres = super::empty::Empty::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "StartContainer", cres);
+        Ok(cres)
+    }
+
+    pub fn remove_container(&self, req: &super::agent::RemoveContainerRequest, timeout_nano: i64) -> ::ttrpc::Result<super::empty::Empty> {
+        let mut cres = super::empty::Empty::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "RemoveContainer", cres);
+        Ok(cres)
+    }
+
+    pub fn exec_process(&self, req: &super::agent::ExecProcessRequest, timeout_nano: i64) -> ::ttrpc::Result<super::empty::Empty> {
+        let mut cres = super::empty::Empty::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "ExecProcess", cres);
+        Ok(cres)
+    }
+
+    pub fn signal_process(&self, req: &super::agent::SignalProcessRequest, timeout_nano: i64) -> ::ttrpc::Result<super::empty::Empty> {
+        let mut cres = super::empty::Empty::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "SignalProcess", cres);
+        Ok(cres)
+    }
+
+    pub fn wait_process(&self, req: &super::agent::WaitProcessRequest, timeout_nano: i64) -> ::ttrpc::Result<super::agent::WaitProcessResponse> {
+        let mut cres = super::agent::WaitProcessResponse::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "WaitProcess", cres);
+        Ok(cres)
+    }
+
+    pub fn list_processes(&self, req: &super::agent::ListProcessesRequest, timeout_nano: i64) -> ::ttrpc::Result<super::agent::ListProcessesResponse> {
+        let mut cres = super::agent::ListProcessesResponse::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "ListProcesses", cres);
+        Ok(cres)
+    }
+
+    pub fn update_container(&self, req: &super::agent::UpdateContainerRequest, timeout_nano: i64) -> ::ttrpc::Result<super::empty::Empty> {
+        let mut cres = super::empty::Empty::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "UpdateContainer", cres);
+        Ok(cres)
+    }
+
+    pub fn stats_container(&self, req: &super::agent::StatsContainerRequest, timeout_nano: i64) -> ::ttrpc::Result<super::agent::StatsContainerResponse> {
+        let mut cres = super::agent::StatsContainerResponse::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "StatsContainer", cres);
+        Ok(cres)
+    }
+
+    pub fn pause_container(&self, req: &super::agent::PauseContainerRequest, timeout_nano: i64) -> ::ttrpc::Result<super::empty::Empty> {
+        let mut cres = super::empty::Empty::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "PauseContainer", cres);
+        Ok(cres)
+    }
+
+    pub fn resume_container(&self, req: &super::agent::ResumeContainerRequest, timeout_nano: i64) -> ::ttrpc::Result<super::empty::Empty> {
+        let mut cres = super::empty::Empty::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "ResumeContainer", cres);
+        Ok(cres)
+    }
+
+    pub fn write_stdin(&self, req: &super::agent::WriteStreamRequest, timeout_nano: i64) -> ::ttrpc::Result<super::agent::WriteStreamResponse> {
+        let mut cres = super::agent::WriteStreamResponse::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "WriteStdin", cres);
+        Ok(cres)
+    }
+
+    pub fn read_stdout(&self, req: &super::agent::ReadStreamRequest, timeout_nano: i64) -> ::ttrpc::Result<super::agent::ReadStreamResponse> {
+        let mut cres = super::agent::ReadStreamResponse::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "ReadStdout", cres);
+        Ok(cres)
+    }
+
+    pub fn read_stderr(&self, req: &super::agent::ReadStreamRequest, timeout_nano: i64) -> ::ttrpc::Result<super::agent::ReadStreamResponse> {
+        let mut cres = super::agent::ReadStreamResponse::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "ReadStderr", cres);
+        Ok(cres)
+    }
+
+    pub fn close_stdin(&self, req: &super::agent::CloseStdinRequest, timeout_nano: i64) -> ::ttrpc::Result<super::empty::Empty> {
+        let mut cres = super::empty::Empty::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "CloseStdin", cres);
+        Ok(cres)
+    }
+
+    pub fn tty_win_resize(&self, req: &super::agent::TtyWinResizeRequest, timeout_nano: i64) -> ::ttrpc::Result<super::empty::Empty> {
+        let mut cres = super::empty::Empty::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "TtyWinResize", cres);
+        Ok(cres)
+    }
+
+    pub fn update_interface(&self, req: &super::agent::UpdateInterfaceRequest, timeout_nano: i64) -> ::ttrpc::Result<super::types::Interface> {
+        let mut cres = super::types::Interface::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "UpdateInterface", cres);
+        Ok(cres)
+    }
+
+    pub fn update_routes(&self, req: &super::agent::UpdateRoutesRequest, timeout_nano: i64) -> ::ttrpc::Result<super::agent::Routes> {
+        let mut cres = super::agent::Routes::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "UpdateRoutes", cres);
+        Ok(cres)
+    }
+
+    pub fn list_interfaces(&self, req: &super::agent::ListInterfacesRequest, timeout_nano: i64) -> ::ttrpc::Result<super::agent::Interfaces> {
+        let mut cres = super::agent::Interfaces::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "ListInterfaces", cres);
+        Ok(cres)
+    }
+
+    pub fn list_routes(&self, req: &super::agent::ListRoutesRequest, timeout_nano: i64) -> ::ttrpc::Result<super::agent::Routes> {
+        let mut cres = super::agent::Routes::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "ListRoutes", cres);
+        Ok(cres)
+    }
+
+    pub fn add_arp_neighbors(&self, req: &super::agent::AddARPNeighborsRequest, timeout_nano: i64) -> ::ttrpc::Result<super::empty::Empty> {
+        let mut cres = super::empty::Empty::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "AddARPNeighbors", cres);
+        Ok(cres)
+    }
+
+    pub fn start_tracing(&self, req: &super::agent::StartTracingRequest, timeout_nano: i64) -> ::ttrpc::Result<super::empty::Empty> {
+        let mut cres = super::empty::Empty::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "StartTracing", cres);
+        Ok(cres)
+    }
+
+    pub fn stop_tracing(&self, req: &super::agent::StopTracingRequest, timeout_nano: i64) -> ::ttrpc::Result<super::empty::Empty> {
+        let mut cres = super::empty::Empty::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "StopTracing", cres);
+        Ok(cres)
+    }
+
+    pub fn get_metrics(&self, req: &super::agent::GetMetricsRequest, timeout_nano: i64) -> ::ttrpc::Result<super::agent::Metrics> {
+        let mut cres = super::agent::Metrics::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "GetMetrics", cres);
+        Ok(cres)
+    }
+
+    pub fn create_sandbox(&self, req: &super::agent::CreateSandboxRequest, timeout_nano: i64) -> ::ttrpc::Result<super::empty::Empty> {
+        let mut cres = super::empty::Empty::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "CreateSandbox", cres);
+        Ok(cres)
+    }
+
+    pub fn destroy_sandbox(&self, req: &super::agent::DestroySandboxRequest, timeout_nano: i64) -> ::ttrpc::Result<super::empty::Empty> {
+        let mut cres = super::empty::Empty::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "DestroySandbox", cres);
+        Ok(cres)
+    }
+
+    pub fn online_cpu_mem(&self, req: &super::agent::OnlineCPUMemRequest, timeout_nano: i64) -> ::ttrpc::Result<super::empty::Empty> {
+        let mut cres = super::empty::Empty::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "OnlineCPUMem", cres);
+        Ok(cres)
+    }
+
+    pub fn reseed_random_dev(&self, req: &super::agent::ReseedRandomDevRequest, timeout_nano: i64) -> ::ttrpc::Result<super::empty::Empty> {
+        let mut cres = super::empty::Empty::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "ReseedRandomDev", cres);
+        Ok(cres)
+    }
+
+    pub fn get_guest_details(&self, req: &super::agent::GuestDetailsRequest, timeout_nano: i64) -> ::ttrpc::Result<super::agent::GuestDetailsResponse> {
+        let mut cres = super::agent::GuestDetailsResponse::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "GetGuestDetails", cres);
+        Ok(cres)
+    }
+
+    pub fn mem_hotplug_by_probe(&self, req: &super::agent::MemHotplugByProbeRequest, timeout_nano: i64) -> ::ttrpc::Result<super::empty::Empty> {
+        let mut cres = super::empty::Empty::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "MemHotplugByProbe", cres);
+        Ok(cres)
+    }
+
+    pub fn set_guest_date_time(&self, req: &super::agent::SetGuestDateTimeRequest, timeout_nano: i64) -> ::ttrpc::Result<super::empty::Empty> {
+        let mut cres = super::empty::Empty::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "SetGuestDateTime", cres);
+        Ok(cres)
+    }
+
+    pub fn copy_file(&self, req: &super::agent::CopyFileRequest, timeout_nano: i64) -> ::ttrpc::Result<super::empty::Empty> {
+        let mut cres = super::empty::Empty::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "CopyFile", cres);
+        Ok(cres)
+    }
+
+    pub fn get_oom_event(&self, req: &super::agent::GetOOMEventRequest, timeout_nano: i64) -> ::ttrpc::Result<super::agent::OOMEvent> {
+        let mut cres = super::agent::OOMEvent::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.AgentService", "GetOOMEvent", cres);
+        Ok(cres)
+    }
+}
+
+struct CreateContainerMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for CreateContainerMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, CreateContainerRequest, create_container);
+        Ok(())
+    }
+}
+
+struct StartContainerMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for StartContainerMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, StartContainerRequest, start_container);
+        Ok(())
+    }
+}
+
+struct RemoveContainerMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for RemoveContainerMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, RemoveContainerRequest, remove_container);
+        Ok(())
+    }
+}
+
+struct ExecProcessMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for ExecProcessMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, ExecProcessRequest, exec_process);
+        Ok(())
+    }
+}
+
+struct SignalProcessMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for SignalProcessMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, SignalProcessRequest, signal_process);
+        Ok(())
+    }
+}
+
+struct WaitProcessMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for WaitProcessMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, WaitProcessRequest, wait_process);
+        Ok(())
+    }
+}
+
+struct ListProcessesMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for ListProcessesMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, ListProcessesRequest, list_processes);
+        Ok(())
+    }
+}
+
+struct UpdateContainerMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for UpdateContainerMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, UpdateContainerRequest, update_container);
+        Ok(())
+    }
+}
+
+struct StatsContainerMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for StatsContainerMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, StatsContainerRequest, stats_container);
+        Ok(())
+    }
+}
+
+struct PauseContainerMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for PauseContainerMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, PauseContainerRequest, pause_container);
+        Ok(())
+    }
+}
+
+struct ResumeContainerMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for ResumeContainerMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, ResumeContainerRequest, resume_container);
+        Ok(())
+    }
+}
+
+struct WriteStdinMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for WriteStdinMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, WriteStreamRequest, write_stdin);
+        Ok(())
+    }
+}
+
+struct ReadStdoutMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for ReadStdoutMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, ReadStreamRequest, read_stdout);
+        Ok(())
+    }
+}
+
+struct ReadStderrMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for ReadStderrMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, ReadStreamRequest, read_stderr);
+        Ok(())
+    }
+}
+
+struct CloseStdinMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for CloseStdinMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, CloseStdinRequest, close_stdin);
+        Ok(())
+    }
+}
+
+struct TtyWinResizeMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for TtyWinResizeMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, TtyWinResizeRequest, tty_win_resize);
+        Ok(())
+    }
+}
+
+struct UpdateInterfaceMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for UpdateInterfaceMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, UpdateInterfaceRequest, update_interface);
+        Ok(())
+    }
+}
+
+struct UpdateRoutesMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for UpdateRoutesMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, UpdateRoutesRequest, update_routes);
+        Ok(())
+    }
+}
+
+struct ListInterfacesMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for ListInterfacesMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, ListInterfacesRequest, list_interfaces);
+        Ok(())
+    }
+}
+
+struct ListRoutesMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for ListRoutesMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, ListRoutesRequest, list_routes);
+        Ok(())
+    }
+}
+
+struct AddArpNeighborsMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for AddArpNeighborsMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, AddARPNeighborsRequest, add_arp_neighbors);
+        Ok(())
+    }
+}
+
+struct StartTracingMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for StartTracingMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, StartTracingRequest, start_tracing);
+        Ok(())
+    }
+}
+
+struct StopTracingMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for StopTracingMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, StopTracingRequest, stop_tracing);
+        Ok(())
+    }
+}
+
+struct GetMetricsMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for GetMetricsMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, GetMetricsRequest, get_metrics);
+        Ok(())
+    }
+}
+
+struct CreateSandboxMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for CreateSandboxMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, CreateSandboxRequest, create_sandbox);
+        Ok(())
+    }
+}
+
+struct DestroySandboxMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for DestroySandboxMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, DestroySandboxRequest, destroy_sandbox);
+        Ok(())
+    }
+}
+
+struct OnlineCpuMemMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for OnlineCpuMemMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, OnlineCPUMemRequest, online_cpu_mem);
+        Ok(())
+    }
+}
+
+struct ReseedRandomDevMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for ReseedRandomDevMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, ReseedRandomDevRequest, reseed_random_dev);
+        Ok(())
+    }
+}
+
+struct GetGuestDetailsMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for GetGuestDetailsMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, GuestDetailsRequest, get_guest_details);
+        Ok(())
+    }
+}
+
+struct MemHotplugByProbeMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for MemHotplugByProbeMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, MemHotplugByProbeRequest, mem_hotplug_by_probe);
+        Ok(())
+    }
+}
+
+struct SetGuestDateTimeMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for SetGuestDateTimeMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, SetGuestDateTimeRequest, set_guest_date_time);
+        Ok(())
+    }
+}
+
+struct CopyFileMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for CopyFileMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, CopyFileRequest, copy_file);
+        Ok(())
+    }
+}
+
+struct GetOomEventMethod {
+    service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for GetOomEventMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, agent, GetOOMEventRequest, get_oom_event);
+        Ok(())
+    }
+}
+
+pub trait AgentService {
+    fn create_container(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::CreateContainerRequest) -> ::ttrpc::Result<super::empty::Empty> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/CreateContainer is not supported".to_string())))
+    }
+    fn start_container(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::StartContainerRequest) -> ::ttrpc::Result<super::empty::Empty> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/StartContainer is not supported".to_string())))
+    }
+    fn remove_container(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::RemoveContainerRequest) -> ::ttrpc::Result<super::empty::Empty> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/RemoveContainer is not supported".to_string())))
+    }
+    fn exec_process(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::ExecProcessRequest) -> ::ttrpc::Result<super::empty::Empty> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/ExecProcess is not supported".to_string())))
+    }
+    fn signal_process(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::SignalProcessRequest) -> ::ttrpc::Result<super::empty::Empty> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/SignalProcess is not supported".to_string())))
+    }
+    fn wait_process(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::WaitProcessRequest) -> ::ttrpc::Result<super::agent::WaitProcessResponse> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/WaitProcess is not supported".to_string())))
+    }
+    fn list_processes(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::ListProcessesRequest) -> ::ttrpc::Result<super::agent::ListProcessesResponse> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/ListProcesses is not supported".to_string())))
+    }
+    fn update_container(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::UpdateContainerRequest) -> ::ttrpc::Result<super::empty::Empty> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/UpdateContainer is not supported".to_string())))
+    }
+    fn stats_container(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::StatsContainerRequest) -> ::ttrpc::Result<super::agent::StatsContainerResponse> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/StatsContainer is not supported".to_string())))
+    }
+    fn pause_container(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::PauseContainerRequest) -> ::ttrpc::Result<super::empty::Empty> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/PauseContainer is not supported".to_string())))
+    }
+    fn resume_container(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::ResumeContainerRequest) -> ::ttrpc::Result<super::empty::Empty> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/ResumeContainer is not supported".to_string())))
+    }
+    fn write_stdin(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::WriteStreamRequest) -> ::ttrpc::Result<super::agent::WriteStreamResponse> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/WriteStdin is not supported".to_string())))
+    }
+    fn read_stdout(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::ReadStreamRequest) -> ::ttrpc::Result<super::agent::ReadStreamResponse> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/ReadStdout is not supported".to_string())))
+    }
+    fn read_stderr(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::ReadStreamRequest) -> ::ttrpc::Result<super::agent::ReadStreamResponse> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/ReadStderr is not supported".to_string())))
+    }
+    fn close_stdin(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::CloseStdinRequest) -> ::ttrpc::Result<super::empty::Empty> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/CloseStdin is not supported".to_string())))
+    }
+    fn tty_win_resize(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::TtyWinResizeRequest) -> ::ttrpc::Result<super::empty::Empty> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/TtyWinResize is not supported".to_string())))
+    }
+    fn update_interface(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::UpdateInterfaceRequest) -> ::ttrpc::Result<super::types::Interface> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/UpdateInterface is not supported".to_string())))
+    }
+    fn update_routes(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::UpdateRoutesRequest) -> ::ttrpc::Result<super::agent::Routes> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/UpdateRoutes is not supported".to_string())))
+    }
+    fn list_interfaces(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::ListInterfacesRequest) -> ::ttrpc::Result<super::agent::Interfaces> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/ListInterfaces is not supported".to_string())))
+    }
+    fn list_routes(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::ListRoutesRequest) -> ::ttrpc::Result<super::agent::Routes> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/ListRoutes is not supported".to_string())))
+    }
+    fn add_arp_neighbors(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::AddARPNeighborsRequest) -> ::ttrpc::Result<super::empty::Empty> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/AddARPNeighbors is not supported".to_string())))
+    }
+    fn start_tracing(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::StartTracingRequest) -> ::ttrpc::Result<super::empty::Empty> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/StartTracing is not supported".to_string())))
+    }
+    fn stop_tracing(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::StopTracingRequest) -> ::ttrpc::Result<super::empty::Empty> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/StopTracing is not supported".to_string())))
+    }
+    fn get_metrics(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::GetMetricsRequest) -> ::ttrpc::Result<super::agent::Metrics> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/GetMetrics is not supported".to_string())))
+    }
+    fn create_sandbox(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::CreateSandboxRequest) -> ::ttrpc::Result<super::empty::Empty> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/CreateSandbox is not supported".to_string())))
+    }
+    fn destroy_sandbox(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::DestroySandboxRequest) -> ::ttrpc::Result<super::empty::Empty> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/DestroySandbox is not supported".to_string())))
+    }
+    fn online_cpu_mem(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::OnlineCPUMemRequest) -> ::ttrpc::Result<super::empty::Empty> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/OnlineCPUMem is not supported".to_string())))
+    }
+    fn reseed_random_dev(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::ReseedRandomDevRequest) -> ::ttrpc::Result<super::empty::Empty> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/ReseedRandomDev is not supported".to_string())))
+    }
+    fn get_guest_details(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::GuestDetailsRequest) -> ::ttrpc::Result<super::agent::GuestDetailsResponse> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/GetGuestDetails is not supported".to_string())))
+    }
+    fn mem_hotplug_by_probe(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::MemHotplugByProbeRequest) -> ::ttrpc::Result<super::empty::Empty> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/MemHotplugByProbe is not supported".to_string())))
+    }
+    fn set_guest_date_time(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::SetGuestDateTimeRequest) -> ::ttrpc::Result<super::empty::Empty> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/SetGuestDateTime is not supported".to_string())))
+    }
+    fn copy_file(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::CopyFileRequest) -> ::ttrpc::Result<super::empty::Empty> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/CopyFile is not supported".to_string())))
+    }
+    fn get_oom_event(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::agent::GetOOMEventRequest) -> ::ttrpc::Result<super::agent::OOMEvent> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/GetOOMEvent is not supported".to_string())))
+    }
+}
+
+pub fn create_agent_service(service: Arc<std::boxed::Box<dyn AgentService + Send + Sync>>) -> HashMap <String, Box<dyn ::ttrpc::MethodHandler + Send + Sync>> {
+    let mut methods = HashMap::new();
+
+    methods.insert("/grpc.AgentService/CreateContainer".to_string(),
+                    std::boxed::Box::new(CreateContainerMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/StartContainer".to_string(),
+                    std::boxed::Box::new(StartContainerMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/RemoveContainer".to_string(),
+                    std::boxed::Box::new(RemoveContainerMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/ExecProcess".to_string(),
+                    std::boxed::Box::new(ExecProcessMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/SignalProcess".to_string(),
+                    std::boxed::Box::new(SignalProcessMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/WaitProcess".to_string(),
+                    std::boxed::Box::new(WaitProcessMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/ListProcesses".to_string(),
+                    std::boxed::Box::new(ListProcessesMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/UpdateContainer".to_string(),
+                    std::boxed::Box::new(UpdateContainerMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/StatsContainer".to_string(),
+                    std::boxed::Box::new(StatsContainerMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/PauseContainer".to_string(),
+                    std::boxed::Box::new(PauseContainerMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/ResumeContainer".to_string(),
+                    std::boxed::Box::new(ResumeContainerMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/WriteStdin".to_string(),
+                    std::boxed::Box::new(WriteStdinMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/ReadStdout".to_string(),
+                    std::boxed::Box::new(ReadStdoutMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/ReadStderr".to_string(),
+                    std::boxed::Box::new(ReadStderrMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/CloseStdin".to_string(),
+                    std::boxed::Box::new(CloseStdinMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/TtyWinResize".to_string(),
+                    std::boxed::Box::new(TtyWinResizeMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/UpdateInterface".to_string(),
+                    std::boxed::Box::new(UpdateInterfaceMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/UpdateRoutes".to_string(),
+                    std::boxed::Box::new(UpdateRoutesMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/ListInterfaces".to_string(),
+                    std::boxed::Box::new(ListInterfacesMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/ListRoutes".to_string(),
+                    std::boxed::Box::new(ListRoutesMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/AddARPNeighbors".to_string(),
+                    std::boxed::Box::new(AddArpNeighborsMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/StartTracing".to_string(),
+                    std::boxed::Box::new(StartTracingMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/StopTracing".to_string(),
+                    std::boxed::Box::new(StopTracingMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/GetMetrics".to_string(),
+                    std::boxed::Box::new(GetMetricsMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/CreateSandbox".to_string(),
+                    std::boxed::Box::new(CreateSandboxMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/DestroySandbox".to_string(),
+                    std::boxed::Box::new(DestroySandboxMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/OnlineCPUMem".to_string(),
+                    std::boxed::Box::new(OnlineCpuMemMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/ReseedRandomDev".to_string(),
+                    std::boxed::Box::new(ReseedRandomDevMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/GetGuestDetails".to_string(),
+                    std::boxed::Box::new(GetGuestDetailsMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/MemHotplugByProbe".to_string(),
+                    std::boxed::Box::new(MemHotplugByProbeMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/SetGuestDateTime".to_string(),
+                    std::boxed::Box::new(SetGuestDateTimeMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/CopyFile".to_string(),
+                    std::boxed::Box::new(CopyFileMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.AgentService/GetOOMEvent".to_string(),
+                    std::boxed::Box::new(GetOomEventMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods
+}

src/agent/protocols/src/empty.rs

@@ -0,0 +1,242 @@
+// Copyright (c) 2019 Ant Financial
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// This file is generated by rust-protobuf 2.14.0. Do not edit
+// @generated
+
+// https://github.com/rust-lang/rust-clippy/issues/702
+#![allow(unknown_lints)]
+#![allow(clippy::all)]
+
+#![cfg_attr(rustfmt, rustfmt_skip)]
+
+#![allow(box_pointers)]
+#![allow(dead_code)]
+#![allow(missing_docs)]
+#![allow(non_camel_case_types)]
+#![allow(non_snake_case)]
+#![allow(non_upper_case_globals)]
+#![allow(trivial_casts)]
+#![allow(unsafe_code)]
+#![allow(unused_imports)]
+#![allow(unused_results)]
+//! Generated file from `google/protobuf/empty.proto`
+
+use protobuf::Message as Message_imported_for_functions;
+use protobuf::ProtobufEnum as ProtobufEnum_imported_for_functions;
+
+/// Generated files are compatible only with the same version
+/// of protobuf runtime.
+// const _PROTOBUF_VERSION_CHECK: () = ::protobuf::VERSION_2_14_0;
+
+#[derive(PartialEq,Clone,Default)]
+pub struct Empty {
+    // special fields
+    pub unknown_fields: ::protobuf::UnknownFields,
+    pub cached_size: ::protobuf::CachedSize,
+}
+
+impl<'a> ::std::default::Default for &'a Empty {
+    fn default() -> &'a Empty {
+        <Empty as ::protobuf::Message>::default_instance()
+    }
+}
+
+impl Empty {
+    pub fn new() -> Empty {
+        ::std::default::Default::default()
+    }
+}
+
+impl ::protobuf::Message for Empty {
+    fn is_initialized(&self) -> bool {
+        true
+    }
+
+    fn merge_from(&mut self, is: &mut ::protobuf::CodedInputStream<'_>) -> ::protobuf::ProtobufResult<()> {
+        while !is.eof()? {
+            let (field_number, wire_type) = is.read_tag_unpack()?;
+            match field_number {
+                _ => {
+                    ::protobuf::rt::read_unknown_or_skip_group(field_number, wire_type, is, self.mut_unknown_fields())?;
+                },
+            };
+        }
+        ::std::result::Result::Ok(())
+    }
+
+    // Compute sizes of nested messages
+    #[allow(unused_variables)]
+    fn compute_size(&self) -> u32 {
+        let mut my_size = 0;
+        my_size += ::protobuf::rt::unknown_fields_size(self.get_unknown_fields());
+        self.cached_size.set(my_size);
+        my_size
+    }
+
+    fn write_to_with_cached_sizes(&self, os: &mut ::protobuf::CodedOutputStream<'_>) -> ::protobuf::ProtobufResult<()> {
+        os.write_unknown_fields(self.get_unknown_fields())?;
+        ::std::result::Result::Ok(())
+    }
+
+    fn get_cached_size(&self) -> u32 {
+        self.cached_size.get()
+    }
+
+    fn get_unknown_fields(&self) -> &::protobuf::UnknownFields {
+        &self.unknown_fields
+    }
+
+    fn mut_unknown_fields(&mut self) -> &mut ::protobuf::UnknownFields {
+        &mut self.unknown_fields
+    }
+
+    fn as_any(&self) -> &dyn (::std::any::Any) {
+        self as &dyn (::std::any::Any)
+    }
+    fn as_any_mut(&mut self) -> &mut dyn (::std::any::Any) {
+        self as &mut dyn (::std::any::Any)
+    }
+    fn into_any(self: Box<Self>) -> ::std::boxed::Box<dyn (::std::any::Any)> {
+        self
+    }
+
+    fn descriptor(&self) -> &'static ::protobuf::reflect::MessageDescriptor {
+        Self::descriptor_static()
+    }
+
+    fn new() -> Empty {
+        Empty::new()
+    }
+
+    fn descriptor_static() -> &'static ::protobuf::reflect::MessageDescriptor {
+        static mut descriptor: ::protobuf::lazy::Lazy<::protobuf::reflect::MessageDescriptor> = ::protobuf::lazy::Lazy::INIT;
+        unsafe {
+            descriptor.get(|| {
+                let fields = ::std::vec::Vec::new();
+                ::protobuf::reflect::MessageDescriptor::new_pb_name::<Empty>(
+                    "Empty",
+                    fields,
+                    file_descriptor_proto()
+                )
+            })
+        }
+    }
+
+    fn default_instance() -> &'static Empty {
+        static mut instance: ::protobuf::lazy::Lazy<Empty> = ::protobuf::lazy::Lazy::INIT;
+        unsafe {
+            instance.get(Empty::new)
+        }
+    }
+}
+
+impl ::protobuf::Clear for Empty {
+    fn clear(&mut self) {
+        self.unknown_fields.clear();
+    }
+}
+
+impl ::std::fmt::Debug for Empty {
+    fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result {
+        ::protobuf::text_format::fmt(self, f)
+    }
+}
+
+impl ::protobuf::reflect::ProtobufValue for Empty {
+    fn as_ref(&self) -> ::protobuf::reflect::ReflectValueRef {
+        ::protobuf::reflect::ReflectValueRef::Message(self)
+    }
+}
+
+static file_descriptor_proto_data: &'static [u8] = b"\
+    \n\x1bgoogle/protobuf/empty.proto\x12\x0fgoogle.protobuf\"\x07\n\x05Empt\
+    yBT\n\x13com.google.protobufB\nEmptyProtoP\x01Z\x05types\xf8\x01\x01\xa2\
+    \x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesJ\xa9\x14\n\x06\x12\
+    \x04\x1e\03\x10\n\xcc\x0c\n\x01\x0c\x12\x03\x1e\0\x122\xc1\x0c\x20Protoc\
+    ol\x20Buffers\x20-\x20Google's\x20data\x20interchange\x20format\n\x20Cop\
+    yright\x202008\x20Google\x20Inc.\x20\x20All\x20rights\x20reserved.\n\x20\
+    https://developers.google.com/protocol-buffers/\n\n\x20Redistribution\
+    \x20and\x20use\x20in\x20source\x20and\x20binary\x20forms,\x20with\x20or\
+    \x20without\n\x20modification,\x20are\x20permitted\x20provided\x20that\
+    \x20the\x20following\x20conditions\x20are\n\x20met:\n\n\x20\x20\x20\x20\
+    \x20*\x20Redistributions\x20of\x20source\x20code\x20must\x20retain\x20th\
+    e\x20above\x20copyright\n\x20notice,\x20this\x20list\x20of\x20conditions\
+    \x20and\x20the\x20following\x20disclaimer.\n\x20\x20\x20\x20\x20*\x20Red\
+    istributions\x20in\x20binary\x20form\x20must\x20reproduce\x20the\x20abov\
+    e\n\x20copyright\x20notice,\x20this\x20list\x20of\x20conditions\x20and\
+    \x20the\x20following\x20disclaimer\n\x20in\x20the\x20documentation\x20an\
+    d/or\x20other\x20materials\x20provided\x20with\x20the\n\x20distribution.\
+    \n\x20\x20\x20\x20\x20*\x20Neither\x20the\x20name\x20of\x20Google\x20Inc\
+    .\x20nor\x20the\x20names\x20of\x20its\n\x20contributors\x20may\x20be\x20\
+    used\x20to\x20endorse\x20or\x20promote\x20products\x20derived\x20from\n\
+    \x20this\x20software\x20without\x20specific\x20prior\x20written\x20permi\
+    ssion.\n\n\x20THIS\x20SOFTWARE\x20IS\x20PROVIDED\x20BY\x20THE\x20COPYRIG\
+    HT\x20HOLDERS\x20AND\x20CONTRIBUTORS\n\x20\"AS\x20IS\"\x20AND\x20ANY\x20\
+    EXPRESS\x20OR\x20IMPLIED\x20WARRANTIES,\x20INCLUDING,\x20BUT\x20NOT\n\
+    \x20LIMITED\x20TO,\x20THE\x20IMPLIED\x20WARRANTIES\x20OF\x20MERCHANTABIL\
+    ITY\x20AND\x20FITNESS\x20FOR\n\x20A\x20PARTICULAR\x20PURPOSE\x20ARE\x20D\
+    ISCLAIMED.\x20IN\x20NO\x20EVENT\x20SHALL\x20THE\x20COPYRIGHT\n\x20OWNER\
+    \x20OR\x20CONTRIBUTORS\x20BE\x20LIABLE\x20FOR\x20ANY\x20DIRECT,\x20INDIR\
+    ECT,\x20INCIDENTAL,\n\x20SPECIAL,\x20EXEMPLARY,\x20OR\x20CONSEQUENTIAL\
+    \x20DAMAGES\x20(INCLUDING,\x20BUT\x20NOT\n\x20LIMITED\x20TO,\x20PROCUREM\
+    ENT\x20OF\x20SUBSTITUTE\x20GOODS\x20OR\x20SERVICES;\x20LOSS\x20OF\x20USE\
+    ,\n\x20DATA,\x20OR\x20PROFITS;\x20OR\x20BUSINESS\x20INTERRUPTION)\x20HOW\
+    EVER\x20CAUSED\x20AND\x20ON\x20ANY\n\x20THEORY\x20OF\x20LIABILITY,\x20WH\
+    ETHER\x20IN\x20CONTRACT,\x20STRICT\x20LIABILITY,\x20OR\x20TORT\n\x20(INC\
+    LUDING\x20NEGLIGENCE\x20OR\x20OTHERWISE)\x20ARISING\x20IN\x20ANY\x20WAY\
+    \x20OUT\x20OF\x20THE\x20USE\n\x20OF\x20THIS\x20SOFTWARE,\x20EVEN\x20IF\
+    \x20ADVISED\x20OF\x20THE\x20POSSIBILITY\x20OF\x20SUCH\x20DAMAGE.\n\n\x08\
+    \n\x01\x02\x12\x03\x20\x08\x17\n\x08\n\x01\x08\x12\x03\"\0;\n\x0b\n\x04\
+    \x08\xe7\x07\0\x12\x03\"\0;\n\x0c\n\x05\x08\xe7\x07\0\x02\x12\x03\"\x07\
+    \x17\n\r\n\x06\x08\xe7\x07\0\x02\0\x12\x03\"\x07\x17\n\x0e\n\x07\x08\xe7\
+    \x07\0\x02\0\x01\x12\x03\"\x07\x17\n\x0c\n\x05\x08\xe7\x07\0\x07\x12\x03\
+    \"\x1a:\n\x08\n\x01\x08\x12\x03#\0\x1c\n\x0b\n\x04\x08\xe7\x07\x01\x12\
+    \x03#\0\x1c\n\x0c\n\x05\x08\xe7\x07\x01\x02\x12\x03#\x07\x11\n\r\n\x06\
+    \x08\xe7\x07\x01\x02\0\x12\x03#\x07\x11\n\x0e\n\x07\x08\xe7\x07\x01\x02\
+    \0\x01\x12\x03#\x07\x11\n\x0c\n\x05\x08\xe7\x07\x01\x07\x12\x03#\x14\x1b\
+    \n\x08\n\x01\x08\x12\x03$\0,\n\x0b\n\x04\x08\xe7\x07\x02\x12\x03$\0,\n\
+    \x0c\n\x05\x08\xe7\x07\x02\x02\x12\x03$\x07\x13\n\r\n\x06\x08\xe7\x07\
+    \x02\x02\0\x12\x03$\x07\x13\n\x0e\n\x07\x08\xe7\x07\x02\x02\0\x01\x12\
+    \x03$\x07\x13\n\x0c\n\x05\x08\xe7\x07\x02\x07\x12\x03$\x16+\n\x08\n\x01\
+    \x08\x12\x03%\0+\n\x0b\n\x04\x08\xe7\x07\x03\x12\x03%\0+\n\x0c\n\x05\x08\
+    \xe7\x07\x03\x02\x12\x03%\x07\x1b\n\r\n\x06\x08\xe7\x07\x03\x02\0\x12\
+    \x03%\x07\x1b\n\x0e\n\x07\x08\xe7\x07\x03\x02\0\x01\x12\x03%\x07\x1b\n\
+    \x0c\n\x05\x08\xe7\x07\x03\x07\x12\x03%\x1e*\n\x08\n\x01\x08\x12\x03&\0\
+    \"\n\x0b\n\x04\x08\xe7\x07\x04\x12\x03&\0\"\n\x0c\n\x05\x08\xe7\x07\x04\
+    \x02\x12\x03&\x07\x1a\n\r\n\x06\x08\xe7\x07\x04\x02\0\x12\x03&\x07\x1a\n\
+    \x0e\n\x07\x08\xe7\x07\x04\x02\0\x01\x12\x03&\x07\x1a\n\x0c\n\x05\x08\
+    \xe7\x07\x04\x03\x12\x03&\x1d!\n\x08\n\x01\x08\x12\x03'\0!\n\x0b\n\x04\
+    \x08\xe7\x07\x05\x12\x03'\0!\n\x0c\n\x05\x08\xe7\x07\x05\x02\x12\x03'\
+    \x07\x18\n\r\n\x06\x08\xe7\x07\x05\x02\0\x12\x03'\x07\x18\n\x0e\n\x07\
+    \x08\xe7\x07\x05\x02\0\x01\x12\x03'\x07\x18\n\x0c\n\x05\x08\xe7\x07\x05\
+    \x07\x12\x03'\x1b\x20\n\x08\n\x01\x08\x12\x03(\0\x1f\n\x0b\n\x04\x08\xe7\
+    \x07\x06\x12\x03(\0\x1f\n\x0c\n\x05\x08\xe7\x07\x06\x02\x12\x03(\x07\x17\
+    \n\r\n\x06\x08\xe7\x07\x06\x02\0\x12\x03(\x07\x17\n\x0e\n\x07\x08\xe7\
+    \x07\x06\x02\0\x01\x12\x03(\x07\x17\n\x0c\n\x05\x08\xe7\x07\x06\x03\x12\
+    \x03(\x1a\x1e\n\xfb\x02\n\x02\x04\0\x12\x033\0\x10\x1a\xef\x02\x20A\x20g\
+    eneric\x20empty\x20message\x20that\x20you\x20can\x20re-use\x20to\x20avoi\
+    d\x20defining\x20duplicated\n\x20empty\x20messages\x20in\x20your\x20APIs\
+    .\x20A\x20typical\x20example\x20is\x20to\x20use\x20it\x20as\x20the\x20re\
+    quest\n\x20or\x20the\x20response\x20type\x20of\x20an\x20API\x20method.\
+    \x20For\x20instance:\n\n\x20\x20\x20\x20\x20service\x20Foo\x20{\n\x20\
+    \x20\x20\x20\x20\x20\x20rpc\x20Bar(google.protobuf.Empty)\x20returns\x20\
+    (google.protobuf.Empty);\n\x20\x20\x20\x20\x20}\n\n\x20The\x20JSON\x20re\
+    presentation\x20for\x20`Empty`\x20is\x20empty\x20JSON\x20object\x20`{}`.\
+    \n\n\n\n\x03\x04\0\x01\x12\x033\x08\rb\x06proto3\
+";
+
+static mut file_descriptor_proto_lazy: ::protobuf::lazy::Lazy<::protobuf::descriptor::FileDescriptorProto> = ::protobuf::lazy::Lazy::INIT;
+
+fn parse_descriptor_proto() -> ::protobuf::descriptor::FileDescriptorProto {
+    ::protobuf::parse_from_bytes(file_descriptor_proto_data).unwrap()
+}
+
+pub fn file_descriptor_proto() -> &'static ::protobuf::descriptor::FileDescriptorProto {
+    unsafe {
+        file_descriptor_proto_lazy.get(|| {
+            parse_descriptor_proto()
+        })
+    }
+}

src/agent/protocols/src/health.rs

@@ -0,0 +1,672 @@
+// This file is generated by rust-protobuf 2.14.0. Do not edit
+// @generated
+
+// https://github.com/rust-lang/rust-clippy/issues/702
+#![allow(unknown_lints)]
+#![allow(clippy::all)]
+
+#![cfg_attr(rustfmt, rustfmt_skip)]
+
+#![allow(box_pointers)]
+#![allow(dead_code)]
+#![allow(missing_docs)]
+#![allow(non_camel_case_types)]
+#![allow(non_snake_case)]
+#![allow(non_upper_case_globals)]
+#![allow(trivial_casts)]
+#![allow(unsafe_code)]
+#![allow(unused_imports)]
+#![allow(unused_results)]
+//! Generated file from `github.com/kata-containers/kata-containers/src/agent/protocols/protos/health.proto`
+
+use protobuf::Message as Message_imported_for_functions;
+use protobuf::ProtobufEnum as ProtobufEnum_imported_for_functions;
+
+/// Generated files are compatible only with the same version
+/// of protobuf runtime.
+// const _PROTOBUF_VERSION_CHECK: () = ::protobuf::VERSION_2_14_0;
+
+#[derive(PartialEq,Clone,Default)]
+pub struct CheckRequest {
+    // message fields
+    pub service: ::std::string::String,
+    // special fields
+    pub unknown_fields: ::protobuf::UnknownFields,
+    pub cached_size: ::protobuf::CachedSize,
+}
+
+impl<'a> ::std::default::Default for &'a CheckRequest {
+    fn default() -> &'a CheckRequest {
+        <CheckRequest as ::protobuf::Message>::default_instance()
+    }
+}
+
+impl CheckRequest {
+    pub fn new() -> CheckRequest {
+        ::std::default::Default::default()
+    }
+
+    // string service = 1;
+
+
+    pub fn get_service(&self) -> &str {
+        &self.service
+    }
+    pub fn clear_service(&mut self) {
+        self.service.clear();
+    }
+
+    // Param is passed by value, moved
+    pub fn set_service(&mut self, v: ::std::string::String) {
+        self.service = v;
+    }
+
+    // Mutable pointer to the field.
+    // If field is not initialized, it is initialized with default value first.
+    pub fn mut_service(&mut self) -> &mut ::std::string::String {
+        &mut self.service
+    }
+
+    // Take field
+    pub fn take_service(&mut self) -> ::std::string::String {
+        ::std::mem::replace(&mut self.service, ::std::string::String::new())
+    }
+}
+
+impl ::protobuf::Message for CheckRequest {
+    fn is_initialized(&self) -> bool {
+        true
+    }
+
+    fn merge_from(&mut self, is: &mut ::protobuf::CodedInputStream<'_>) -> ::protobuf::ProtobufResult<()> {
+        while !is.eof()? {
+            let (field_number, wire_type) = is.read_tag_unpack()?;
+            match field_number {
+                1 => {
+                    ::protobuf::rt::read_singular_proto3_string_into(wire_type, is, &mut self.service)?;
+                },
+                _ => {
+                    ::protobuf::rt::read_unknown_or_skip_group(field_number, wire_type, is, self.mut_unknown_fields())?;
+                },
+            };
+        }
+        ::std::result::Result::Ok(())
+    }
+
+    // Compute sizes of nested messages
+    #[allow(unused_variables)]
+    fn compute_size(&self) -> u32 {
+        let mut my_size = 0;
+        if !self.service.is_empty() {
+            my_size += ::protobuf::rt::string_size(1, &self.service);
+        }
+        my_size += ::protobuf::rt::unknown_fields_size(self.get_unknown_fields());
+        self.cached_size.set(my_size);
+        my_size
+    }
+
+    fn write_to_with_cached_sizes(&self, os: &mut ::protobuf::CodedOutputStream<'_>) -> ::protobuf::ProtobufResult<()> {
+        if !self.service.is_empty() {
+            os.write_string(1, &self.service)?;
+        }
+        os.write_unknown_fields(self.get_unknown_fields())?;
+        ::std::result::Result::Ok(())
+    }
+
+    fn get_cached_size(&self) -> u32 {
+        self.cached_size.get()
+    }
+
+    fn get_unknown_fields(&self) -> &::protobuf::UnknownFields {
+        &self.unknown_fields
+    }
+
+    fn mut_unknown_fields(&mut self) -> &mut ::protobuf::UnknownFields {
+        &mut self.unknown_fields
+    }
+
+    fn as_any(&self) -> &dyn (::std::any::Any) {
+        self as &dyn (::std::any::Any)
+    }
+    fn as_any_mut(&mut self) -> &mut dyn (::std::any::Any) {
+        self as &mut dyn (::std::any::Any)
+    }
+    fn into_any(self: Box<Self>) -> ::std::boxed::Box<dyn (::std::any::Any)> {
+        self
+    }
+
+    fn descriptor(&self) -> &'static ::protobuf::reflect::MessageDescriptor {
+        Self::descriptor_static()
+    }
+
+    fn new() -> CheckRequest {
+        CheckRequest::new()
+    }
+
+    fn descriptor_static() -> &'static ::protobuf::reflect::MessageDescriptor {
+        static mut descriptor: ::protobuf::lazy::Lazy<::protobuf::reflect::MessageDescriptor> = ::protobuf::lazy::Lazy::INIT;
+        unsafe {
+            descriptor.get(|| {
+                let mut fields = ::std::vec::Vec::new();
+                fields.push(::protobuf::reflect::accessor::make_simple_field_accessor::<_, ::protobuf::types::ProtobufTypeString>(
+                    "service",
+                    |m: &CheckRequest| { &m.service },
+                    |m: &mut CheckRequest| { &mut m.service },
+                ));
+                ::protobuf::reflect::MessageDescriptor::new_pb_name::<CheckRequest>(
+                    "CheckRequest",
+                    fields,
+                    file_descriptor_proto()
+                )
+            })
+        }
+    }
+
+    fn default_instance() -> &'static CheckRequest {
+        static mut instance: ::protobuf::lazy::Lazy<CheckRequest> = ::protobuf::lazy::Lazy::INIT;
+        unsafe {
+            instance.get(CheckRequest::new)
+        }
+    }
+}
+
+impl ::protobuf::Clear for CheckRequest {
+    fn clear(&mut self) {
+        self.service.clear();
+        self.unknown_fields.clear();
+    }
+}
+
+impl ::std::fmt::Debug for CheckRequest {
+    fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result {
+        ::protobuf::text_format::fmt(self, f)
+    }
+}
+
+impl ::protobuf::reflect::ProtobufValue for CheckRequest {
+    fn as_ref(&self) -> ::protobuf::reflect::ReflectValueRef {
+        ::protobuf::reflect::ReflectValueRef::Message(self)
+    }
+}
+
+#[derive(PartialEq,Clone,Default)]
+pub struct HealthCheckResponse {
+    // message fields
+    pub status: HealthCheckResponse_ServingStatus,
+    // special fields
+    pub unknown_fields: ::protobuf::UnknownFields,
+    pub cached_size: ::protobuf::CachedSize,
+}
+
+impl<'a> ::std::default::Default for &'a HealthCheckResponse {
+    fn default() -> &'a HealthCheckResponse {
+        <HealthCheckResponse as ::protobuf::Message>::default_instance()
+    }
+}
+
+impl HealthCheckResponse {
+    pub fn new() -> HealthCheckResponse {
+        ::std::default::Default::default()
+    }
+
+    // .grpc.HealthCheckResponse.ServingStatus status = 1;
+
+
+    pub fn get_status(&self) -> HealthCheckResponse_ServingStatus {
+        self.status
+    }
+    pub fn clear_status(&mut self) {
+        self.status = HealthCheckResponse_ServingStatus::UNKNOWN;
+    }
+
+    // Param is passed by value, moved
+    pub fn set_status(&mut self, v: HealthCheckResponse_ServingStatus) {
+        self.status = v;
+    }
+}
+
+impl ::protobuf::Message for HealthCheckResponse {
+    fn is_initialized(&self) -> bool {
+        true
+    }
+
+    fn merge_from(&mut self, is: &mut ::protobuf::CodedInputStream<'_>) -> ::protobuf::ProtobufResult<()> {
+        while !is.eof()? {
+            let (field_number, wire_type) = is.read_tag_unpack()?;
+            match field_number {
+                1 => {
+                    ::protobuf::rt::read_proto3_enum_with_unknown_fields_into(wire_type, is, &mut self.status, 1, &mut self.unknown_fields)?
+                },
+                _ => {
+                    ::protobuf::rt::read_unknown_or_skip_group(field_number, wire_type, is, self.mut_unknown_fields())?;
+                },
+            };
+        }
+        ::std::result::Result::Ok(())
+    }
+
+    // Compute sizes of nested messages
+    #[allow(unused_variables)]
+    fn compute_size(&self) -> u32 {
+        let mut my_size = 0;
+        if self.status != HealthCheckResponse_ServingStatus::UNKNOWN {
+            my_size += ::protobuf::rt::enum_size(1, self.status);
+        }
+        my_size += ::protobuf::rt::unknown_fields_size(self.get_unknown_fields());
+        self.cached_size.set(my_size);
+        my_size
+    }
+
+    fn write_to_with_cached_sizes(&self, os: &mut ::protobuf::CodedOutputStream<'_>) -> ::protobuf::ProtobufResult<()> {
+        if self.status != HealthCheckResponse_ServingStatus::UNKNOWN {
+            os.write_enum(1, self.status.value())?;
+        }
+        os.write_unknown_fields(self.get_unknown_fields())?;
+        ::std::result::Result::Ok(())
+    }
+
+    fn get_cached_size(&self) -> u32 {
+        self.cached_size.get()
+    }
+
+    fn get_unknown_fields(&self) -> &::protobuf::UnknownFields {
+        &self.unknown_fields
+    }
+
+    fn mut_unknown_fields(&mut self) -> &mut ::protobuf::UnknownFields {
+        &mut self.unknown_fields
+    }
+
+    fn as_any(&self) -> &dyn (::std::any::Any) {
+        self as &dyn (::std::any::Any)
+    }
+    fn as_any_mut(&mut self) -> &mut dyn (::std::any::Any) {
+        self as &mut dyn (::std::any::Any)
+    }
+    fn into_any(self: Box<Self>) -> ::std::boxed::Box<dyn (::std::any::Any)> {
+        self
+    }
+
+    fn descriptor(&self) -> &'static ::protobuf::reflect::MessageDescriptor {
+        Self::descriptor_static()
+    }
+
+    fn new() -> HealthCheckResponse {
+        HealthCheckResponse::new()
+    }
+
+    fn descriptor_static() -> &'static ::protobuf::reflect::MessageDescriptor {
+        static mut descriptor: ::protobuf::lazy::Lazy<::protobuf::reflect::MessageDescriptor> = ::protobuf::lazy::Lazy::INIT;
+        unsafe {
+            descriptor.get(|| {
+                let mut fields = ::std::vec::Vec::new();
+                fields.push(::protobuf::reflect::accessor::make_simple_field_accessor::<_, ::protobuf::types::ProtobufTypeEnum<HealthCheckResponse_ServingStatus>>(
+                    "status",
+                    |m: &HealthCheckResponse| { &m.status },
+                    |m: &mut HealthCheckResponse| { &mut m.status },
+                ));
+                ::protobuf::reflect::MessageDescriptor::new_pb_name::<HealthCheckResponse>(
+                    "HealthCheckResponse",
+                    fields,
+                    file_descriptor_proto()
+                )
+            })
+        }
+    }
+
+    fn default_instance() -> &'static HealthCheckResponse {
+        static mut instance: ::protobuf::lazy::Lazy<HealthCheckResponse> = ::protobuf::lazy::Lazy::INIT;
+        unsafe {
+            instance.get(HealthCheckResponse::new)
+        }
+    }
+}
+
+impl ::protobuf::Clear for HealthCheckResponse {
+    fn clear(&mut self) {
+        self.status = HealthCheckResponse_ServingStatus::UNKNOWN;
+        self.unknown_fields.clear();
+    }
+}
+
+impl ::std::fmt::Debug for HealthCheckResponse {
+    fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result {
+        ::protobuf::text_format::fmt(self, f)
+    }
+}
+
+impl ::protobuf::reflect::ProtobufValue for HealthCheckResponse {
+    fn as_ref(&self) -> ::protobuf::reflect::ReflectValueRef {
+        ::protobuf::reflect::ReflectValueRef::Message(self)
+    }
+}
+
+#[derive(Clone,PartialEq,Eq,Debug,Hash)]
+pub enum HealthCheckResponse_ServingStatus {
+    UNKNOWN = 0,
+    SERVING = 1,
+    NOT_SERVING = 2,
+}
+
+impl ::protobuf::ProtobufEnum for HealthCheckResponse_ServingStatus {
+    fn value(&self) -> i32 {
+        *self as i32
+    }
+
+    fn from_i32(value: i32) -> ::std::option::Option<HealthCheckResponse_ServingStatus> {
+        match value {
+            0 => ::std::option::Option::Some(HealthCheckResponse_ServingStatus::UNKNOWN),
+            1 => ::std::option::Option::Some(HealthCheckResponse_ServingStatus::SERVING),
+            2 => ::std::option::Option::Some(HealthCheckResponse_ServingStatus::NOT_SERVING),
+            _ => ::std::option::Option::None
+        }
+    }
+
+    fn values() -> &'static [Self] {
+        static values: &'static [HealthCheckResponse_ServingStatus] = &[
+            HealthCheckResponse_ServingStatus::UNKNOWN,
+            HealthCheckResponse_ServingStatus::SERVING,
+            HealthCheckResponse_ServingStatus::NOT_SERVING,
+        ];
+        values
+    }
+
+    fn enum_descriptor_static() -> &'static ::protobuf::reflect::EnumDescriptor {
+        static mut descriptor: ::protobuf::lazy::Lazy<::protobuf::reflect::EnumDescriptor> = ::protobuf::lazy::Lazy::INIT;
+        unsafe {
+            descriptor.get(|| {
+                ::protobuf::reflect::EnumDescriptor::new_pb_name::<HealthCheckResponse_ServingStatus>("HealthCheckResponse.ServingStatus", file_descriptor_proto())
+            })
+        }
+    }
+}
+
+impl ::std::marker::Copy for HealthCheckResponse_ServingStatus {
+}
+
+impl ::std::default::Default for HealthCheckResponse_ServingStatus {
+    fn default() -> Self {
+        HealthCheckResponse_ServingStatus::UNKNOWN
+    }
+}
+
+impl ::protobuf::reflect::ProtobufValue for HealthCheckResponse_ServingStatus {
+    fn as_ref(&self) -> ::protobuf::reflect::ReflectValueRef {
+        ::protobuf::reflect::ReflectValueRef::Enum(self.descriptor())
+    }
+}
+
+#[derive(PartialEq,Clone,Default)]
+pub struct VersionCheckResponse {
+    // message fields
+    pub grpc_version: ::std::string::String,
+    pub agent_version: ::std::string::String,
+    // special fields
+    pub unknown_fields: ::protobuf::UnknownFields,
+    pub cached_size: ::protobuf::CachedSize,
+}
+
+impl<'a> ::std::default::Default for &'a VersionCheckResponse {
+    fn default() -> &'a VersionCheckResponse {
+        <VersionCheckResponse as ::protobuf::Message>::default_instance()
+    }
+}
+
+impl VersionCheckResponse {
+    pub fn new() -> VersionCheckResponse {
+        ::std::default::Default::default()
+    }
+
+    // string grpc_version = 1;
+
+
+    pub fn get_grpc_version(&self) -> &str {
+        &self.grpc_version
+    }
+    pub fn clear_grpc_version(&mut self) {
+        self.grpc_version.clear();
+    }
+
+    // Param is passed by value, moved
+    pub fn set_grpc_version(&mut self, v: ::std::string::String) {
+        self.grpc_version = v;
+    }
+
+    // Mutable pointer to the field.
+    // If field is not initialized, it is initialized with default value first.
+    pub fn mut_grpc_version(&mut self) -> &mut ::std::string::String {
+        &mut self.grpc_version
+    }
+
+    // Take field
+    pub fn take_grpc_version(&mut self) -> ::std::string::String {
+        ::std::mem::replace(&mut self.grpc_version, ::std::string::String::new())
+    }
+
+    // string agent_version = 2;
+
+
+    pub fn get_agent_version(&self) -> &str {
+        &self.agent_version
+    }
+    pub fn clear_agent_version(&mut self) {
+        self.agent_version.clear();
+    }
+
+    // Param is passed by value, moved
+    pub fn set_agent_version(&mut self, v: ::std::string::String) {
+        self.agent_version = v;
+    }
+
+    // Mutable pointer to the field.
+    // If field is not initialized, it is initialized with default value first.
+    pub fn mut_agent_version(&mut self) -> &mut ::std::string::String {
+        &mut self.agent_version
+    }
+
+    // Take field
+    pub fn take_agent_version(&mut self) -> ::std::string::String {
+        ::std::mem::replace(&mut self.agent_version, ::std::string::String::new())
+    }
+}
+
+impl ::protobuf::Message for VersionCheckResponse {
+    fn is_initialized(&self) -> bool {
+        true
+    }
+
+    fn merge_from(&mut self, is: &mut ::protobuf::CodedInputStream<'_>) -> ::protobuf::ProtobufResult<()> {
+        while !is.eof()? {
+            let (field_number, wire_type) = is.read_tag_unpack()?;
+            match field_number {
+                1 => {
+                    ::protobuf::rt::read_singular_proto3_string_into(wire_type, is, &mut self.grpc_version)?;
+                },
+                2 => {
+                    ::protobuf::rt::read_singular_proto3_string_into(wire_type, is, &mut self.agent_version)?;
+                },
+                _ => {
+                    ::protobuf::rt::read_unknown_or_skip_group(field_number, wire_type, is, self.mut_unknown_fields())?;
+                },
+            };
+        }
+        ::std::result::Result::Ok(())
+    }
+
+    // Compute sizes of nested messages
+    #[allow(unused_variables)]
+    fn compute_size(&self) -> u32 {
+        let mut my_size = 0;
+        if !self.grpc_version.is_empty() {
+            my_size += ::protobuf::rt::string_size(1, &self.grpc_version);
+        }
+        if !self.agent_version.is_empty() {
+            my_size += ::protobuf::rt::string_size(2, &self.agent_version);
+        }
+        my_size += ::protobuf::rt::unknown_fields_size(self.get_unknown_fields());
+        self.cached_size.set(my_size);
+        my_size
+    }
+
+    fn write_to_with_cached_sizes(&self, os: &mut ::protobuf::CodedOutputStream<'_>) -> ::protobuf::ProtobufResult<()> {
+        if !self.grpc_version.is_empty() {
+            os.write_string(1, &self.grpc_version)?;
+        }
+        if !self.agent_version.is_empty() {
+            os.write_string(2, &self.agent_version)?;
+        }
+        os.write_unknown_fields(self.get_unknown_fields())?;
+        ::std::result::Result::Ok(())
+    }
+
+    fn get_cached_size(&self) -> u32 {
+        self.cached_size.get()
+    }
+
+    fn get_unknown_fields(&self) -> &::protobuf::UnknownFields {
+        &self.unknown_fields
+    }
+
+    fn mut_unknown_fields(&mut self) -> &mut ::protobuf::UnknownFields {
+        &mut self.unknown_fields
+    }
+
+    fn as_any(&self) -> &dyn (::std::any::Any) {
+        self as &dyn (::std::any::Any)
+    }
+    fn as_any_mut(&mut self) -> &mut dyn (::std::any::Any) {
+        self as &mut dyn (::std::any::Any)
+    }
+    fn into_any(self: Box<Self>) -> ::std::boxed::Box<dyn (::std::any::Any)> {
+        self
+    }
+
+    fn descriptor(&self) -> &'static ::protobuf::reflect::MessageDescriptor {
+        Self::descriptor_static()
+    }
+
+    fn new() -> VersionCheckResponse {
+        VersionCheckResponse::new()
+    }
+
+    fn descriptor_static() -> &'static ::protobuf::reflect::MessageDescriptor {
+        static mut descriptor: ::protobuf::lazy::Lazy<::protobuf::reflect::MessageDescriptor> = ::protobuf::lazy::Lazy::INIT;
+        unsafe {
+            descriptor.get(|| {
+                let mut fields = ::std::vec::Vec::new();
+                fields.push(::protobuf::reflect::accessor::make_simple_field_accessor::<_, ::protobuf::types::ProtobufTypeString>(
+                    "grpc_version",
+                    |m: &VersionCheckResponse| { &m.grpc_version },
+                    |m: &mut VersionCheckResponse| { &mut m.grpc_version },
+                ));
+                fields.push(::protobuf::reflect::accessor::make_simple_field_accessor::<_, ::protobuf::types::ProtobufTypeString>(
+                    "agent_version",
+                    |m: &VersionCheckResponse| { &m.agent_version },
+                    |m: &mut VersionCheckResponse| { &mut m.agent_version },
+                ));
+                ::protobuf::reflect::MessageDescriptor::new_pb_name::<VersionCheckResponse>(
+                    "VersionCheckResponse",
+                    fields,
+                    file_descriptor_proto()
+                )
+            })
+        }
+    }
+
+    fn default_instance() -> &'static VersionCheckResponse {
+        static mut instance: ::protobuf::lazy::Lazy<VersionCheckResponse> = ::protobuf::lazy::Lazy::INIT;
+        unsafe {
+            instance.get(VersionCheckResponse::new)
+        }
+    }
+}
+
+impl ::protobuf::Clear for VersionCheckResponse {
+    fn clear(&mut self) {
+        self.grpc_version.clear();
+        self.agent_version.clear();
+        self.unknown_fields.clear();
+    }
+}
+
+impl ::std::fmt::Debug for VersionCheckResponse {
+    fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result {
+        ::protobuf::text_format::fmt(self, f)
+    }
+}
+
+impl ::protobuf::reflect::ProtobufValue for VersionCheckResponse {
+    fn as_ref(&self) -> ::protobuf::reflect::ReflectValueRef {
+        ::protobuf::reflect::ReflectValueRef::Message(self)
+    }
+}
+
+static file_descriptor_proto_data: &'static [u8] = b"\
+    \nRgithub.com/kata-containers/kata-containers/src/agent/protocols/protos\
+    /health.proto\x12\x04grpc\x1a-github.com/gogo/protobuf/gogoproto/gogo.pr\
+    oto\"(\n\x0cCheckRequest\x12\x18\n\x07service\x18\x01\x20\x01(\tR\x07ser\
+    vice\"\x92\x01\n\x13HealthCheckResponse\x12?\n\x06status\x18\x01\x20\x01\
+    (\x0e2'.grpc.HealthCheckResponse.ServingStatusR\x06status\":\n\rServingS\
+    tatus\x12\x0b\n\x07UNKNOWN\x10\0\x12\x0b\n\x07SERVING\x10\x01\x12\x0f\n\
+    \x0bNOT_SERVING\x10\x02\"^\n\x14VersionCheckResponse\x12!\n\x0cgrpc_vers\
+    ion\x18\x01\x20\x01(\tR\x0bgrpcVersion\x12#\n\ragent_version\x18\x02\x20\
+    \x01(\tR\x0cagentVersion2{\n\x06Health\x126\n\x05Check\x12\x12.grpc.Chec\
+    kRequest\x1a\x19.grpc.HealthCheckResponse\x129\n\x07Version\x12\x12.grpc\
+    .CheckRequest\x1a\x1a.grpc.VersionCheckResponseBpZ^github.com/kata-conta\
+    iners/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grp\
+    c\xb8\xe2\x1e\x01\xc0\xe2\x1e\x01\xa8\xe2\x1e\x01\xf8\xe1\x1e\x01J\x90\
+    \x07\n\x06\x12\x04\x07\0)\x01\nq\n\x01\x0c\x12\x03\x07\0\x122g\n\x20Copy\
+    right\x202017\x20HyperHQ\x20Inc.\n\x20Copyright\x20(c)\x202019\x20Ant\
+    \x20Financial\n\n\x20SPDX-License-Identifier:\x20Apache-2.0\n\n\n\x08\n\
+    \x01\x08\x12\x03\t\0u\n\t\n\x02\x08\x0b\x12\x03\t\0u\n\x08\n\x01\x02\x12\
+    \x03\x0b\0\r\n\t\n\x02\x03\0\x12\x03\r\07\n\x08\n\x01\x08\x12\x03\x0f\0$\
+    \n\x0b\n\x04\x08\xa5\xec\x03\x12\x03\x0f\0$\n\x08\n\x01\x08\x12\x03\x10\
+    \0'\n\x0b\n\x04\x08\x9f\xec\x03\x12\x03\x10\0'\n\x08\n\x01\x08\x12\x03\
+    \x11\0&\n\x0b\n\x04\x08\xa7\xec\x03\x12\x03\x11\0&\n\x08\n\x01\x08\x12\
+    \x03\x12\0'\n\x0b\n\x04\x08\xa8\xec\x03\x12\x03\x12\0'\n\n\n\x02\x04\0\
+    \x12\x04\x14\0\x16\x01\n\n\n\x03\x04\0\x01\x12\x03\x14\x08\x14\n\x0b\n\
+    \x04\x04\0\x02\0\x12\x03\x15\x08\x1b\n\r\n\x05\x04\0\x02\0\x04\x12\x04\
+    \x15\x08\x14\x16\n\x0c\n\x05\x04\0\x02\0\x05\x12\x03\x15\x08\x0e\n\x0c\n\
+    \x05\x04\0\x02\0\x01\x12\x03\x15\x0f\x16\n\x0c\n\x05\x04\0\x02\0\x03\x12\
+    \x03\x15\x19\x1a\n\n\n\x02\x04\x01\x12\x04\x18\0\x1f\x01\n\n\n\x03\x04\
+    \x01\x01\x12\x03\x18\x08\x1b\n\x0c\n\x04\x04\x01\x04\0\x12\x04\x19\x08\
+    \x1d\t\n\x0c\n\x05\x04\x01\x04\0\x01\x12\x03\x19\r\x1a\n\r\n\x06\x04\x01\
+    \x04\0\x02\0\x12\x03\x1a\x10\x1c\n\x0e\n\x07\x04\x01\x04\0\x02\0\x01\x12\
+    \x03\x1a\x10\x17\n\x0e\n\x07\x04\x01\x04\0\x02\0\x02\x12\x03\x1a\x1a\x1b\
+    \n\r\n\x06\x04\x01\x04\0\x02\x01\x12\x03\x1b\x10\x1c\n\x0e\n\x07\x04\x01\
+    \x04\0\x02\x01\x01\x12\x03\x1b\x10\x17\n\x0e\n\x07\x04\x01\x04\0\x02\x01\
+    \x02\x12\x03\x1b\x1a\x1b\n\r\n\x06\x04\x01\x04\0\x02\x02\x12\x03\x1c\x10\
+    \x20\n\x0e\n\x07\x04\x01\x04\0\x02\x02\x01\x12\x03\x1c\x10\x1b\n\x0e\n\
+    \x07\x04\x01\x04\0\x02\x02\x02\x12\x03\x1c\x1e\x1f\n\x0b\n\x04\x04\x01\
+    \x02\0\x12\x03\x1e\x08!\n\r\n\x05\x04\x01\x02\0\x04\x12\x04\x1e\x08\x1d\
+    \t\n\x0c\n\x05\x04\x01\x02\0\x06\x12\x03\x1e\x08\x15\n\x0c\n\x05\x04\x01\
+    \x02\0\x01\x12\x03\x1e\x16\x1c\n\x0c\n\x05\x04\x01\x02\0\x03\x12\x03\x1e\
+    \x1f\x20\n\n\n\x02\x04\x02\x12\x04!\0$\x01\n\n\n\x03\x04\x02\x01\x12\x03\
+    !\x08\x1c\n\x0b\n\x04\x04\x02\x02\0\x12\x03\"\x08\x20\n\r\n\x05\x04\x02\
+    \x02\0\x04\x12\x04\"\x08!\x1e\n\x0c\n\x05\x04\x02\x02\0\x05\x12\x03\"\
+    \x08\x0e\n\x0c\n\x05\x04\x02\x02\0\x01\x12\x03\"\x0f\x1b\n\x0c\n\x05\x04\
+    \x02\x02\0\x03\x12\x03\"\x1e\x1f\n\x0b\n\x04\x04\x02\x02\x01\x12\x03#\
+    \x08!\n\r\n\x05\x04\x02\x02\x01\x04\x12\x04#\x08\"\x20\n\x0c\n\x05\x04\
+    \x02\x02\x01\x05\x12\x03#\x08\x0e\n\x0c\n\x05\x04\x02\x02\x01\x01\x12\
+    \x03#\x0f\x1c\n\x0c\n\x05\x04\x02\x02\x01\x03\x12\x03#\x1f\x20\n\n\n\x02\
+    \x06\0\x12\x04&\0)\x01\n\n\n\x03\x06\0\x01\x12\x03&\x08\x0e\n\x0b\n\x04\
+    \x06\0\x02\0\x12\x03'\x08>\n\x0c\n\x05\x06\0\x02\0\x01\x12\x03'\x0c\x11\
+    \n\x0c\n\x05\x06\0\x02\0\x02\x12\x03'\x12\x1e\n\x0c\n\x05\x06\0\x02\0\
+    \x03\x12\x03')<\n\x0b\n\x04\x06\0\x02\x01\x12\x03(\x08A\n\x0c\n\x05\x06\
+    \0\x02\x01\x01\x12\x03(\x0c\x13\n\x0c\n\x05\x06\0\x02\x01\x02\x12\x03(\
+    \x14\x20\n\x0c\n\x05\x06\0\x02\x01\x03\x12\x03(+?b\x06proto3\
+";
+
+static mut file_descriptor_proto_lazy: ::protobuf::lazy::Lazy<::protobuf::descriptor::FileDescriptorProto> = ::protobuf::lazy::Lazy::INIT;
+
+fn parse_descriptor_proto() -> ::protobuf::descriptor::FileDescriptorProto {
+    ::protobuf::parse_from_bytes(file_descriptor_proto_data).unwrap()
+}
+
+pub fn file_descriptor_proto() -> &'static ::protobuf::descriptor::FileDescriptorProto {
+    unsafe {
+        file_descriptor_proto_lazy.get(|| {
+            parse_descriptor_proto()
+        })
+    }
+}

src/agent/protocols/src/health_ttrpc.rs

@@ -0,0 +1,90 @@
+// This file is generated by ttrpc-compiler 0.3.0. Do not edit
+// @generated
+
+// https://github.com/Manishearth/rust-clippy/issues/702
+#![allow(unknown_lints)]
+#![allow(clipto_camel_casepy)]
+
+#![cfg_attr(rustfmt, rustfmt_skip)]
+
+#![allow(box_pointers)]
+#![allow(dead_code)]
+#![allow(missing_docs)]
+#![allow(non_camel_case_types)]
+#![allow(non_snake_case)]
+#![allow(non_upper_case_globals)]
+#![allow(trivial_casts)]
+#![allow(unsafe_code)]
+#![allow(unused_imports)]
+#![allow(unused_results)]
+use protobuf::{CodedInputStream, CodedOutputStream, Message};
+use std::collections::HashMap;
+use std::sync::Arc;
+
+#[derive(Clone)]
+pub struct HealthClient {
+    client: ::ttrpc::Client,
+}
+
+impl HealthClient {
+    pub fn new(client: ::ttrpc::Client) -> Self {
+        HealthClient {
+            client: client,
+        }
+    }
+
+    pub fn check(&self, req: &super::health::CheckRequest, timeout_nano: i64) -> ::ttrpc::Result<super::health::HealthCheckResponse> {
+        let mut cres = super::health::HealthCheckResponse::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.Health", "Check", cres);
+        Ok(cres)
+    }
+
+    pub fn version(&self, req: &super::health::CheckRequest, timeout_nano: i64) -> ::ttrpc::Result<super::health::VersionCheckResponse> {
+        let mut cres = super::health::VersionCheckResponse::new();
+        ::ttrpc::client_request!(self, req, timeout_nano, "grpc.Health", "Version", cres);
+        Ok(cres)
+    }
+}
+
+struct CheckMethod {
+    service: Arc<std::boxed::Box<dyn Health + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for CheckMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, health, CheckRequest, check);
+        Ok(())
+    }
+}
+
+struct VersionMethod {
+    service: Arc<std::boxed::Box<dyn Health + Send + Sync>>,
+}
+
+impl ::ttrpc::MethodHandler for VersionMethod {
+    fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {
+        ::ttrpc::request_handler!(self, ctx, req, health, CheckRequest, version);
+        Ok(())
+    }
+}
+
+pub trait Health {
+    fn check(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::health::CheckRequest) -> ::ttrpc::Result<super::health::HealthCheckResponse> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.Health/Check is not supported".to_string())))
+    }
+    fn version(&self, _ctx: &::ttrpc::TtrpcContext, _req: super::health::CheckRequest) -> ::ttrpc::Result<super::health::VersionCheckResponse> {
+        Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.Health/Version is not supported".to_string())))
+    }
+}
+
+pub fn create_health(service: Arc<std::boxed::Box<dyn Health + Send + Sync>>) -> HashMap <String, Box<dyn ::ttrpc::MethodHandler + Send + Sync>> {
+    let mut methods = HashMap::new();
+
+    methods.insert("/grpc.Health/Check".to_string(),
+                    std::boxed::Box::new(CheckMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods.insert("/grpc.Health/Version".to_string(),
+                    std::boxed::Box::new(VersionMethod{service: service.clone()}) as std::boxed::Box<dyn ::ttrpc::MethodHandler + Send + Sync>);
+
+    methods
+}

src/agent/protocols/src/lib.rs

@@ -3,7 +3,6 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 #![allow(bare_trait_objects)]
-#![allow(clippy::redundant_field_names)]
 
 pub mod agent;
 pub mod agent_ttrpc;
@@ -12,3 +11,11 @@ pub mod health;
 pub mod health_ttrpc;
 pub mod oci;
 pub mod types;
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn it_works() {
+        assert_eq!(2 + 2, 4);
+    }
+}

src/agent/rustjail/Cargo.toml

@@ -10,27 +10,22 @@ serde_json = "1.0.39"
 serde_derive = "1.0.91"
 oci = { path = "../oci" }
 protocols = { path ="../protocols" }
-caps = "0.5.0"
+caps = "0.3.0"
 nix = "0.17.0"
 scopeguard = "1.0.0"
 prctl = "1.0.0"
 lazy_static = "1.3.0"
 libc = "0.2.58"
-protobuf = "=2.14.0"
+protobuf = "2.8.1"
 slog = "2.5.2"
 slog-scope = "4.1.2"
 scan_fmt = "0.2"
 regex = "1.1"
 path-absolutize = "1.2.0"
+dirs = "3.0.1"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.2.5" }
+cgroups = { git = "https://github.com/kata-containers/cgroups-rs", branch = "stable-0.1.1"}
 tempfile = "3.1.0"
-rlimit = "0.5.3"
-
-tokio = { version = "1.2.0", features = ["sync", "io-util", "process", "time", "macros"] }
-futures = "0.3"
-async-trait = "0.1.31"
-inotify = "0.9.2"
 
 [dev-dependencies]
 serial_test = "0.5.0"

src/agent/rustjail/src/capabilities.rs

@@ -6,47 +6,102 @@
 // looks like we can use caps to manipulate capabilities
 // conveniently, use caps to do it directly.. maybe
 
+use lazy_static;
+
 use crate::log_child;
 use crate::sync::write_count;
 use anyhow::{anyhow, Result};
-use caps::{self, runtime, CapSet, Capability, CapsHashSet};
+use caps::{self, CapSet, Capability, CapsHashSet};
 use oci::LinuxCapabilities;
+use std::collections::HashMap;
 use std::os::unix::io::RawFd;
-use std::str::FromStr;
+
+lazy_static! {
+    pub static ref CAPSMAP: HashMap<String, Capability> = {
+        let mut m = HashMap::new();
+        m.insert("CAP_CHOWN".to_string(), Capability::CAP_CHOWN);
+        m.insert("CAP_DAC_OVERRIDE".to_string(), Capability::CAP_DAC_OVERRIDE);
+        m.insert(
+            "CAP_DAC_READ_SEARCH".to_string(),
+            Capability::CAP_DAC_READ_SEARCH,
+        );
+        m.insert("CAP_FOWNER".to_string(), Capability::CAP_FOWNER);
+        m.insert("CAP_FSETID".to_string(), Capability::CAP_FSETID);
+        m.insert("CAP_KILL".to_string(), Capability::CAP_KILL);
+        m.insert("CAP_SETGID".to_string(), Capability::CAP_SETGID);
+        m.insert("CAP_SETUID".to_string(), Capability::CAP_SETUID);
+        m.insert("CAP_SETPCAP".to_string(), Capability::CAP_SETPCAP);
+        m.insert(
+            "CAP_LINUX_IMMUTABLE".to_string(),
+            Capability::CAP_LINUX_IMMUTABLE,
+        );
+        m.insert(
+            "CAP_NET_BIND_SERVICE".to_string(),
+            Capability::CAP_NET_BIND_SERVICE,
+        );
+        m.insert(
+            "CAP_NET_BROADCAST".to_string(),
+            Capability::CAP_NET_BROADCAST,
+        );
+        m.insert("CAP_NET_ADMIN".to_string(), Capability::CAP_NET_ADMIN);
+        m.insert("CAP_NET_RAW".to_string(), Capability::CAP_NET_RAW);
+        m.insert("CAP_IPC_LOCK".to_string(), Capability::CAP_IPC_LOCK);
+        m.insert("CAP_IPC_OWNER".to_string(), Capability::CAP_IPC_OWNER);
+        m.insert("CAP_SYS_MODULE".to_string(), Capability::CAP_SYS_MODULE);
+        m.insert("CAP_SYS_RAWIO".to_string(), Capability::CAP_SYS_RAWIO);
+        m.insert("CAP_SYS_CHROOT".to_string(), Capability::CAP_SYS_CHROOT);
+        m.insert("CAP_SYS_PTRACE".to_string(), Capability::CAP_SYS_PTRACE);
+        m.insert("CAP_SYS_PACCT".to_string(), Capability::CAP_SYS_PACCT);
+        m.insert("CAP_SYS_ADMIN".to_string(), Capability::CAP_SYS_ADMIN);
+        m.insert("CAP_SYS_BOOT".to_string(), Capability::CAP_SYS_BOOT);
+        m.insert("CAP_SYS_NICE".to_string(), Capability::CAP_SYS_NICE);
+        m.insert("CAP_SYS_RESOURCE".to_string(), Capability::CAP_SYS_RESOURCE);
+        m.insert("CAP_SYS_TIME".to_string(), Capability::CAP_SYS_TIME);
+        m.insert(
+            "CAP_SYS_TTY_CONFIG".to_string(),
+            Capability::CAP_SYS_TTY_CONFIG,
+        );
+        m.insert("CAP_MKNOD".to_string(), Capability::CAP_MKNOD);
+        m.insert("CAP_LEASE".to_string(), Capability::CAP_LEASE);
+        m.insert("CAP_AUDIT_WRITE".to_string(), Capability::CAP_AUDIT_WRITE);
+        m.insert("CAP_AUDIT_CONTROL".to_string(), Capability::CAP_AUDIT_WRITE);
+        m.insert("CAP_SETFCAP".to_string(), Capability::CAP_SETFCAP);
+        m.insert("CAP_MAC_OVERRIDE".to_string(), Capability::CAP_MAC_OVERRIDE);
+        m.insert("CAP_SYSLOG".to_string(), Capability::CAP_SYSLOG);
+        m.insert("CAP_WAKE_ALARM".to_string(), Capability::CAP_WAKE_ALARM);
+        m.insert(
+            "CAP_BLOCK_SUSPEND".to_string(),
+            Capability::CAP_BLOCK_SUSPEND,
+        );
+        m.insert("CAP_AUDIT_READ".to_string(), Capability::CAP_AUDIT_READ);
+        m
+    };
+}
 
 fn to_capshashset(cfd_log: RawFd, caps: &[String]) -> CapsHashSet {
     let mut r = CapsHashSet::new();
 
     for cap in caps.iter() {
-        match Capability::from_str(cap) {
-            Err(_) => {
-                log_child!(cfd_log, "{} is not a cap", cap);
-                continue;
-            }
-            Ok(c) => r.insert(c),
-        };
+        let c = CAPSMAP.get(cap);
+
+        if c.is_none() {
+            log_child!(cfd_log, "{} is not a cap", cap);
+            continue;
+        }
+
+        r.insert(*c.unwrap());
     }
 
     r
 }
 
-pub fn get_all_caps() -> CapsHashSet {
-    let mut caps_set =
-        runtime::procfs_all_supported(None).unwrap_or_else(|_| runtime::thread_all_supported());
-    if caps_set.is_empty() {
-        caps_set = caps::all();
-    }
-    caps_set
-}
-
 pub fn reset_effective() -> Result<()> {
-    let all = get_all_caps();
-    caps::set(None, CapSet::Effective, &all).map_err(|e| anyhow!(e.to_string()))?;
+    caps::set(None, CapSet::Effective, caps::all()).map_err(|e| anyhow!(e.to_string()))?;
     Ok(())
 }
 
 pub fn drop_privileges(cfd_log: RawFd, caps: &LinuxCapabilities) -> Result<()> {
-    let all = get_all_caps();
+    let all = caps::all();
 
     for c in all.difference(&to_capshashset(cfd_log, caps.bounding.as_ref())) {
         caps::drop(None, CapSet::Bounding, *c).map_err(|e| anyhow!(e.to_string()))?;
@@ -55,26 +110,26 @@ pub fn drop_privileges(cfd_log: RawFd, caps: &LinuxCapabilities) -> Result<()> {
     caps::set(
         None,
         CapSet::Effective,
-        &to_capshashset(cfd_log, caps.effective.as_ref()),
+        to_capshashset(cfd_log, caps.effective.as_ref()),
     )
     .map_err(|e| anyhow!(e.to_string()))?;
     caps::set(
         None,
         CapSet::Permitted,
-        &to_capshashset(cfd_log, caps.permitted.as_ref()),
+        to_capshashset(cfd_log, caps.permitted.as_ref()),
     )
     .map_err(|e| anyhow!(e.to_string()))?;
     caps::set(
         None,
         CapSet::Inheritable,
-        &to_capshashset(cfd_log, caps.inheritable.as_ref()),
+        to_capshashset(cfd_log, caps.inheritable.as_ref()),
     )
     .map_err(|e| anyhow!(e.to_string()))?;
 
     let _ = caps::set(
         None,
         CapSet::Ambient,
-        &to_capshashset(cfd_log, caps.ambient.as_ref()),
+        to_capshashset(cfd_log, caps.ambient.as_ref()),
     )
     .map_err(|_| log_child!(cfd_log, "failed to set ambient capability"));
 

src/agent/rustjail/src/cgroups/fs/mod.rs

@@ -21,10 +21,11 @@ use cgroups::{
 use crate::cgroups::Manager as CgroupManager;
 use crate::container::DEFAULT_DEVICES;
 use anyhow::{anyhow, Context, Result};
+use lazy_static;
 use libc::{self, pid_t};
 use nix::errno::Errno;
 use oci::{
-    LinuxBlockIo, LinuxCpu, LinuxDevice, LinuxDeviceCgroup, LinuxHugepageLimit, LinuxMemory,
+    LinuxBlockIO, LinuxCPU, LinuxDevice, LinuxDeviceCgroup, LinuxHugepageLimit, LinuxMemory,
     LinuxNetwork, LinuxPids, LinuxResources,
 };
 
@@ -37,8 +38,6 @@ use std::collections::HashMap;
 use std::fs;
 use std::path::Path;
 
-const GUEST_CPUS_PATH: &str = "/sys/devices/system/cpu/online";
-
 // Convenience macro to obtain the scope logger
 macro_rules! sl {
     () => {
@@ -46,22 +45,34 @@ macro_rules! sl {
     };
 }
 
-macro_rules! get_controller_or_return_singular_none {
-    ($cg:ident) => {
-        match $cg.controller_of() {
-            Some(c) => c,
-            None => return SingularPtrField::none(),
-        }
-    };
+pub fn load_or_create<'a>(h: Box<&'a dyn cgroups::Hierarchy>, path: &str) -> Cgroup<'a> {
+    let valid_path = path.trim_start_matches("/").to_string();
+    let cg = load(h.clone(), &valid_path);
+    if cg.is_none() {
+        info!(sl!(), "create new cgroup: {}", &valid_path);
+        cgroups::Cgroup::new(h, valid_path.as_str())
+    } else {
+        cg.unwrap()
+    }
+}
+
+pub fn load<'a>(h: Box<&'a dyn cgroups::Hierarchy>, path: &str) -> Option<Cgroup<'a>> {
+    let valid_path = path.trim_start_matches("/").to_string();
+    let cg = cgroups::Cgroup::load(h, valid_path.as_str());
+    let cpu_controller: &CpuController = cg.controller_of().unwrap();
+    if cpu_controller.exists() {
+        Some(cg)
+    } else {
+        None
+    }
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct Manager {
     pub paths: HashMap<String, String>,
     pub mounts: HashMap<String, String>,
+    // pub rels: HashMap<String, String>,
     pub cpath: String,
-    #[serde(skip)]
-    cgroup: cgroups::Cgroup,
 }
 
 // set_resource is used to set reources by cgroup controller.
@@ -76,11 +87,17 @@ macro_rules! set_resource {
 
 impl CgroupManager for Manager {
     fn apply(&self, pid: pid_t) -> Result<()> {
-        self.cgroup.add_task(CgroupPid::from(pid as u64))?;
+        let h = cgroups::hierarchies::auto();
+        let h = Box::new(&*h);
+        let cg = load_or_create(h, &self.cpath);
+        cg.add_task(CgroupPid::from(pid as u64))?;
         Ok(())
     }
 
     fn set(&self, r: &LinuxResources, update: bool) -> Result<()> {
+        let h = cgroups::hierarchies::auto();
+        let h = Box::new(&*h);
+        let cg = load_or_create(h, &self.cpath);
         info!(
             sl!(),
             "cgroup manager set resources for container. Resources input {:?}", r
@@ -90,49 +107,53 @@ impl CgroupManager for Manager {
 
         // set cpuset and cpu reources
         if let Some(cpu) = &r.cpu {
-            set_cpu_resources(&self.cgroup, cpu)?;
+            set_cpu_resources(&cg, cpu)?;
         }
 
         // set memory resources
         if let Some(memory) = &r.memory {
-            set_memory_resources(&self.cgroup, memory, update)?;
+            set_memory_resources(&cg, memory, update)?;
         }
 
         // set pids resources
         if let Some(pids_resources) = &r.pids {
-            set_pids_resources(&self.cgroup, pids_resources)?;
+            set_pids_resources(&cg, pids_resources)?;
         }
 
         // set block_io resources
         if let Some(blkio) = &r.block_io {
-            set_block_io_resources(&self.cgroup, blkio, res);
+            set_block_io_resources(&cg, blkio, res)?;
         }
 
         // set hugepages resources
-        if !r.hugepage_limits.is_empty() {
-            set_hugepages_resources(&self.cgroup, &r.hugepage_limits, res);
+        if r.hugepage_limits.len() > 0 {
+            set_hugepages_resources(&cg, &r.hugepage_limits, res)?;
         }
 
         // set network resources
         if let Some(network) = &r.network {
-            set_network_resources(&self.cgroup, network, res);
+            set_network_resources(&cg, network, res)?;
         }
 
         // set devices resources
-        set_devices_resources(&self.cgroup, &r.devices, res);
+        set_devices_resources(&cg, &r.devices, res)?;
         info!(sl!(), "resources after processed {:?}", res);
 
         // apply resources
-        self.cgroup.apply(res)?;
+        cg.apply(res)?;
 
         Ok(())
     }
 
     fn get_stats(&self) -> Result<CgroupStats> {
-        // CpuStats
-        let cpu_usage = get_cpuacct_stats(&self.cgroup);
+        let h = cgroups::hierarchies::auto();
+        let h = Box::new(&*h);
+        let cg = load_or_create(h, &self.cpath);
 
-        let throttling_data = get_cpu_stats(&self.cgroup);
+        // CpuStats
+        let cpu_usage = get_cpuacct_stats(&cg);
+
+        let throttling_data = get_cpu_stats(&cg);
 
         let cpu_stats = SingularPtrField::some(CpuStats {
             cpu_usage,
@@ -142,17 +163,17 @@ impl CgroupManager for Manager {
         });
 
         // Memorystats
-        let memory_stats = get_memory_stats(&self.cgroup);
+        let memory_stats = get_memory_stats(&cg);
 
         // PidsStats
-        let pids_stats = get_pids_stats(&self.cgroup);
+        let pids_stats = get_pids_stats(&cg);
 
         // BlkioStats
         // note that virtiofs has no blkio stats
-        let blkio_stats = get_blkio_stats(&self.cgroup);
+        let blkio_stats = get_blkio_stats(&cg);
 
         // HugetlbStats
-        let hugetlb_stats = get_hugetlb_stats(&self.cgroup);
+        let hugetlb_stats = get_hugetlb_stats(&cg);
 
         Ok(CgroupStats {
             cpu_stats,
@@ -166,7 +187,10 @@ impl CgroupManager for Manager {
     }
 
     fn freeze(&self, state: FreezerState) -> Result<()> {
-        let freezer_controller: &FreezerController = self.cgroup.controller_of().unwrap();
+        let h = cgroups::hierarchies::auto();
+        let h = Box::new(&*h);
+        let cg = load_or_create(h, &self.cpath);
+        let freezer_controller: &FreezerController = cg.controller_of().unwrap();
         match state {
             FreezerState::Thawed => {
                 freezer_controller.thaw()?;
@@ -183,12 +207,20 @@ impl CgroupManager for Manager {
     }
 
     fn destroy(&mut self) -> Result<()> {
-        let _ = self.cgroup.delete();
+        let h = cgroups::hierarchies::auto();
+        let h = Box::new(&*h);
+        let cg = load(h, &self.cpath);
+        if cg.is_some() {
+            cg.unwrap().delete();
+        }
         Ok(())
     }
 
     fn get_pids(&self) -> Result<Vec<pid_t>> {
-        let mem_controller: &MemController = self.cgroup.controller_of().unwrap();
+        let h = cgroups::hierarchies::auto();
+        let h = Box::new(&*h);
+        let cg = load_or_create(h, &self.cpath);
+        let mem_controller: &MemController = cg.controller_of().unwrap();
         let pids = mem_controller.tasks();
         let result = pids.iter().map(|x| x.pid as i32).collect::<Vec<i32>>();
 
@@ -200,14 +232,14 @@ fn set_network_resources(
     _cg: &cgroups::Cgroup,
     network: &LinuxNetwork,
     res: &mut cgroups::Resources,
-) {
+) -> Result<()> {
     info!(sl!(), "cgroup manager set network");
 
     // set classid
     // description can be found at https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v1/net_cls.html
     let class_id = network.class_id.unwrap_or(0) as u64;
     if class_id != 0 {
-        res.network.class_id = Some(class_id);
+        res.network.class_id = class_id;
     }
 
     // set network priorities
@@ -220,14 +252,16 @@ fn set_network_resources(
         });
     }
 
+    res.network.update_values = true;
     res.network.priorities = priorities;
+    Ok(())
 }
 
 fn set_devices_resources(
     _cg: &cgroups::Cgroup,
-    device_resources: &[LinuxDeviceCgroup],
+    device_resources: &Vec<LinuxDeviceCgroup>,
     res: &mut cgroups::Resources,
-) {
+) -> Result<()> {
     info!(sl!(), "cgroup manager set devices");
     let mut devices = vec![];
 
@@ -249,15 +283,19 @@ fn set_devices_resources(
         }
     }
 
+    res.devices.update_values = true;
     res.devices.devices = devices;
+
+    Ok(())
 }
 
 fn set_hugepages_resources(
     _cg: &cgroups::Cgroup,
-    hugepage_limits: &[LinuxHugepageLimit],
+    hugepage_limits: &Vec<LinuxHugepageLimit>,
     res: &mut cgroups::Resources,
-) {
+) -> Result<()> {
     info!(sl!(), "cgroup manager set hugepage");
+    res.hugepages.update_values = true;
     let mut limits = vec![];
 
     for l in hugepage_limits.iter() {
@@ -268,25 +306,42 @@ fn set_hugepages_resources(
         limits.push(hr);
     }
     res.hugepages.limits = limits;
+
+    Ok(())
 }
 
 fn set_block_io_resources(
-    _cg: &cgroups::Cgroup,
-    blkio: &LinuxBlockIo,
+    cg: &cgroups::Cgroup,
+    blkio: &LinuxBlockIO,
     res: &mut cgroups::Resources,
-) {
+) -> Result<()> {
     info!(sl!(), "cgroup manager set block io");
+    res.blkio.update_values = true;
 
-    res.blkio.weight = blkio.weight;
-    res.blkio.leaf_weight = blkio.leaf_weight;
+    if cg.v2() {
+        res.blkio.weight = convert_blk_io_to_v2_value(blkio.weight);
+        res.blkio.leaf_weight = convert_blk_io_to_v2_value(blkio.leaf_weight);
+    } else {
+        res.blkio.weight = blkio.weight;
+        res.blkio.leaf_weight = blkio.leaf_weight;
+    }
 
     let mut blk_device_resources = vec![];
     for d in blkio.weight_device.iter() {
+        let (w, lw) = if cg.v2() {
+            (
+                convert_blk_io_to_v2_value(blkio.weight),
+                convert_blk_io_to_v2_value(blkio.leaf_weight),
+            )
+        } else {
+            (blkio.weight, blkio.leaf_weight)
+        };
+
         let dr = BlkIoDeviceResource {
             major: d.blk.major as u64,
             minor: d.blk.minor as u64,
-            weight: blkio.weight,
-            leaf_weight: blkio.leaf_weight,
+            weight: w,
+            leaf_weight: lw,
         };
         blk_device_resources.push(dr);
     }
@@ -300,17 +355,17 @@ fn set_block_io_resources(
         build_blk_io_device_throttle_resource(&blkio.throttle_read_iops_device);
     res.blkio.throttle_write_iops_device =
         build_blk_io_device_throttle_resource(&blkio.throttle_write_iops_device);
+
+    Ok(())
 }
 
-fn set_cpu_resources(cg: &cgroups::Cgroup, cpu: &LinuxCpu) -> Result<()> {
+fn set_cpu_resources(cg: &cgroups::Cgroup, cpu: &LinuxCPU) -> Result<()> {
     info!(sl!(), "cgroup manager set cpu");
 
     let cpuset_controller: &CpuSetController = cg.controller_of().unwrap();
 
     if !cpu.cpus.is_empty() {
-        if let Err(e) = cpuset_controller.set_cpus(&cpu.cpus) {
-            warn!(sl!(), "write cpuset failed: {:?}", e);
-        }
+        cpuset_controller.set_cpus(&cpu.cpus)?;
     }
 
     if !cpu.mems.is_empty() {
@@ -366,13 +421,13 @@ fn set_memory_resources(cg: &cgroups::Cgroup, memory: &LinuxMemory, update: bool
         }
     }
 
-    if let Some(swappiness) = memory.swappiness {
-        if (0..=100).contains(&swappiness) {
-            mem_controller.set_swappiness(swappiness as u64)?;
+    if let Some(swapiness) = memory.swapiness {
+        if swapiness >= 0 && swapiness <= 100 {
+            mem_controller.set_swappiness(swapiness as u64)?;
         } else {
             return Err(anyhow!(
                 "invalid value:{}. valid memory swappiness range is 0-100",
-                swappiness
+                swapiness
             ));
         }
     }
@@ -398,7 +453,7 @@ fn set_pids_resources(cg: &cgroups::Cgroup, pids: &LinuxPids) -> Result<()> {
 }
 
 fn build_blk_io_device_throttle_resource(
-    input: &[oci::LinuxThrottleDevice],
+    input: &Vec<oci::LinuxThrottleDevice>,
 ) -> Vec<BlkIoDeviceThrottleResource> {
     let mut blk_io_device_throttle_resources = vec![];
     for d in input.iter() {
@@ -489,67 +544,71 @@ lazy_static! {
     };
 
     pub static ref DEFAULT_ALLOWED_DEVICES: Vec<LinuxDeviceCgroup> = {
-        vec![
-            // all mknod to all char devices
-            LinuxDeviceCgroup {
-                allow: true,
-                r#type: "c".to_string(),
-                major: Some(WILDCARD),
-                minor: Some(WILDCARD),
-                access: "m".to_string(),
-            },
+        let mut v = Vec::new();
 
-            // all mknod to all block devices
-            LinuxDeviceCgroup {
-                allow: true,
-                r#type: "b".to_string(),
-                major: Some(WILDCARD),
-                minor: Some(WILDCARD),
-                access: "m".to_string(),
-            },
+        // all mknod to all char devices
+        v.push(LinuxDeviceCgroup {
+            allow: true,
+            r#type: "c".to_string(),
+            major: Some(WILDCARD),
+            minor: Some(WILDCARD),
+            access: "m".to_string(),
+        });
 
-            // all read/write/mknod to char device /dev/console
-            LinuxDeviceCgroup {
-                allow: true,
-                r#type: "c".to_string(),
-                major: Some(5),
-                minor: Some(1),
-                access: "rwm".to_string(),
-            },
+        // all mknod to all block devices
+        v.push(LinuxDeviceCgroup {
+            allow: true,
+            r#type: "b".to_string(),
+            major: Some(WILDCARD),
+            minor: Some(WILDCARD),
+            access: "m".to_string(),
+        });
 
-            // all read/write/mknod to char device /dev/pts/<N>
-            LinuxDeviceCgroup {
-                allow: true,
-                r#type: "c".to_string(),
-                major: Some(136),
-                minor: Some(WILDCARD),
-                access: "rwm".to_string(),
-            },
+        // all read/write/mknod to char device /dev/console
+        v.push(LinuxDeviceCgroup {
+            allow: true,
+            r#type: "c".to_string(),
+            major: Some(5),
+            minor: Some(1),
+            access: "rwm".to_string(),
+        });
 
-            // all read/write/mknod to char device /dev/ptmx
-            LinuxDeviceCgroup {
-                allow: true,
-                r#type: "c".to_string(),
-                major: Some(5),
-                minor: Some(2),
-                access: "rwm".to_string(),
-            },
+        // all read/write/mknod to char device /dev/pts/<N>
+        v.push(LinuxDeviceCgroup {
+            allow: true,
+            r#type: "c".to_string(),
+            major: Some(136),
+            minor: Some(WILDCARD),
+            access: "rwm".to_string(),
+        });
 
-            // all read/write/mknod to char device /dev/net/tun
-            LinuxDeviceCgroup {
-                allow: true,
-                r#type: "c".to_string(),
-                major: Some(10),
-                minor: Some(200),
-                access: "rwm".to_string(),
-            },
-        ]
+        // all read/write/mknod to char device /dev/ptmx
+        v.push(LinuxDeviceCgroup {
+            allow: true,
+            r#type: "c".to_string(),
+            major: Some(5),
+            minor: Some(2),
+            access: "rwm".to_string(),
+        });
+
+        // all read/write/mknod to char device /dev/net/tun
+        v.push(LinuxDeviceCgroup {
+            allow: true,
+            r#type: "c".to_string(),
+            major: Some(10),
+            minor: Some(200),
+            access: "rwm".to_string(),
+        });
+
+        v
     };
 }
 
 fn get_cpu_stats(cg: &cgroups::Cgroup) -> SingularPtrField<ThrottlingData> {
-    let cpu_controller: &CpuController = get_controller_or_return_singular_none!(cg);
+    let cpu_controller: &CpuController = cg.controller_of().unwrap();
+
     let stat = cpu_controller.cpu().stat;
+
     let h = lines_to_map(&stat);
 
     SingularPtrField::some(ThrottlingData {
@@ -562,18 +621,27 @@ fn get_cpu_stats(cg: &cgroups::Cgroup) -> SingularPtrField<ThrottlingData> {
 }
 
 fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> SingularPtrField<CpuUsage> {
-    if let Some(cpuacct_controller) = cg.controller_of::<CpuAcctController>() {
-        let cpuacct = cpuacct_controller.cpuacct();
+    let cpuacct_controller: Option<&CpuAcctController> = cg.controller_of();
+    if cpuacct_controller.is_none() {
+        if cg.v2() {
+            return SingularPtrField::some(CpuUsage {
+                total_usage: 0,
+                percpu_usage: vec![],
+                usage_in_kernelmode: 0,
+                usage_in_usermode: 0,
+                unknown_fields: UnknownFields::default(),
+                cached_size: CachedSize::default(),
+            });
+        }
 
-        let h = lines_to_map(&cpuacct.stat);
-        let usage_in_usermode =
-            (((*h.get("user").unwrap() * NANO_PER_SECOND) as f64) / *CLOCK_TICKS) as u64;
-        let usage_in_kernelmode =
-            (((*h.get("system").unwrap() * NANO_PER_SECOND) as f64) / *CLOCK_TICKS) as u64;
-
-        let total_usage = cpuacct.usage;
-
-        let percpu_usage = line_to_vec(&cpuacct.usage_percpu);
+        // try to get from cpu controller
+        let cpu_controller: &CpuController = cg.controller_of().unwrap();
+        let stat = cpu_controller.cpu().stat;
+        let h = lines_to_map(&stat);
+        let usage_in_usermode = *h.get("user_usec").unwrap();
+        let usage_in_kernelmode = *h.get("system_usec").unwrap();
+        let total_usage = *h.get("usage_usec").unwrap();
+        let percpu_usage = vec![];
 
         return SingularPtrField::some(CpuUsage {
             total_usage,
@@ -585,25 +653,18 @@ fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> SingularPtrField<CpuUsage> {
         });
     }
 
-    if cg.v2() {
-        return SingularPtrField::some(CpuUsage {
-            total_usage: 0,
-            percpu_usage: vec![],
-            usage_in_kernelmode: 0,
-            usage_in_usermode: 0,
-            unknown_fields: UnknownFields::default(),
-            cached_size: CachedSize::default(),
-        });
-    }
+    let cpuacct_controller = cpuacct_controller.unwrap();
+    let cpuacct = cpuacct_controller.cpuacct();
 
-    // try to get from cpu controller
-    let cpu_controller: &CpuController = get_controller_or_return_singular_none!(cg);
-    let stat = cpu_controller.cpu().stat;
-    let h = lines_to_map(&stat);
-    let usage_in_usermode = *h.get("user_usec").unwrap();
-    let usage_in_kernelmode = *h.get("system_usec").unwrap();
-    let total_usage = *h.get("usage_usec").unwrap();
-    let percpu_usage = vec![];
+    let h = lines_to_map(&cpuacct.stat);
+    let usage_in_usermode =
+        (((*h.get("user").unwrap() * NANO_PER_SECOND) as f64) / *CLOCK_TICKS) as u64;
+    let usage_in_kernelmode =
+        (((*h.get("system").unwrap() * NANO_PER_SECOND) as f64) / *CLOCK_TICKS) as u64;
+
+    let total_usage = cpuacct.usage;
+
+    let percpu_usage = line_to_vec(&cpuacct.usage_percpu);
 
     SingularPtrField::some(CpuUsage {
         total_usage,
@@ -616,7 +677,7 @@ fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> SingularPtrField<CpuUsage> {
 }
 
 fn get_memory_stats(cg: &cgroups::Cgroup) -> SingularPtrField<MemoryStats> {
-    let memory_controller: &MemController = get_controller_or_return_singular_none!(cg);
+    let memory_controller: &MemController = cg.controller_of().unwrap();
 
     // cache from memory stat
     let memory = memory_controller.memory_stat();
@@ -624,7 +685,7 @@ fn get_memory_stats(cg: &cgroups::Cgroup) -> SingularPtrField<MemoryStats> {
 
     // use_hierarchy
     let value = memory.use_hierarchy;
-    let use_hierarchy = value == 1;
+    let use_hierarchy = if value == 1 { true } else { false };
 
     // gte memory datas
     let usage = SingularPtrField::some(MemoryData {
@@ -673,17 +734,18 @@ fn get_memory_stats(cg: &cgroups::Cgroup) -> SingularPtrField<MemoryStats> {
 }
 
 fn get_pids_stats(cg: &cgroups::Cgroup) -> SingularPtrField<PidsStats> {
-    let pid_controller: &PidController = get_controller_or_return_singular_none!(cg);
+    let pid_controller: &PidController = cg.controller_of().unwrap();
 
     let current = pid_controller.get_pid_current().unwrap_or(0);
     let max = pid_controller.get_pid_max();
 
-    let limit = match max {
-        Err(_) => 0,
-        Ok(max) => match max {
+    let limit = if max.is_err() {
+        0
+    } else {
+        match max.unwrap() {
             MaxValue::Value(v) => v,
             MaxValue::Max => 0,
-        },
+        }
     } as u64;
 
     SingularPtrField::some(PidsStats {
@@ -726,9 +788,9 @@ https://github.com/opencontainers/runc/blob/a5847db387ae28c0ca4ebe4beee1a76900c8
     Total 0
 */
 
-fn get_blkio_stat_blkiodata(blkiodata: &[BlkIoData]) -> RepeatedField<BlkioStatsEntry> {
+fn get_blkio_stat_blkiodata(blkiodata: &Vec<BlkIoData>) -> RepeatedField<BlkioStatsEntry> {
     let mut m = RepeatedField::new();
-    if blkiodata.is_empty() {
+    if blkiodata.len() == 0 {
         return m;
     }
 
@@ -748,10 +810,10 @@ fn get_blkio_stat_blkiodata(blkiodata: &[BlkIoData]) -> RepeatedField<BlkioStats
     m
 }
 
-fn get_blkio_stat_ioservice(services: &[IoService]) -> RepeatedField<BlkioStatsEntry> {
+fn get_blkio_stat_ioservice(services: &Vec<IoService>) -> RepeatedField<BlkioStatsEntry> {
     let mut m = RepeatedField::new();
 
-    if services.is_empty() {
+    if services.len() == 0 {
         return m;
     }
 
@@ -772,14 +834,14 @@ fn build_blkio_stats_entry(major: i16, minor: i16, op: &str, value: u64) -> Blki
         major: major as u64,
         minor: minor as u64,
         op: op.to_string(),
-        value,
+        value: value,
         unknown_fields: UnknownFields::default(),
         cached_size: CachedSize::default(),
     }
 }
 
 fn get_blkio_stats_v2(cg: &cgroups::Cgroup) -> SingularPtrField<BlkioStats> {
-    let blkio_controller: &BlkIoController = get_controller_or_return_singular_none!(cg);
+    let blkio_controller: &BlkIoController = cg.controller_of().unwrap();
     let blkio = blkio_controller.blkio();
 
     let mut resp = BlkioStats::new();
@@ -807,13 +869,13 @@ fn get_blkio_stats(cg: &cgroups::Cgroup) -> SingularPtrField<BlkioStats> {
         return get_blkio_stats_v2(&cg);
     }
 
-    let blkio_controller: &BlkIoController = get_controller_or_return_singular_none!(cg);
+    let blkio_controller: &BlkIoController = cg.controller_of().unwrap();
     let blkio = blkio_controller.blkio();
 
     let mut m = BlkioStats::new();
     let io_serviced_recursive = blkio.io_serviced_recursive;
 
-    if io_serviced_recursive.is_empty() {
+    if io_serviced_recursive.len() == 0 {
         // fall back to generic stats
         // blkio.throttle.io_service_bytes,
         // maybe io_service_bytes_recursive?
@@ -868,8 +930,8 @@ fn get_hugetlb_stats(cg: &cgroups::Cgroup) -> HashMap<String, HugetlbStats> {
     h
 }
 
-pub const PATHS: &str = "/proc/self/cgroup";
-pub const MOUNTS: &str = "/proc/self/mountinfo";
+pub const PATHS: &'static str = "/proc/self/cgroup";
+pub const MOUNTS: &'static str = "/proc/self/mountinfo";
 
 pub fn get_paths() -> Result<HashMap<String, String>> {
     let mut m = HashMap::new();
@@ -924,11 +986,6 @@ pub fn get_mounts() -> Result<HashMap<String, String>> {
     Ok(m)
 }
 
-fn new_cgroup(h: Box<dyn cgroups::Hierarchy>, path: &str) -> Cgroup {
-    let valid_path = path.trim_start_matches('/').to_string();
-    cgroups::Cgroup::new(h, valid_path.as_str())
-}
-
 impl Manager {
     pub fn new(cpath: &str) -> Result<Self> {
         let mut m = HashMap::new();
@@ -936,14 +993,18 @@ impl Manager {
         let paths = get_paths()?;
         let mounts = get_mounts()?;
 
-        for key in paths.keys() {
+        for (key, value) in &paths {
             let mnt = mounts.get(key);
 
             if mnt.is_none() {
                 continue;
             }
 
-            let p = format!("{}/{}", mnt.unwrap(), cpath);
+            let p = if value == "/" {
+                format!("{}/{}", mnt.unwrap(), cpath)
+            } else {
+                format!("{}{}/{}", mnt.unwrap(), value, cpath)
+            };
 
             m.insert(key.to_string(), p);
         }
@@ -953,26 +1014,29 @@ impl Manager {
             mounts,
             // rels: paths,
             cpath: cpath.to_string(),
-            cgroup: new_cgroup(cgroups::hierarchies::auto(), cpath),
         })
     }
 
-    pub fn update_cpuset_path(&self, guest_cpuset: &str, container_cpuset: &str) -> Result<()> {
-        if guest_cpuset.is_empty() {
+    pub fn update_cpuset_path(&self, cpuset_cpus: &str) -> Result<()> {
+        if cpuset_cpus == "" {
             return Ok(());
         }
-        info!(sl!(), "update_cpuset_path to: {}", guest_cpuset);
+        info!(sl!(), "update_cpuset_path to: {}", cpuset_cpus);
 
         let h = cgroups::hierarchies::auto();
-        let root_cg = h.root_control_group();
+        let h = Box::new(&*h);
+        let root_cg = load_or_create(h, "");
 
         let root_cpuset_controller: &CpuSetController = root_cg.controller_of().unwrap();
         let path = root_cpuset_controller.path();
         let root_path = Path::new(path);
         info!(sl!(), "root cpuset path: {:?}", &path);
 
-        let container_cpuset_controller: &CpuSetController = self.cgroup.controller_of().unwrap();
-        let path = container_cpuset_controller.path();
+        let h = cgroups::hierarchies::auto();
+        let h = Box::new(&*h);
+        let cg = load_or_create(h, &self.cpath);
+        let cpuset_controller: &CpuSetController = cg.controller_of().unwrap();
+        let path = cpuset_controller.path();
         let container_path = Path::new(path);
         info!(sl!(), "container cpuset path: {:?}", &path);
 
@@ -981,36 +1045,30 @@ impl Manager {
             if ancestor == root_path {
                 break;
             }
-            paths.push(ancestor);
+            if ancestor != container_path {
+                paths.push(ancestor);
+            }
         }
-        info!(sl!(), "parent paths to update cpuset: {:?}", &paths);
+        info!(sl!(), "paths to update cpuset: {:?}", &paths);
 
         let mut i = paths.len();
         loop {
             if i == 0 {
                 break;
             }
-            i -= 1;
+            i = i - 1;
+            let h = cgroups::hierarchies::auto();
+            let h = Box::new(&*h);
 
             // remove cgroup root from path
             let r_path = &paths[i]
                 .to_str()
                 .unwrap()
                 .trim_start_matches(root_path.to_str().unwrap());
-            info!(sl!(), "updating cpuset for parent path {:?}", &r_path);
-            let cg = new_cgroup(cgroups::hierarchies::auto(), &r_path);
+            info!(sl!(), "updating cpuset for path {:?}", &r_path);
+            let cg = load_or_create(h, &r_path);
             let cpuset_controller: &CpuSetController = cg.controller_of().unwrap();
-            cpuset_controller.set_cpus(guest_cpuset)?;
-        }
-
-        if !container_cpuset.is_empty() {
-            info!(
-                sl!(),
-                "updating cpuset for container path: {:?} cpuset: {}",
-                &container_path,
-                container_cpuset
-            );
-            container_cpuset_controller.set_cpus(container_cpuset)?;
+            cpuset_controller.set_cpus(cpuset_cpus)?;
         }
 
         Ok(())
@@ -1027,10 +1085,23 @@ impl Manager {
     }
 }
 
-// get the guest's online cpus.
 pub fn get_guest_cpuset() -> Result<String> {
-    let c = fs::read_to_string(GUEST_CPUS_PATH)?;
-    Ok(c.trim().to_string())
+    // for cgroup v2
+    if cgroups::hierarchies::is_cgroup2_unified_mode() {
+        let c = fs::read_to_string("/sys/fs/cgroup/cpuset.cpus.effective")?;
+        return Ok(c);
+    }
+
+    // for cgroup v1
+    let m = get_mounts()?;
+    if m.get("cpuset").is_none() {
+        warn!(sl!(), "no cpuset cgroup!");
+        return Err(nix::Error::Sys(Errno::ENOENT).into());
+    }
+
+    let p = format!("{}/cpuset.cpus", m.get("cpuset").unwrap());
+    let c = fs::read_to_string(p.as_str())?;
+    Ok(c)
 }
 
 // Since the OCI spec is designed for cgroup v1, in some cases
@@ -1073,6 +1144,20 @@ fn convert_memory_swap_to_v2_value(memory_swap: i64, memory: i64) -> Result<i64>
     Ok(memory_swap - memory)
 }
 
+// Since the OCI spec is designed for cgroup v1, in some cases
+// there is need to convert from the cgroup v1 configuration to cgroup v2
+// the formula for BlkIOWeight is y = (1 + (x - 10) * 9999 / 990)
+// convert linearly from [10-1000] to [1-10000]
+// https://github.com/opencontainers/runc/blob/a5847db387ae28c0ca4ebe4beee1a76900c86414/libcontainer/cgroups/utils.go#L382
+fn convert_blk_io_to_v2_value(blk_io_weight: Option<u16>) -> Option<u16> {
+    let v = blk_io_weight.unwrap_or(0);
+    if v != 0 {
+        return None;
+    }
+
+    Some(1 + (v - 10) * 9999 / 990 as u16)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/agent/rustjail/src/cgroups/mock.rs

@@ -1,74 +0,0 @@
-// Copyright (c) 2020 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-use protobuf::{CachedSize, SingularPtrField, UnknownFields};
-
-use crate::cgroups::Manager as CgroupManager;
-use crate::protocols::agent::{BlkioStats, CgroupStats, CpuStats, MemoryStats, PidsStats};
-use anyhow::Result;
-use cgroups::freezer::FreezerState;
-use libc::{self, pid_t};
-use oci::LinuxResources;
-use std::collections::HashMap;
-use std::string::String;
-
-#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct Manager {
-    pub paths: HashMap<String, String>,
-    pub mounts: HashMap<String, String>,
-    pub cpath: String,
-}
-
-impl CgroupManager for Manager {
-    fn apply(&self, _: pid_t) -> Result<()> {
-        Ok(())
-    }
-
-    fn set(&self, _: &LinuxResources, _: bool) -> Result<()> {
-        Ok(())
-    }
-
-    fn get_stats(&self) -> Result<CgroupStats> {
-        Ok(CgroupStats {
-            cpu_stats: SingularPtrField::some(CpuStats::default()),
-            memory_stats: SingularPtrField::some(MemoryStats::new()),
-            pids_stats: SingularPtrField::some(PidsStats::new()),
-            blkio_stats: SingularPtrField::some(BlkioStats::new()),
-            hugetlb_stats: HashMap::new(),
-            unknown_fields: UnknownFields::default(),
-            cached_size: CachedSize::default(),
-        })
-    }
-
-    fn freeze(&self, _: FreezerState) -> Result<()> {
-        Ok(())
-    }
-
-    fn destroy(&mut self) -> Result<()> {
-        Ok(())
-    }
-
-    fn get_pids(&self) -> Result<Vec<pid_t>> {
-        Ok(Vec::new())
-    }
-}
-
-impl Manager {
-    pub fn new(cpath: &str) -> Result<Self> {
-        Ok(Self {
-            paths: HashMap::new(),
-            mounts: HashMap::new(),
-            cpath: cpath.to_string(),
-        })
-    }
-
-    pub fn update_cpuset_path(&self, _: &str, _: &str) -> Result<()> {
-        Ok(())
-    }
-
-    pub fn get_cg_path(&self, _: &str) -> Option<String> {
-        Some("".to_string())
-    }
-}

src/agent/rustjail/src/cgroups/mod.rs

@@ -10,7 +10,6 @@ use protocols::agent::CgroupStats;
 use cgroups::freezer::FreezerState;
 
 pub mod fs;
-pub mod mock;
 pub mod notifier;
 pub mod systemd;
 

src/agent/rustjail/src/cgroups/notifier.rs

@@ -3,18 +3,16 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use anyhow::{anyhow, Context, Result};
+use anyhow::{anyhow, Result};
 use eventfd::{eventfd, EfdFlags};
 use nix::sys::eventfd;
+use nix::sys::inotify::{AddWatchFlags, InitFlags, Inotify};
 use std::fs::{self, File};
+use std::io::Read;
 use std::os::unix::io::{AsRawFd, FromRawFd};
-use std::path::Path;
-
-use crate::pipestream::PipeStream;
-use futures::StreamExt as _;
-use inotify::{Inotify, WatchMask};
-use tokio::io::AsyncReadExt;
-use tokio::sync::mpsc::{channel, Receiver};
+use std::path::{Path, PathBuf};
+use std::sync::mpsc::{self, Receiver};
+use std::thread;
 
 // Convenience macro to obtain the scope logger
 macro_rules! sl {
@@ -23,11 +21,11 @@ macro_rules! sl {
     };
 }
 
-pub async fn notify_oom(cid: &str, cg_dir: String) -> Result<Receiver<String>> {
+pub fn notify_oom(cid: &str, cg_dir: String) -> Result<Receiver<String>> {
     if cgroups::hierarchies::is_cgroup2_unified_mode() {
-        return notify_on_oom_v2(cid, cg_dir).await;
+        return notify_on_oom_v2(cid, cg_dir);
     }
-    notify_on_oom(cid, cg_dir).await
+    notify_on_oom(cid, cg_dir)
 }
 
 // get_value_from_cgroup parse cgroup file with `Flat keyed`
@@ -35,7 +33,7 @@ pub async fn notify_oom(cid: &str, cg_dir: String) -> Result<Receiver<String>> {
 // Flat keyed file format:
 //   KEY0 VAL0\n
 //   KEY1 VAL1\n
-fn get_value_from_cgroup(path: &Path, key: &str) -> Result<i64> {
+fn get_value_from_cgroup(path: &PathBuf, key: &str) -> Result<i64> {
     let content = fs::read_to_string(path)?;
     info!(
         sl!(),
@@ -43,7 +41,7 @@ fn get_value_from_cgroup(path: &Path, key: &str) -> Result<i64> {
     );
 
     for line in content.lines() {
-        let arr: Vec<&str> = line.split(' ').collect();
+        let arr: Vec<&str> = line.split(" ").collect();
         if arr.len() == 2 && arr[0] == key {
             let r = arr[1].parse::<i64>()?;
             return Ok(r);
@@ -54,11 +52,11 @@ fn get_value_from_cgroup(path: &Path, key: &str) -> Result<i64> {
 
 // notify_on_oom returns channel on which you can expect event about OOM,
 // if process died without OOM this channel will be closed.
-pub async fn notify_on_oom_v2(containere_id: &str, cg_dir: String) -> Result<Receiver<String>> {
-    register_memory_event_v2(containere_id, cg_dir, "memory.events", "cgroup.events").await
+pub fn notify_on_oom_v2(containere_id: &str, cg_dir: String) -> Result<Receiver<String>> {
+    register_memory_event_v2(containere_id, cg_dir, "memory.events", "cgroup.events")
 }
 
-async fn register_memory_event_v2(
+fn register_memory_event_v2(
     containere_id: &str,
     cg_dir: String,
     memory_event_name: &str,
@@ -75,49 +73,49 @@ async fn register_memory_event_v2(
         "register_memory_event_v2 cgroup_event_control_path: {:?}", &cgroup_event_control_path
     );
 
-    let mut inotify = Inotify::init().context("Failed to initialize inotify")?;
+    let fd = Inotify::init(InitFlags::empty()).unwrap();
 
     // watching oom kill
-    let ev_wd = inotify.add_watch(&event_control_path, WatchMask::MODIFY)?;
+    let ev_fd = fd
+        .add_watch(&event_control_path, AddWatchFlags::IN_MODIFY)
+        .unwrap();
     // Because no `unix.IN_DELETE|unix.IN_DELETE_SELF` event for cgroup file system, so watching all process exited
-    let cg_wd = inotify.add_watch(&cgroup_event_control_path, WatchMask::MODIFY)?;
+    let cg_fd = fd
+        .add_watch(&cgroup_event_control_path, AddWatchFlags::IN_MODIFY)
+        .unwrap();
+    info!(sl!(), "ev_fd: {:?}", ev_fd);
+    info!(sl!(), "cg_fd: {:?}", cg_fd);
 
-    info!(sl!(), "ev_wd: {:?}", ev_wd);
-    info!(sl!(), "cg_wd: {:?}", cg_wd);
-
-    let (sender, receiver) = channel(100);
+    let (sender, receiver) = mpsc::channel();
     let containere_id = containere_id.to_string();
 
-    tokio::spawn(async move {
-        let mut buffer = [0; 32];
-        let mut stream = inotify
-            .event_stream(&mut buffer)
-            .expect("create inotify event stream failed");
-
-        while let Some(event_or_error) = stream.next().await {
-            let event = event_or_error.unwrap();
+    thread::spawn(move || {
+        loop {
+            let events = fd.read_events().unwrap();
             info!(
                 sl!(),
-                "container[{}] get event for container: {:?}", &containere_id, &event
+                "container[{}] get events for container: {:?}", &containere_id, &events
             );
-            // info!("is1: {}", event.wd == wd1);
-            info!(sl!(), "event.wd: {:?}", event.wd);
 
-            if event.wd == ev_wd {
-                let oom = get_value_from_cgroup(&event_control_path, "oom_kill");
-                if oom.unwrap_or(0) > 0 {
-                    let _ = sender.send(containere_id.clone()).await.map_err(|e| {
-                        error!(sl!(), "send containere_id failed, error: {:?}", e);
-                    });
-                    return;
+            for event in events {
+                if event.mask & AddWatchFlags::IN_MODIFY != AddWatchFlags::IN_MODIFY {
+                    continue;
                 }
-            } else if event.wd == cg_wd {
-                let pids = get_value_from_cgroup(&cgroup_event_control_path, "populated");
-                if pids.unwrap_or(-1) == 0 {
-                    return;
+                info!(sl!(), "event.wd: {:?}", event.wd);
+
+                if event.wd == ev_fd {
+                    let oom = get_value_from_cgroup(&event_control_path, "oom_kill");
+                    if oom.unwrap_or(0) > 0 {
+                        sender.send(containere_id.clone()).unwrap();
+                        return;
+                    }
+                } else if event.wd == cg_fd {
+                    let pids = get_value_from_cgroup(&cgroup_event_control_path, "populated");
+                    if pids.unwrap_or(-1) == 0 {
+                        return;
+                    }
                 }
             }
-
             // When a cgroup is destroyed, an event is sent to eventfd.
             // So if the control path is gone, return instead of notifying.
             if !Path::new(&event_control_path).exists() {
@@ -131,17 +129,17 @@ async fn register_memory_event_v2(
 
 // notify_on_oom returns channel on which you can expect event about OOM,
 // if process died without OOM this channel will be closed.
-async fn notify_on_oom(cid: &str, dir: String) -> Result<Receiver<String>> {
-    if dir.is_empty() {
+fn notify_on_oom(cid: &str, dir: String) -> Result<Receiver<String>> {
+    if dir == "" {
         return Err(anyhow!("memory controller missing"));
     }
 
-    register_memory_event(cid, dir, "memory.oom_control", "").await
+    register_memory_event(cid, dir, "memory.oom_control", "")
 }
 
 // level is one of "low", "medium", or "critical"
-async fn notify_memory_pressure(cid: &str, dir: String, level: &str) -> Result<Receiver<String>> {
-    if dir.is_empty() {
+fn notify_memory_pressure(cid: &str, dir: String, level: &str) -> Result<Receiver<String>> {
+    if dir == "" {
         return Err(anyhow!("memory controller missing"));
     }
 
@@ -149,10 +147,10 @@ async fn notify_memory_pressure(cid: &str, dir: String, level: &str) -> Result<R
         return Err(anyhow!("invalid pressure level {}", level));
     }
 
-    register_memory_event(cid, dir, "memory.pressure_level", level).await
+    register_memory_event(cid, dir, "memory.pressure_level", level)
 }
 
-async fn register_memory_event(
+fn register_memory_event(
     cid: &str,
     cg_dir: String,
     event_name: &str,
@@ -165,7 +163,7 @@ async fn register_memory_event(
 
     let event_control_path = Path::new(&cg_dir).join("cgroup.event_control");
     let data;
-    if arg.is_empty() {
+    if arg == "" {
         data = format!("{} {}", eventfd, event_file.as_raw_fd());
     } else {
         data = format!("{} {} {}", eventfd, event_file.as_raw_fd(), arg);
@@ -173,16 +171,15 @@ async fn register_memory_event(
 
     fs::write(&event_control_path, data)?;
 
-    let mut eventfd_stream = unsafe { PipeStream::from_raw_fd(eventfd) };
+    let mut eventfd_file = unsafe { File::from_raw_fd(eventfd) };
 
-    let (sender, receiver) = tokio::sync::mpsc::channel(100);
+    let (sender, receiver) = mpsc::channel();
     let containere_id = cid.to_string();
 
-    tokio::spawn(async move {
+    thread::spawn(move || {
         loop {
-            let sender = sender.clone();
-            let mut buf = [0u8; 8];
-            match eventfd_stream.read(&mut buf).await {
+            let mut buf = [0; 8];
+            match eventfd_file.read(&mut buf) {
                 Err(err) => {
                     warn!(sl!(), "failed to read from eventfd: {:?}", err);
                     return;
@@ -191,10 +188,7 @@ async fn register_memory_event(
                     let content = fs::read_to_string(path.clone());
                     info!(
                         sl!(),
-                        "cgroup event for container: {}, path: {:?}, content: {:?}",
-                        &containere_id,
-                        &path,
-                        content
+                        "OOM event for container: {}, content: {:?}", &containere_id, content
                     );
                 }
             }
@@ -204,10 +198,7 @@ async fn register_memory_event(
             if !Path::new(&event_control_path).exists() {
                 return;
             }
-
-            let _ = sender.send(containere_id.clone()).await.map_err(|e| {
-                error!(sl!(), "send containere_id failed, error: {:?}", e);
-            });
+            sender.send(containere_id.clone()).unwrap();
         }
     });
 

src/agent/rustjail/src/configs/mod.rs

@@ -54,8 +54,6 @@ pub struct Seccomp {
     #[serde(default)]
     architectures: Vec<String>,
     #[serde(default)]
-    flags: Vec<String>,
-    #[serde(default)]
     syscalls: Vec<Syscall>,
 }
 
@@ -76,11 +74,9 @@ pub struct Arg {
 #[derive(Serialize, Deserialize, Debug)]
 pub struct Syscall {
     #[serde(default, skip_serializing_if = "String::is_empty")]
-    names: String,
+    name: String,
     #[serde(default)]
     action: Action,
-    #[serde(default, rename = "errnoRet")]
-    errno_ret: u32,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     args: Vec<Arg>,
 }

src/agent/rustjail/src/lib.rs

@@ -40,13 +40,11 @@ pub mod capabilities;
 pub mod cgroups;
 pub mod container;
 pub mod mount;
-pub mod pipestream;
 pub mod process;
 pub mod specconv;
 pub mod sync;
-pub mod sync_with_async;
-pub mod utils;
 pub mod validator;
+
 // pub mod factory;
 //pub mod configs;
 // pub mod devices;
@@ -58,16 +56,24 @@ pub mod validator;
 // pub mod user;
 //pub mod intelrdt;
 
+// construtc ociSpec from grpcSpec, which is needed for hook
+// execution. since hooks read config.json
+
+use oci::{
+    Box as ociBox, Hooks as ociHooks, Linux as ociLinux, LinuxCapabilities as ociLinuxCapabilities,
+    Mount as ociMount, POSIXRlimit as ociPOSIXRlimit, Process as ociProcess, Root as ociRoot,
+    Spec as ociSpec, User as ociUser,
+};
+use protocols::oci::{
+    Hooks as grpcHooks, Linux as grpcLinux, Mount as grpcMount, Process as grpcProcess,
+    Root as grpcRoot, Spec as grpcSpec,
+};
 use std::collections::HashMap;
 
-use protocols::oci as grpc;
-
-// construct ociSpec from grpc::Spec, which is needed for hook
-// execution. since hooks read config.json
-pub fn process_grpc_to_oci(p: &grpc::Process) -> oci::Process {
+pub fn process_grpc_to_oci(p: &grpcProcess) -> ociProcess {
     let console_size = if p.ConsoleSize.is_some() {
         let c = p.ConsoleSize.as_ref().unwrap();
-        Some(oci::Box {
+        Some(ociBox {
             height: c.Height,
             width: c.Width,
         })
@@ -77,14 +83,14 @@ pub fn process_grpc_to_oci(p: &grpc::Process) -> oci::Process {
 
     let user = if p.User.is_some() {
         let u = p.User.as_ref().unwrap();
-        oci::User {
+        ociUser {
             uid: u.UID,
             gid: u.GID,
             additional_gids: u.AdditionalGids.clone(),
             username: u.Username.clone(),
         }
     } else {
-        oci::User {
+        ociUser {
             uid: 0,
             gid: 0,
             additional_gids: vec![],
@@ -95,7 +101,7 @@ pub fn process_grpc_to_oci(p: &grpc::Process) -> oci::Process {
     let capabilities = if p.Capabilities.is_some() {
         let cap = p.Capabilities.as_ref().unwrap();
 
-        Some(oci::LinuxCapabilities {
+        Some(ociLinuxCapabilities {
             bounding: cap.Bounding.clone().into_vec(),
             effective: cap.Effective.clone().into_vec(),
             inheritable: cap.Inheritable.clone().into_vec(),
@@ -109,7 +115,7 @@ pub fn process_grpc_to_oci(p: &grpc::Process) -> oci::Process {
     let rlimits = {
         let mut r = Vec::new();
         for lm in p.Rlimits.iter() {
-            r.push(oci::PosixRlimit {
+            r.push(ociPOSIXRlimit {
                 r#type: lm.Type.clone(),
                 hard: lm.Hard,
                 soft: lm.Soft,
@@ -118,7 +124,7 @@ pub fn process_grpc_to_oci(p: &grpc::Process) -> oci::Process {
         r
     };
 
-    oci::Process {
+    ociProcess {
         terminal: p.Terminal,
         console_size,
         user,
@@ -134,15 +140,15 @@ pub fn process_grpc_to_oci(p: &grpc::Process) -> oci::Process {
     }
 }
 
-fn root_grpc_to_oci(root: &grpc::Root) -> oci::Root {
-    oci::Root {
+fn root_grpc_to_oci(root: &grpcRoot) -> ociRoot {
+    ociRoot {
         path: root.Path.clone(),
         readonly: root.Readonly,
     }
 }
 
-fn mount_grpc_to_oci(m: &grpc::Mount) -> oci::Mount {
-    oci::Mount {
+fn mount_grpc_to_oci(m: &grpcMount) -> ociMount {
+    ociMount {
         destination: m.destination.clone(),
         r#type: m.field_type.clone(),
         source: m.source.clone(),
@@ -150,12 +156,13 @@ fn mount_grpc_to_oci(m: &grpc::Mount) -> oci::Mount {
     }
 }
 
+use oci::Hook as ociHook;
 use protocols::oci::Hook as grpcHook;
 
-fn hook_grpc_to_oci(h: &[grpcHook]) -> Vec<oci::Hook> {
+fn hook_grpc_to_oci(h: &[grpcHook]) -> Vec<ociHook> {
     let mut r = Vec::new();
     for e in h.iter() {
-        r.push(oci::Hook {
+        r.push(ociHook {
             path: e.Path.clone(),
             args: e.Args.clone().into_vec(),
             env: e.Env.clone().into_vec(),
@@ -165,29 +172,39 @@ fn hook_grpc_to_oci(h: &[grpcHook]) -> Vec<oci::Hook> {
     r
 }
 
-fn hooks_grpc_to_oci(h: &grpc::Hooks) -> oci::Hooks {
+fn hooks_grpc_to_oci(h: &grpcHooks) -> ociHooks {
     let prestart = hook_grpc_to_oci(h.Prestart.as_ref());
 
     let poststart = hook_grpc_to_oci(h.Poststart.as_ref());
 
     let poststop = hook_grpc_to_oci(h.Poststop.as_ref());
 
-    oci::Hooks {
+    ociHooks {
         prestart,
         poststart,
         poststop,
     }
 }
 
-fn idmap_grpc_to_oci(im: &grpc::LinuxIDMapping) -> oci::LinuxIdMapping {
-    oci::LinuxIdMapping {
+use oci::{
+    LinuxDevice as ociLinuxDevice, LinuxIDMapping as ociLinuxIDMapping,
+    LinuxIntelRdt as ociLinuxIntelRdt, LinuxNamespace as ociLinuxNamespace,
+    LinuxResources as ociLinuxResources, LinuxSeccomp as ociLinuxSeccomp,
+};
+use protocols::oci::{
+    LinuxIDMapping as grpcLinuxIDMapping, LinuxResources as grpcLinuxResources,
+    LinuxSeccomp as grpcLinuxSeccomp,
+};
+
+fn idmap_grpc_to_oci(im: &grpcLinuxIDMapping) -> ociLinuxIDMapping {
+    ociLinuxIDMapping {
         container_id: im.ContainerID,
         host_id: im.HostID,
         size: im.Size,
     }
 }
 
-fn idmaps_grpc_to_oci(ims: &[grpc::LinuxIDMapping]) -> Vec<oci::LinuxIdMapping> {
+fn idmaps_grpc_to_oci(ims: &[grpcLinuxIDMapping]) -> Vec<ociLinuxIDMapping> {
     let mut r = Vec::new();
     for im in ims.iter() {
         r.push(idmap_grpc_to_oci(im));
@@ -195,13 +212,24 @@ fn idmaps_grpc_to_oci(ims: &[grpc::LinuxIDMapping]) -> Vec<oci::LinuxIdMapping>
     r
 }
 
-fn throttle_devices_grpc_to_oci(
-    tds: &[grpc::LinuxThrottleDevice],
-) -> Vec<oci::LinuxThrottleDevice> {
+use oci::{
+    LinuxBlockIO as ociLinuxBlockIO, LinuxBlockIODevice as ociLinuxBlockIODevice,
+    LinuxCPU as ociLinuxCPU, LinuxDeviceCgroup as ociLinuxDeviceCgroup,
+    LinuxHugepageLimit as ociLinuxHugepageLimit,
+    LinuxInterfacePriority as ociLinuxInterfacePriority, LinuxMemory as ociLinuxMemory,
+    LinuxNetwork as ociLinuxNetwork, LinuxPids as ociLinuxPids,
+    LinuxThrottleDevice as ociLinuxThrottleDevice, LinuxWeightDevice as ociLinuxWeightDevice,
+};
+use protocols::oci::{
+    LinuxBlockIO as grpcLinuxBlockIO, LinuxThrottleDevice as grpcLinuxThrottleDevice,
+    LinuxWeightDevice as grpcLinuxWeightDevice,
+};
+
+fn throttle_devices_grpc_to_oci(tds: &[grpcLinuxThrottleDevice]) -> Vec<ociLinuxThrottleDevice> {
     let mut r = Vec::new();
     for td in tds.iter() {
-        r.push(oci::LinuxThrottleDevice {
-            blk: oci::LinuxBlockIoDevice {
+        r.push(ociLinuxThrottleDevice {
+            blk: ociLinuxBlockIODevice {
                 major: td.Major,
                 minor: td.Minor,
             },
@@ -211,11 +239,11 @@ fn throttle_devices_grpc_to_oci(
     r
 }
 
-fn weight_devices_grpc_to_oci(wds: &[grpc::LinuxWeightDevice]) -> Vec<oci::LinuxWeightDevice> {
+fn weight_devices_grpc_to_oci(wds: &[grpcLinuxWeightDevice]) -> Vec<ociLinuxWeightDevice> {
     let mut r = Vec::new();
     for wd in wds.iter() {
-        r.push(oci::LinuxWeightDevice {
-            blk: oci::LinuxBlockIoDevice {
+        r.push(ociLinuxWeightDevice {
+            blk: ociLinuxBlockIODevice {
                 major: wd.Major,
                 minor: wd.Minor,
             },
@@ -226,7 +254,7 @@ fn weight_devices_grpc_to_oci(wds: &[grpc::LinuxWeightDevice]) -> Vec<oci::Linux
     r
 }
 
-fn blockio_grpc_to_oci(blk: &grpc::LinuxBlockIO) -> oci::LinuxBlockIo {
+fn blockio_grpc_to_oci(blk: &grpcLinuxBlockIO) -> ociLinuxBlockIO {
     let weight_device = weight_devices_grpc_to_oci(blk.WeightDevice.as_ref());
     let throttle_read_bps_device = throttle_devices_grpc_to_oci(blk.ThrottleReadBpsDevice.as_ref());
     let throttle_write_bps_device =
@@ -236,7 +264,7 @@ fn blockio_grpc_to_oci(blk: &grpc::LinuxBlockIO) -> oci::LinuxBlockIo {
     let throttle_write_iops_device =
         throttle_devices_grpc_to_oci(blk.ThrottleWriteIOPSDevice.as_ref());
 
-    oci::LinuxBlockIo {
+    ociLinuxBlockIO {
         weight: Some(blk.Weight as u16),
         leaf_weight: Some(blk.LeafWeight as u16),
         weight_device,
@@ -247,7 +275,7 @@ fn blockio_grpc_to_oci(blk: &grpc::LinuxBlockIO) -> oci::LinuxBlockIo {
     }
 }
 
-pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources {
+pub fn resources_grpc_to_oci(res: &grpcLinuxResources) -> ociLinuxResources {
     let devices = {
         let mut d = Vec::new();
         for dev in res.Devices.iter() {
@@ -262,7 +290,7 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
             } else {
                 Some(dev.Minor)
             };
-            d.push(oci::LinuxDeviceCgroup {
+            d.push(ociLinuxDeviceCgroup {
                 allow: dev.Allow,
                 r#type: dev.Type.clone(),
                 major,
@@ -275,13 +303,13 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
 
     let memory = if res.Memory.is_some() {
         let mem = res.Memory.as_ref().unwrap();
-        Some(oci::LinuxMemory {
+        Some(ociLinuxMemory {
             limit: Some(mem.Limit),
             reservation: Some(mem.Reservation),
             swap: Some(mem.Swap),
             kernel: Some(mem.Kernel),
             kernel_tcp: Some(mem.KernelTCP),
-            swappiness: Some(mem.Swappiness as i64),
+            swapiness: Some(mem.Swappiness as i64),
             disable_oom_killer: Some(mem.DisableOOMKiller),
         })
     } else {
@@ -290,7 +318,7 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
 
     let cpu = if res.CPU.is_some() {
         let c = res.CPU.as_ref().unwrap();
-        Some(oci::LinuxCpu {
+        Some(ociLinuxCPU {
             shares: Some(c.Shares),
             quota: Some(c.Quota),
             period: Some(c.Period),
@@ -305,7 +333,7 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
 
     let pids = if res.Pids.is_some() {
         let p = res.Pids.as_ref().unwrap();
-        Some(oci::LinuxPids { limit: p.Limit })
+        Some(ociLinuxPids { limit: p.Limit })
     } else {
         None
     };
@@ -321,7 +349,7 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
     let hugepage_limits = {
         let mut r = Vec::new();
         for hl in res.HugepageLimits.iter() {
-            r.push(oci::LinuxHugepageLimit {
+            r.push(ociLinuxHugepageLimit {
                 page_size: hl.Pagesize.clone(),
                 limit: hl.Limit,
             });
@@ -334,14 +362,14 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
         let priorities = {
             let mut r = Vec::new();
             for pr in net.Priorities.iter() {
-                r.push(oci::LinuxInterfacePriority {
+                r.push(ociLinuxInterfacePriority {
                     name: pr.Name.clone(),
                     priority: pr.Priority,
                 });
             }
             r
         };
-        Some(oci::LinuxNetwork {
+        Some(ociLinuxNetwork {
             class_id: Some(net.ClassID),
             priorities,
         })
@@ -349,7 +377,7 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
         None
     };
 
-    oci::LinuxResources {
+    ociLinuxResources {
         devices,
         memory,
         cpu,
@@ -361,22 +389,17 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
     }
 }
 
-fn seccomp_grpc_to_oci(sec: &grpc::LinuxSeccomp) -> oci::LinuxSeccomp {
+use oci::{LinuxSeccompArg as ociLinuxSeccompArg, LinuxSyscall as ociLinuxSyscall};
+
+fn seccomp_grpc_to_oci(sec: &grpcLinuxSeccomp) -> ociLinuxSeccomp {
     let syscalls = {
         let mut r = Vec::new();
 
         for sys in sec.Syscalls.iter() {
             let mut args = Vec::new();
-            let errno_ret: u32;
-
-            if sys.has_errnoret() {
-                errno_ret = sys.get_errnoret();
-            } else {
-                errno_ret = libc::EPERM as u32;
-            }
 
             for arg in sys.Args.iter() {
-                args.push(oci::LinuxSeccompArg {
+                args.push(ociLinuxSeccompArg {
                     index: arg.Index as u32,
                     value: arg.Value,
                     value_two: arg.ValueTwo,
@@ -384,25 +407,23 @@ fn seccomp_grpc_to_oci(sec: &grpc::LinuxSeccomp) -> oci::LinuxSeccomp {
                 });
             }
 
-            r.push(oci::LinuxSyscall {
+            r.push(ociLinuxSyscall {
                 names: sys.Names.clone().into_vec(),
                 action: sys.Action.clone(),
-                errno_ret,
                 args,
             });
         }
         r
     };
 
-    oci::LinuxSeccomp {
+    ociLinuxSeccomp {
         default_action: sec.DefaultAction.clone(),
         architectures: sec.Architectures.clone().into_vec(),
-        flags: sec.Flags.clone().into_vec(),
         syscalls,
     }
 }
 
-fn linux_grpc_to_oci(l: &grpc::Linux) -> oci::Linux {
+fn linux_grpc_to_oci(l: &grpcLinux) -> ociLinux {
     let uid_mappings = idmaps_grpc_to_oci(l.UIDMappings.as_ref());
     let gid_mappings = idmaps_grpc_to_oci(l.GIDMappings.as_ref());
 
@@ -422,7 +443,7 @@ fn linux_grpc_to_oci(l: &grpc::Linux) -> oci::Linux {
         let mut r = Vec::new();
 
         for ns in l.Namespaces.iter() {
-            r.push(oci::LinuxNamespace {
+            r.push(ociLinuxNamespace {
                 r#type: ns.Type.clone(),
                 path: ns.Path.clone(),
             });
@@ -434,7 +455,7 @@ fn linux_grpc_to_oci(l: &grpc::Linux) -> oci::Linux {
         let mut r = Vec::new();
 
         for d in l.Devices.iter() {
-            r.push(oci::LinuxDevice {
+            r.push(ociLinuxDevice {
                 path: d.Path.clone(),
                 r#type: d.Type.clone(),
                 major: d.Major,
@@ -450,14 +471,14 @@ fn linux_grpc_to_oci(l: &grpc::Linux) -> oci::Linux {
     let intel_rdt = if l.IntelRdt.is_some() {
         let rdt = l.IntelRdt.as_ref().unwrap();
 
-        Some(oci::LinuxIntelRdt {
+        Some(ociLinuxIntelRdt {
             l3_cache_schema: rdt.L3CacheSchema.clone(),
         })
     } else {
         None
     };
 
-    oci::Linux {
+    ociLinux {
         uid_mappings,
         gid_mappings,
         sysctl: l.Sysctl.clone(),
@@ -474,11 +495,11 @@ fn linux_grpc_to_oci(l: &grpc::Linux) -> oci::Linux {
     }
 }
 
-fn linux_oci_to_grpc(_l: &oci::Linux) -> grpc::Linux {
-    grpc::Linux::default()
+fn linux_oci_to_grpc(_l: &ociLinux) -> grpcLinux {
+    grpcLinux::default()
 }
 
-pub fn grpc_to_oci(grpc: &grpc::Spec) -> oci::Spec {
+pub fn grpc_to_oci(grpc: &grpcSpec) -> ociSpec {
     // process
     let process = if grpc.Process.is_some() {
         Some(process_grpc_to_oci(grpc.Process.as_ref().unwrap()))
@@ -516,7 +537,7 @@ pub fn grpc_to_oci(grpc: &grpc::Spec) -> oci::Spec {
         None
     };
 
-    oci::Spec {
+    ociSpec {
         version: grpc.Version.clone(),
         process,
         root,

src/agent/rustjail/src/mount.rs

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use anyhow::{anyhow, bail, Context, Result};
+use anyhow::{anyhow, bail, Context, Error, Result};
 use libc::uid_t;
 use nix::errno::Errno;
 use nix::fcntl::{self, OFlag};
@@ -22,11 +22,13 @@ use std::os::unix::io::RawFd;
 use std::path::{Path, PathBuf};
 
 use path_absolutize::*;
+use scan_fmt;
 use std::fs::File;
 use std::io::{BufRead, BufReader};
 
 use crate::container::DEFAULT_DEVICES;
 use crate::sync::write_count;
+use lazy_static;
 use std::string::ToString;
 
 use crate::log_child;
@@ -48,16 +50,14 @@ pub struct Info {
     vfs_opts: String,
 }
 
-const MOUNTINFOFORMAT: &str = "{d} {d} {d}:{d} {} {} {} {}";
+const MOUNTINFOFORMAT: &'static str = "{d} {d} {d}:{d} {} {} {} {}";
 const PROC_PATH: &str = "/proc";
 
 // since libc didn't defined this const for musl, thus redefined it here.
-#[cfg(all(target_os = "linux", target_env = "gnu", not(target_arch = "s390x")))]
+#[cfg(all(target_os = "linux", target_env = "gnu"))]
 const PROC_SUPER_MAGIC: libc::c_long = 0x00009fa0;
 #[cfg(all(target_os = "linux", target_env = "musl"))]
 const PROC_SUPER_MAGIC: libc::c_ulong = 0x00009fa0;
-#[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "s390x"))]
-const PROC_SUPER_MAGIC: libc::c_uint = 0x00009fa0;
 
 lazy_static! {
     static ref PROPAGATION: HashMap<&'static str, MsFlags> = {
@@ -68,8 +68,6 @@ lazy_static! {
         m.insert("rprivate", MsFlags::MS_PRIVATE | MsFlags::MS_REC);
         m.insert("slave", MsFlags::MS_SLAVE);
         m.insert("rslave", MsFlags::MS_SLAVE | MsFlags::MS_REC);
-        m.insert("unbindable", MsFlags::MS_UNBINDABLE);
-        m.insert("runbindable", MsFlags::MS_UNBINDABLE | MsFlags::MS_REC);
         m
     };
     static ref OPTIONS: HashMap<&'static str, (bool, MsFlags)> = {
@@ -95,6 +93,17 @@ lazy_static! {
         m.insert("nodiratime", (false, MsFlags::MS_NODIRATIME));
         m.insert("bind", (false, MsFlags::MS_BIND));
         m.insert("rbind", (false, MsFlags::MS_BIND | MsFlags::MS_REC));
+        m.insert("unbindable", (false, MsFlags::MS_UNBINDABLE));
+        m.insert(
+            "runbindable",
+            (false, MsFlags::MS_UNBINDABLE | MsFlags::MS_REC),
+        );
+        m.insert("private", (false, MsFlags::MS_PRIVATE));
+        m.insert("rprivate", (false, MsFlags::MS_PRIVATE | MsFlags::MS_REC));
+        m.insert("shared", (false, MsFlags::MS_SHARED));
+        m.insert("rshared", (false, MsFlags::MS_SHARED | MsFlags::MS_REC));
+        m.insert("slave", (false, MsFlags::MS_SLAVE));
+        m.insert("rslave", (false, MsFlags::MS_SLAVE | MsFlags::MS_REC));
         m.insert("relatime", (false, MsFlags::MS_RELATIME));
         m.insert("norelatime", (true, MsFlags::MS_RELATIME));
         m.insert("strictatime", (false, MsFlags::MS_STRICTATIME));
@@ -105,12 +114,7 @@ lazy_static! {
 
 #[inline(always)]
 #[allow(unused_variables)]
-pub fn mount<
-    P1: ?Sized + NixPath,
-    P2: ?Sized + NixPath,
-    P3: ?Sized + NixPath,
-    P4: ?Sized + NixPath,
->(
+fn mount<P1: ?Sized + NixPath, P2: ?Sized + NixPath, P3: ?Sized + NixPath, P4: ?Sized + NixPath>(
     source: Option<&P1>,
     target: &P2,
     fstype: Option<&P3>,
@@ -125,7 +129,7 @@ pub fn mount<
 
 #[inline(always)]
 #[allow(unused_variables)]
-pub fn umount2<P: ?Sized + NixPath>(
+fn umount2<P: ?Sized + NixPath>(
     target: &P,
     flags: MntFlags,
 ) -> std::result::Result<(), nix::Error> {
@@ -149,7 +153,7 @@ pub fn init_rootfs(
     let linux = &spec
         .linux
         .as_ref()
-        .ok_or_else(|| anyhow!("Could not get linux configuration from spec"))?;
+        .ok_or::<Error>(anyhow!("Could not get linux configuration from spec"))?;
 
     let mut flags = MsFlags::MS_REC;
     match PROPAGATION.get(&linux.rootfs_propagation.as_str()) {
@@ -160,14 +164,14 @@ pub fn init_rootfs(
     let root = spec
         .root
         .as_ref()
-        .ok_or_else(|| anyhow!("Could not get rootfs path from spec"))
+        .ok_or(anyhow!("Could not get rootfs path from spec"))
         .and_then(|r| {
             fs::canonicalize(r.path.as_str()).context("Could not canonicalize rootfs path")
         })?;
 
     let rootfs = (*root)
         .to_str()
-        .ok_or_else(|| anyhow!("Could not convert rootfs path to string"))?;
+        .ok_or(anyhow!("Could not convert rootfs path to string"))?;
 
     mount(None::<&str>, "/", None::<&str>, flags, None::<&str>)?;
 
@@ -181,10 +185,9 @@ pub fn init_rootfs(
         None::<&str>,
     )?;
 
-    let mut bind_mount_dev = false;
     for m in &spec.mounts {
-        let (mut flags, pgflags, data) = parse_mount(&m);
-        if !m.destination.starts_with('/') || m.destination.contains("..") {
+        let (mut flags, data) = parse_mount(&m);
+        if !m.destination.starts_with("/") || m.destination.contains("..") {
             return Err(anyhow!(
                 "the mount destination {} is invalid",
                 m.destination
@@ -195,9 +198,6 @@ pub fn init_rootfs(
             mount_cgroups(cfd_log, &m, rootfs, flags, &data, cpath, mounts)?;
         } else {
             if m.destination == "/dev" {
-                if m.r#type == "bind" {
-                    bind_mount_dev = true;
-                }
                 flags &= !MsFlags::MS_RDONLY;
             }
 
@@ -225,15 +225,13 @@ pub fn init_rootfs(
             // effective.
             // first check that we have non-default options required before attempting a
             // remount
-            if m.r#type == "bind" && !pgflags.is_empty() {
-                let dest = secure_join(rootfs, &m.destination);
-                mount(
-                    None::<&str>,
-                    dest.as_str(),
-                    None::<&str>,
-                    pgflags,
-                    None::<&str>,
-                )?;
+            if m.r#type == "bind" {
+                for o in &m.options {
+                    if let Some(fl) = PROPAGATION.get(o.as_str()) {
+                        let dest = format!("{}{}", &rootfs, &m.destination);
+                        mount(None::<&str>, dest.as_str(), None::<&str>, *fl, None::<&str>)?;
+                    }
+                }
             }
         }
     }
@@ -241,14 +239,9 @@ pub fn init_rootfs(
     let olddir = unistd::getcwd()?;
     unistd::chdir(rootfs)?;
 
-    // in case the /dev directory was binded mount from guest,
-    // then there's no need to create devices nodes and symlinks
-    // in /dev.
-    if !bind_mount_dev {
-        default_symlinks()?;
-        create_devices(&linux.devices, bind_device)?;
-        ensure_ptmx()?;
-    }
+    default_symlinks()?;
+    create_devices(&linux.devices, bind_device)?;
+    ensure_ptmx()?;
 
     unistd::chdir(&olddir)?;
 
@@ -280,9 +273,9 @@ fn check_proc_mount(m: &Mount) -> Result<()> {
         // only allow a mount on-top of proc if it's source is "proc"
         unsafe {
             let mut stats = MaybeUninit::<libc::statfs>::uninit();
-            if m.source
+            if let Ok(_) = m
+                .source
                 .with_nix_path(|path| libc::statfs(path.as_ptr(), stats.as_mut_ptr()))
-                .is_ok()
             {
                 if stats.assume_init().f_type == PROC_SUPER_MAGIC {
                     return Ok(());
@@ -305,7 +298,7 @@ fn check_proc_mount(m: &Mount) -> Result<()> {
         )));
     }
 
-    Ok(())
+    return Ok(());
 }
 
 fn mount_cgroups_v2(cfd_log: RawFd, m: &Mount, rootfs: &str, flags: MsFlags) -> Result<()> {
@@ -593,14 +586,15 @@ pub fn ms_move_root(rootfs: &str) -> Result<bool> {
     let abs_root_buf = root_path.absolutize()?;
     let abs_root = abs_root_buf
         .to_str()
-        .ok_or_else(|| anyhow!("failed to parse {} to absolute path", rootfs))?;
+        .ok_or::<Error>(anyhow!("failed to parse {} to absolute path", rootfs))?;
 
     for info in mount_infos.iter() {
         let mount_point = Path::new(&info.mount_point);
         let abs_mount_buf = mount_point.absolutize()?;
-        let abs_mount_point = abs_mount_buf
-            .to_str()
-            .ok_or_else(|| anyhow!("failed to parse {} to absolute path", info.mount_point))?;
+        let abs_mount_point = abs_mount_buf.to_str().ok_or::<Error>(anyhow!(
+            "failed to parse {} to absolute path",
+            info.mount_point
+        ))?;
         let abs_mount_point_string = String::from(abs_mount_point);
 
         // Umount every syfs and proc file systems, except those under the container rootfs
@@ -650,73 +644,26 @@ pub fn ms_move_root(rootfs: &str) -> Result<bool> {
     Ok(true)
 }
 
-fn parse_mount(m: &Mount) -> (MsFlags, MsFlags, String) {
+fn parse_mount(m: &Mount) -> (MsFlags, String) {
     let mut flags = MsFlags::empty();
-    let mut pgflags = MsFlags::empty();
     let mut data = Vec::new();
 
     for o in &m.options {
-        if let Some(v) = OPTIONS.get(o.as_str()) {
-            let (clear, fl) = *v;
-            if clear {
-                flags &= !fl;
-            } else {
-                flags |= fl;
-            }
-        } else if let Some(fl) = PROPAGATION.get(o.as_str()) {
-            pgflags |= *fl;
-        } else {
-            data.push(o.clone());
-        }
-    }
-
-    (flags, pgflags, data.join(","))
-}
-
-// This function constructs a canonicalized path by combining the `rootfs` and `unsafe_path` elements.
-// The resulting path is guaranteed to be ("below" / "in a directory under") the `rootfs` directory.
-//
-// Parameters:
-//
-// - `rootfs` is the absolute path to the root of the containers root filesystem directory.
-// - `unsafe_path` is path inside a container. It is unsafe since it may try to "escape" from the containers
-//    rootfs by using one or more "../" path elements or is its a symlink to path.
-fn secure_join(rootfs: &str, unsafe_path: &str) -> String {
-    let mut path = PathBuf::from(format!("{}/", rootfs));
-    let unsafe_p = Path::new(&unsafe_path);
-
-    for it in unsafe_p.iter() {
-        let it_p = Path::new(&it);
-
-        // if it_p leads with "/", path.push(it) will be replace as it, so ignore "/"
-        if it_p.has_root() {
-            continue;
-        };
-
-        path.push(it);
-        if let Ok(v) = path.read_link() {
-            if v.is_absolute() {
-                path = PathBuf::from(format!("{}{}", rootfs, v.to_str().unwrap().to_string()));
-            } else {
-                path.pop();
-                for it in v.iter() {
-                    path.push(it);
-                    if path.exists() {
-                        path = path.canonicalize().unwrap();
-                        if !path.starts_with(rootfs) {
-                            path = PathBuf::from(rootfs.to_string());
-                        }
-                    }
+        match OPTIONS.get(o.as_str()) {
+            Some(v) => {
+                let (clear, fl) = *v;
+                if clear {
+                    flags &= !fl;
+                } else {
+                    flags |= fl;
                 }
             }
-        }
-        // skip any ".."
-        if path.ends_with("..") {
-            path.pop();
+
+            None => data.push(o.clone()),
         }
     }
 
-    path.to_str().unwrap().to_string()
+    (flags, data.join(","))
 }
 
 fn mount_from(
@@ -728,14 +675,14 @@ fn mount_from(
     _label: &str,
 ) -> Result<()> {
     let d = String::from(data);
-    let dest = secure_join(rootfs, &m.destination);
+    let dest = format!("{}{}", rootfs, &m.destination);
 
     let src = if m.r#type.as_str() == "bind" {
         let src = fs::canonicalize(m.source.as_str())?;
-        let dir = if src.is_dir() {
-            Path::new(&dest)
-        } else {
+        let dir = if src.is_file() {
             Path::new(&dest).parent().unwrap()
+        } else {
+            Path::new(&dest)
         };
 
         let _ = fs::create_dir_all(&dir).map_err(|e| {
@@ -748,7 +695,7 @@ fn mount_from(
         });
 
         // make sure file exists so we can bind over it
-        if !src.is_dir() {
+        if src.is_file() {
             let _ = OpenOptions::new().create(true).write(true).open(&dest);
         }
         src.to_str().unwrap().to_string()
@@ -808,7 +755,7 @@ fn mount_from(
     Ok(())
 }
 
-static SYMLINKS: &[(&str, &str)] = &[
+static SYMLINKS: &'static [(&'static str, &'static str)] = &[
     ("/proc/self/fd", "dev/fd"),
     ("/proc/self/fd/0", "dev/stdin"),
     ("/proc/self/fd/1", "dev/stdout"),
@@ -916,7 +863,7 @@ pub fn finish_rootfs(cfd_log: RawFd, spec: &Spec) -> Result<()> {
 
     for m in spec.mounts.iter() {
         if m.destination == "/dev" {
-            let (flags, _, _) = parse_mount(m);
+            let (flags, _) = parse_mount(m);
             if flags.contains(MsFlags::MS_RDONLY) {
                 mount(
                     Some("/dev"),
@@ -941,7 +888,7 @@ pub fn finish_rootfs(cfd_log: RawFd, spec: &Spec) -> Result<()> {
 }
 
 fn mask_path(path: &str) -> Result<()> {
-    if !path.starts_with('/') || path.contains("..") {
+    if !path.starts_with("/") || path.contains("..") {
         return Err(nix::Error::Sys(Errno::EINVAL).into());
     }
 
@@ -970,7 +917,7 @@ fn mask_path(path: &str) -> Result<()> {
 }
 
 fn readonly_path(path: &str) -> Result<()> {
-    if !path.starts_with('/') || path.contains("..") {
+    if !path.starts_with("/") || path.contains("..") {
         return Err(nix::Error::Sys(Errno::EINVAL).into());
     }
 
@@ -1012,10 +959,6 @@ fn readonly_path(path: &str) -> Result<()> {
 mod tests {
     use super::*;
     use crate::skip_if_not_root;
-    use std::fs::create_dir;
-    use std::fs::create_dir_all;
-    use std::fs::remove_dir_all;
-    use std::os::unix::fs;
     use std::os::unix::io::AsRawFd;
     use tempfile::tempdir;
 
@@ -1045,7 +988,7 @@ mod tests {
         );
 
         let rootfs = tempdir().unwrap();
-        let ret = create_dir(rootfs.path().join("dev"));
+        let ret = fs::create_dir(rootfs.path().join("dev"));
         assert!(ret.is_ok(), "Got: {:?}", ret);
 
         spec.root = Some(oci::Root {
@@ -1056,8 +999,8 @@ mod tests {
         // there is no spec.mounts, but should pass
         let ret = init_rootfs(stdout_fd, &spec, &cpath, &mounts, true);
         assert!(ret.is_ok(), "Should pass. Got: {:?}", ret);
-        let _ = remove_dir_all(rootfs.path().join("dev"));
-        let _ = create_dir(rootfs.path().join("dev"));
+        let _ = fs::remove_dir_all(rootfs.path().join("dev"));
+        let _ = fs::create_dir(rootfs.path().join("dev"));
 
         // Adding bad mount point to spec.mounts
         spec.mounts.push(oci::Mount {
@@ -1075,8 +1018,8 @@ mod tests {
             ret
         );
         spec.mounts.pop();
-        let _ = remove_dir_all(rootfs.path().join("dev"));
-        let _ = create_dir(rootfs.path().join("dev"));
+        let _ = fs::remove_dir_all(rootfs.path().join("dev"));
+        let _ = fs::create_dir(rootfs.path().join("dev"));
 
         // mounting a cgroup
         spec.mounts.push(oci::Mount {
@@ -1089,8 +1032,8 @@ mod tests {
         let ret = init_rootfs(stdout_fd, &spec, &cpath, &mounts, true);
         assert!(ret.is_ok(), "Should pass. Got: {:?}", ret);
         spec.mounts.pop();
-        let _ = remove_dir_all(rootfs.path().join("dev"));
-        let _ = create_dir(rootfs.path().join("dev"));
+        let _ = fs::remove_dir_all(rootfs.path().join("dev"));
+        let _ = fs::create_dir(rootfs.path().join("dev"));
 
         // mounting /dev
         spec.mounts.push(oci::Mount {
@@ -1127,11 +1070,11 @@ mod tests {
         cgroup_mounts.insert("cpu".to_string(), "cpu".to_string());
         cgroup_mounts.insert("memory".to_string(), "memory".to_string());
 
-        let ret = create_dir_all(tempdir.path().join("cgroups"));
+        let ret = fs::create_dir_all(tempdir.path().join("cgroups"));
         assert!(ret.is_ok(), "Should pass. Got {:?}", ret);
-        let ret = create_dir_all(tempdir.path().join("cpu"));
+        let ret = fs::create_dir_all(tempdir.path().join("cpu"));
         assert!(ret.is_ok(), "Should pass. Got {:?}", ret);
-        let ret = create_dir_all(tempdir.path().join("memory"));
+        let ret = fs::create_dir_all(tempdir.path().join("memory"));
         assert!(ret.is_ok(), "Should pass. Got {:?}", ret);
 
         let ret = mount_cgroups(
@@ -1279,89 +1222,4 @@ mod tests {
 
         assert!(check_proc_mount(&mount).is_err());
     }
-
-    #[test]
-    fn test_secure_join() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            name: &'a str,
-            rootfs: &'a str,
-            unsafe_path: &'a str,
-            symlink_path: &'a str,
-            result: &'a str,
-        }
-
-        // create tempory directory to simulate container rootfs with symlink
-        let rootfs_dir = tempdir().expect("failed to create tmpdir");
-        let rootfs_path = rootfs_dir.path().to_str().unwrap();
-
-        let tests = &[
-            TestData {
-                name: "rootfs_not_exist",
-                rootfs: "/home/rootfs",
-                unsafe_path: "a/b/c",
-                symlink_path: "",
-                result: "/home/rootfs/a/b/c",
-            },
-            TestData {
-                name: "relative_path",
-                rootfs: "/home/rootfs",
-                unsafe_path: "../../../a/b/c",
-                symlink_path: "",
-                result: "/home/rootfs/a/b/c",
-            },
-            TestData {
-                name: "skip any ..",
-                rootfs: "/home/rootfs",
-                unsafe_path: "../../../a/../../b/../../c",
-                symlink_path: "",
-                result: "/home/rootfs/a/b/c",
-            },
-            TestData {
-                name: "rootfs is null",
-                rootfs: "",
-                unsafe_path: "",
-                symlink_path: "",
-                result: "/",
-            },
-            TestData {
-                name: "relative softlink beyond container rootfs",
-                rootfs: rootfs_path,
-                unsafe_path: "1",
-                symlink_path: "../../../",
-                result: rootfs_path,
-            },
-            TestData {
-                name: "abs softlink points to the non-exist directory",
-                rootfs: rootfs_path,
-                unsafe_path: "2",
-                symlink_path: "/dddd",
-                result: &format!("{}/dddd", rootfs_path).as_str().to_owned(),
-            },
-            TestData {
-                name: "abs softlink points to the root",
-                rootfs: rootfs_path,
-                unsafe_path: "3",
-                symlink_path: "/",
-                result: &format!("{}/", rootfs_path).as_str().to_owned(),
-            },
-        ];
-
-        for (i, t) in tests.iter().enumerate() {
-            // Create a string containing details of the test
-            let msg = format!("test[{}]: {:?}", i, t);
-
-            // if is_symlink, then should be prepare the softlink environment
-            if t.symlink_path != "" {
-                fs::symlink(t.symlink_path, format!("{}/{}", t.rootfs, t.unsafe_path)).unwrap();
-            }
-            let result = secure_join(t.rootfs, t.unsafe_path);
-
-            // Update the test details string with the results of the call
-            let msg = format!("{}, result: {:?}", msg, result);
-
-            // Perform the checks
-            assert!(result == t.result, "{}", msg);
-        }
-    }
 }

src/agent/rustjail/src/pipestream.rs

@@ -1,203 +0,0 @@
-// Copyright (c) 2020 Ant Group
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-//! Async support for pipe or something has file descriptor
-
-use nix::unistd;
-use std::{
-    fmt, io,
-    io::{Read, Result, Write},
-    mem,
-    os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd},
-    pin::Pin,
-    task::{Context, Poll},
-};
-
-use futures::ready;
-use tokio::io::{unix::AsyncFd, AsyncRead, AsyncWrite, ReadBuf};
-
-fn set_nonblocking(fd: RawFd) {
-    unsafe {
-        libc::fcntl(fd, libc::F_SETFL, libc::O_NONBLOCK);
-    }
-}
-
-struct StreamFd(RawFd);
-
-impl io::Read for &StreamFd {
-    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
-        match unistd::read(self.0, buf) {
-            Ok(l) => Ok(l),
-            Err(e) => Err(e.as_errno().unwrap().into()),
-        }
-    }
-}
-
-impl io::Write for &StreamFd {
-    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
-        match unistd::write(self.0, buf) {
-            Ok(l) => Ok(l),
-            Err(e) => Err(e.as_errno().unwrap().into()),
-        }
-    }
-
-    fn flush(&mut self) -> io::Result<()> {
-        Ok(())
-    }
-}
-
-impl StreamFd {
-    fn close(&mut self) -> io::Result<()> {
-        match unistd::close(self.0) {
-            Ok(()) => Ok(()),
-            Err(e) => Err(e.as_errno().unwrap().into()),
-        }
-    }
-}
-
-impl Drop for StreamFd {
-    fn drop(&mut self) {
-        self.close().ok();
-    }
-}
-
-impl AsRawFd for StreamFd {
-    fn as_raw_fd(&self) -> RawFd {
-        self.0
-    }
-}
-
-pub struct PipeStream(AsyncFd<StreamFd>);
-
-impl PipeStream {
-    pub fn new(fd: RawFd) -> Result<Self> {
-        set_nonblocking(fd);
-        Ok(Self(AsyncFd::new(StreamFd(fd))?))
-    }
-
-    pub fn from_fd(fd: RawFd) -> Self {
-        unsafe { Self::from_raw_fd(fd) }
-    }
-}
-
-impl AsRawFd for PipeStream {
-    fn as_raw_fd(&self) -> RawFd {
-        self.0.as_raw_fd()
-    }
-}
-
-impl IntoRawFd for PipeStream {
-    fn into_raw_fd(self) -> RawFd {
-        let fd = self.as_raw_fd();
-        mem::forget(self);
-        fd
-    }
-}
-
-impl FromRawFd for PipeStream {
-    unsafe fn from_raw_fd(fd: RawFd) -> Self {
-        Self::new(fd).unwrap()
-    }
-}
-
-impl fmt::Debug for PipeStream {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "PipeStream({})", self.as_raw_fd())
-    }
-}
-
-impl AsyncRead for PipeStream {
-    fn poll_read(
-        self: Pin<&mut Self>,
-        cx: &mut Context<'_>,
-        buf: &mut ReadBuf<'_>,
-    ) -> Poll<Result<()>> {
-        let b;
-        unsafe {
-            b = &mut *(buf.unfilled_mut() as *mut [mem::MaybeUninit<u8>] as *mut [u8]);
-        };
-
-        loop {
-            let mut guard = ready!(self.0.poll_read_ready(cx))?;
-
-            match guard.try_io(|inner| inner.get_ref().read(b)) {
-                Ok(Ok(n)) => {
-                    unsafe {
-                        buf.assume_init(n);
-                    }
-                    buf.advance(n);
-                    return Ok(()).into();
-                }
-                Ok(Err(e)) => return Err(e).into(),
-                Err(_would_block) => {
-                    continue;
-                }
-            }
-        }
-    }
-}
-
-impl AsyncWrite for PipeStream {
-    fn poll_write(
-        self: Pin<&mut Self>,
-        cx: &mut Context<'_>,
-        buf: &[u8],
-    ) -> Poll<io::Result<usize>> {
-        loop {
-            let mut guard = ready!(self.0.poll_write_ready(cx))?;
-
-            match guard.try_io(|inner| inner.get_ref().write(buf)) {
-                Ok(result) => return Poll::Ready(result),
-                Err(_would_block) => continue,
-            }
-        }
-    }
-
-    fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
-        Poll::Ready(Ok(()))
-    }
-
-    fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
-        // Do nothing in shutdown is very important
-        // The only right way to shutdown pipe is drop it
-        // Otherwise PipeStream will conflict with its twins
-        // Because they both have same fd, and both registered.
-        Poll::Ready(Ok(()))
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use nix::fcntl::OFlag;
-    use tokio::io::{AsyncReadExt, AsyncWriteExt};
-
-    #[tokio::test]
-    // Shutdown should never close the inner fd.
-    async fn test_pipestream_shutdown() {
-        let (_, wfd1) = unistd::pipe2(OFlag::O_CLOEXEC).unwrap();
-        let mut writer1 = PipeStream::new(wfd1).unwrap();
-
-        // if close fd in shutdown, the fd will be reused
-        // and the test will failed
-        let _ = writer1.shutdown().await.unwrap();
-
-        // let _ = unistd::close(wfd1);
-
-        let (rfd2, wfd2) = unistd::pipe2(OFlag::O_CLOEXEC).unwrap(); // reuse fd number, rfd2 == wfd1
-
-        let mut reader2 = PipeStream::new(rfd2).unwrap();
-        let mut writer2 = PipeStream::new(wfd2).unwrap();
-
-        // deregister writer1, then reader2 which has the same fd will be deregistered from epoll
-        drop(writer1);
-
-        let _ = writer2.write(b"1").await;
-
-        let mut content = vec![0u8; 1];
-        // Will Block here if shutdown close the fd.
-        let _ = reader2.read(&mut content).await;
-    }
-}

src/agent/rustjail/src/process.rs

@@ -6,7 +6,7 @@
 use libc::pid_t;
 use std::fs::File;
 use std::os::unix::io::RawFd;
-use tokio::sync::mpsc::Sender;
+use std::sync::mpsc::Sender;
 
 use nix::fcntl::{fcntl, FcntlArg, OFlag};
 use nix::sys::signal::{self, Signal};
@@ -17,35 +17,14 @@ use nix::Result;
 use oci::Process as OCIProcess;
 use slog::Logger;
 
-use crate::pipestream::PipeStream;
-use std::collections::HashMap;
-use std::sync::Arc;
-use tokio::io::{split, ReadHalf, WriteHalf};
-use tokio::sync::Mutex;
-use tokio::sync::Notify;
-
-#[derive(Debug, PartialEq, Eq, Hash, Clone)]
-pub enum StreamType {
-    Stdin,
-    Stdout,
-    Stderr,
-    TermMaster,
-    ParentStdin,
-    ParentStdout,
-    ParentStderr,
-}
-
-type Reader = Arc<Mutex<ReadHalf<PipeStream>>>;
-type Writer = Arc<Mutex<WriteHalf<PipeStream>>>;
-
 #[derive(Debug)]
 pub struct Process {
     pub exec_id: String,
     pub stdin: Option<RawFd>,
     pub stdout: Option<RawFd>,
     pub stderr: Option<RawFd>,
-    pub exit_tx: Option<tokio::sync::watch::Sender<bool>>,
-    pub exit_rx: Option<tokio::sync::watch::Receiver<bool>>,
+    pub exit_pipe_r: Option<RawFd>,
+    pub exit_pipe_w: Option<RawFd>,
     pub extra_files: Vec<File>,
     pub term_master: Option<RawFd>,
     pub tty: bool,
@@ -61,10 +40,6 @@ pub struct Process {
     pub exit_watchers: Vec<Sender<i32>>,
     pub oci: OCIProcess,
     pub logger: Logger,
-    pub term_exit_notifier: Arc<Notify>,
-
-    readers: HashMap<StreamType, Reader>,
-    writers: HashMap<StreamType, Writer>,
 }
 
 pub trait ProcessOperations {
@@ -96,15 +71,14 @@ impl Process {
         pipe_size: i32,
     ) -> Result<Self> {
         let logger = logger.new(o!("subsystem" => "process"));
-        let (exit_tx, exit_rx) = tokio::sync::watch::channel(false);
 
         let mut p = Process {
             exec_id: String::from(id),
             stdin: None,
             stdout: None,
             stderr: None,
-            exit_tx: Some(exit_tx),
-            exit_rx: Some(exit_rx),
+            exit_pipe_w: None,
+            exit_pipe_r: None,
             extra_files: Vec::new(),
             tty: ocip.terminal,
             term_master: None,
@@ -117,9 +91,6 @@ impl Process {
             exit_watchers: Vec::new(),
             oci: ocip.clone(),
             logger: logger.clone(),
-            term_exit_notifier: Arc::new(Notify::new()),
-            readers: HashMap::new(),
-            writers: HashMap::new(),
         };
 
         info!(logger, "before create console socket!");
@@ -141,60 +112,6 @@ impl Process {
         }
         Ok(p)
     }
-
-    pub fn notify_term_close(&mut self) {
-        let notify = self.term_exit_notifier.clone();
-        notify.notify_one();
-    }
-
-    fn get_fd(&self, stream_type: &StreamType) -> Option<RawFd> {
-        match stream_type {
-            StreamType::Stdin => self.stdin,
-            StreamType::Stdout => self.stdout,
-            StreamType::Stderr => self.stderr,
-            StreamType::TermMaster => self.term_master,
-            StreamType::ParentStdin => self.parent_stdin,
-            StreamType::ParentStdout => self.parent_stdout,
-            StreamType::ParentStderr => self.parent_stderr,
-        }
-    }
-
-    fn get_stream_and_store(&mut self, stream_type: StreamType) -> Option<(Reader, Writer)> {
-        let fd = self.get_fd(&stream_type)?;
-        let stream = PipeStream::from_fd(fd);
-
-        let (reader, writer) = split(stream);
-        let reader = Arc::new(Mutex::new(reader));
-        let writer = Arc::new(Mutex::new(writer));
-
-        self.readers.insert(stream_type.clone(), reader.clone());
-        self.writers.insert(stream_type, writer.clone());
-
-        Some((reader, writer))
-    }
-
-    pub fn get_reader(&mut self, stream_type: StreamType) -> Option<Reader> {
-        if let Some(reader) = self.readers.get(&stream_type) {
-            return Some(reader.clone());
-        }
-
-        let (reader, _) = self.get_stream_and_store(stream_type)?;
-        Some(reader)
-    }
-
-    pub fn get_writer(&mut self, stream_type: StreamType) -> Option<Writer> {
-        if let Some(writer) = self.writers.get(&stream_type) {
-            return Some(writer.clone());
-        }
-
-        let (_, writer) = self.get_stream_and_store(stream_type)?;
-        Some(writer)
-    }
-
-    pub fn close_stream(&mut self, stream_type: StreamType) {
-        let _ = self.readers.remove(&stream_type);
-        let _ = self.writers.remove(&stream_type);
-    }
 }
 
 fn create_extended_pipe(flags: OFlag, pipe_size: i32) -> Result<(RawFd, RawFd)> {
@@ -252,6 +169,7 @@ mod tests {
 
         // -1 by default
         assert_eq!(process.pid, -1);
+        assert!(process.wait().is_err());
         // signal to every process in the process
         // group of the calling process.
         process.pid = 0;

src/agent/rustjail/src/sync.rs

@@ -14,8 +14,8 @@ pub const SYNC_SUCCESS: i32 = 1;
 pub const SYNC_FAILED: i32 = 2;
 pub const SYNC_DATA: i32 = 3;
 
-pub const DATA_SIZE: usize = 100;
-pub const MSG_SIZE: usize = mem::size_of::<i32>();
+const DATA_SIZE: usize = 100;
+const MSG_SIZE: usize = mem::size_of::<i32>();
 
 #[macro_export]
 macro_rules! log_child {
@@ -72,15 +72,7 @@ fn read_count(fd: RawFd, count: usize) -> Result<Vec<u8>> {
         }
     }
 
-    if len != count {
-        Err(anyhow::anyhow!(
-            "invalid read count expect {} get {}",
-            count,
-            len
-        ))
-    } else {
-        Ok(v[0..len].to_vec())
-    }
+    Ok(v[0..len].to_vec())
 }
 
 pub fn read_sync(fd: RawFd) -> Result<Vec<u8>> {
@@ -96,14 +88,14 @@ pub fn read_sync(fd: RawFd) -> Result<Vec<u8>> {
     let buf_array: [u8; MSG_SIZE] = [buf[0], buf[1], buf[2], buf[3]];
     let msg: i32 = i32::from_be_bytes(buf_array);
     match msg {
-        SYNC_SUCCESS => Ok(Vec::new()),
+        SYNC_SUCCESS => return Ok(Vec::new()),
         SYNC_DATA => {
             let buf = read_count(fd, MSG_SIZE)?;
             let buf_array: [u8; MSG_SIZE] = [buf[0], buf[1], buf[2], buf[3]];
             let msg_length: i32 = i32::from_be_bytes(buf_array);
             let data_buf = read_count(fd, msg_length as usize)?;
 
-            Ok(data_buf)
+            return Ok(data_buf);
         }
         SYNC_FAILED => {
             let mut error_buf = vec![];
@@ -127,9 +119,9 @@ pub fn read_sync(fd: RawFd) -> Result<Vec<u8>> {
                 }
             };
 
-            Err(anyhow!(error_str))
+            return Err(anyhow!(error_str));
         }
-        _ => Err(anyhow!("error in receive sync message")),
+        _ => return Err(anyhow!("error in receive sync message")),
     }
 }
 

src/agent/rustjail/src/sync_with_async.rs

@@ -1,140 +0,0 @@
-// Copyright (c) 2020 Ant Group
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-//! The async version of sync module used for IPC
-
-use crate::pipestream::PipeStream;
-use anyhow::{anyhow, Result};
-use nix::errno::Errno;
-use tokio::io::{AsyncReadExt, AsyncWriteExt};
-
-use crate::sync::{DATA_SIZE, MSG_SIZE, SYNC_DATA, SYNC_FAILED, SYNC_SUCCESS};
-
-async fn write_count(pipe_w: &mut PipeStream, buf: &[u8], count: usize) -> Result<usize> {
-    let mut len = 0;
-
-    loop {
-        match pipe_w.write(&buf[len..]).await {
-            Ok(l) => {
-                len += l;
-                if len == count {
-                    break;
-                }
-            }
-
-            Err(e) => {
-                if e.raw_os_error().unwrap() != Errno::EINTR as i32 {
-                    return Err(e.into());
-                }
-            }
-        }
-    }
-
-    Ok(len)
-}
-
-async fn read_count(pipe_r: &mut PipeStream, count: usize) -> Result<Vec<u8>> {
-    let mut v: Vec<u8> = vec![0; count];
-    let mut len = 0;
-
-    loop {
-        match pipe_r.read(&mut v[len..]).await {
-            Ok(l) => {
-                len += l;
-                if len == count || l == 0 {
-                    break;
-                }
-            }
-
-            Err(e) => {
-                if e.raw_os_error().unwrap() != Errno::EINTR as i32 {
-                    return Err(e.into());
-                }
-            }
-        }
-    }
-
-    Ok(v[0..len].to_vec())
-}
-
-pub async fn read_async(pipe_r: &mut PipeStream) -> Result<Vec<u8>> {
-    let buf = read_count(pipe_r, MSG_SIZE).await?;
-    if buf.len() != MSG_SIZE {
-        return Err(anyhow!(
-            "process: {} failed to receive async message from peer: got msg length: {}, expected: {}",
-            std::process::id(),
-            buf.len(),
-            MSG_SIZE
-        ));
-    }
-    let buf_array: [u8; MSG_SIZE] = [buf[0], buf[1], buf[2], buf[3]];
-    let msg: i32 = i32::from_be_bytes(buf_array);
-    match msg {
-        SYNC_SUCCESS => Ok(Vec::new()),
-        SYNC_DATA => {
-            let buf = read_count(pipe_r, MSG_SIZE).await?;
-            let buf_array: [u8; MSG_SIZE] = [buf[0], buf[1], buf[2], buf[3]];
-            let msg_length: i32 = i32::from_be_bytes(buf_array);
-            let data_buf = read_count(pipe_r, msg_length as usize).await?;
-
-            Ok(data_buf)
-        }
-        SYNC_FAILED => {
-            let mut error_buf = vec![];
-            loop {
-                let buf = read_count(pipe_r, DATA_SIZE).await?;
-
-                error_buf.extend(&buf);
-                if DATA_SIZE == buf.len() {
-                    continue;
-                } else {
-                    break;
-                }
-            }
-
-            let error_str = match std::str::from_utf8(&error_buf) {
-                Ok(v) => String::from(v),
-                Err(e) => {
-                    return Err(
-                        anyhow!(e).context("receive error message from child process failed")
-                    );
-                }
-            };
-
-            Err(anyhow!(error_str))
-        }
-        _ => Err(anyhow!("error in receive sync message")),
-    }
-}
-
-pub async fn write_async(pipe_w: &mut PipeStream, msg_type: i32, data_str: &str) -> Result<()> {
-    let buf = msg_type.to_be_bytes();
-    let count = write_count(pipe_w, &buf, MSG_SIZE).await?;
-    if count != MSG_SIZE {
-        return Err(anyhow!("error in send sync message"));
-    }
-
-    match msg_type {
-        SYNC_FAILED => {
-            if let Err(e) = write_count(pipe_w, data_str.as_bytes(), data_str.len()).await {
-                return Err(anyhow!(e).context("error in send message to process"));
-            }
-        }
-        SYNC_DATA => {
-            let length: i32 = data_str.len() as i32;
-            write_count(pipe_w, &length.to_be_bytes(), MSG_SIZE)
-                .await
-                .map_err(|e| anyhow!(e).context("error in send message to process"))?;
-
-            write_count(pipe_w, data_str.as_bytes(), data_str.len())
-                .await
-                .map_err(|e| anyhow!(e).context("error in send message to process"))?;
-        }
-
-        _ => (),
-    };
-
-    Ok(())
-}

src/agent/rustjail/src/utils.rs

@@ -1,119 +0,0 @@
-// Copyright (c) 2021 Ant Group
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-use anyhow::{anyhow, Context, Result};
-use libc::gid_t;
-use libc::uid_t;
-use std::fs::File;
-use std::io::{BufRead, BufReader};
-
-const PASSWD_FILE: &str = "/etc/passwd";
-
-// An entry from /etc/passwd
-#[derive(Debug, PartialEq, PartialOrd)]
-pub struct PasswdEntry {
-    // username
-    pub name: String,
-    // user password
-    pub passwd: String,
-    // user id
-    pub uid: uid_t,
-    // group id
-    pub gid: gid_t,
-    // user Information
-    pub gecos: String,
-    // home directory
-    pub dir: String,
-    // User's Shell
-    pub shell: String,
-}
-
-// get an entry for a given `uid` from `/etc/passwd`
-fn get_entry_by_uid(uid: uid_t, path: &str) -> Result<PasswdEntry> {
-    let file = File::open(path).with_context(|| format!("open file {}", path))?;
-    let mut reader = BufReader::new(file);
-
-    let mut line = String::new();
-    loop {
-        line.clear();
-        match reader.read_line(&mut line) {
-            Ok(0) => return Err(anyhow!(format!("file {} is empty", path))),
-            Ok(_) => (),
-            Err(e) => {
-                return Err(anyhow!(format!(
-                    "failed to read file {} with {:?}",
-                    path, e
-                )))
-            }
-        }
-
-        if line.starts_with('#') {
-            continue;
-        }
-
-        let parts: Vec<&str> = line.split(':').map(|part| part.trim()).collect();
-        if parts.len() != 7 {
-            continue;
-        }
-
-        match parts[2].parse() {
-            Err(_e) => continue,
-            Ok(new_uid) => {
-                if uid != new_uid {
-                    continue;
-                }
-
-                let entry = PasswdEntry {
-                    name: parts[0].to_string(),
-                    passwd: parts[1].to_string(),
-                    uid: new_uid,
-                    gid: parts[3].parse().unwrap_or(0),
-                    gecos: parts[4].to_string(),
-                    dir: parts[5].to_string(),
-                    shell: parts[6].to_string(),
-                };
-
-                return Ok(entry);
-            }
-        }
-    }
-}
-
-pub fn home_dir(uid: uid_t) -> Result<String> {
-    get_entry_by_uid(uid, PASSWD_FILE).map(|entry| entry.dir)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::io::Write;
-    use tempfile::Builder;
-
-    #[test]
-    fn test_get_entry_by_uid() {
-        let tmpdir = Builder::new().tempdir().unwrap();
-        let tmpdir_path = tmpdir.path().to_str().unwrap();
-        let temp_passwd = format!("{}/passwd", tmpdir_path);
-
-        let mut tempf = File::create(temp_passwd.as_str()).unwrap();
-        writeln!(tempf, "root:x:0:0:root:/root0:/bin/bash").unwrap();
-        writeln!(tempf, "root:x:1:0:root:/root1:/bin/bash").unwrap();
-        writeln!(tempf, "#root:x:1:0:root:/rootx:/bin/bash").unwrap();
-        writeln!(tempf, "root:x:2:0:root:/root2:/bin/bash").unwrap();
-        writeln!(tempf, "root:x:3:0:root:/root3").unwrap();
-        writeln!(tempf, "root:x:3:0:root:/root3:/bin/bash").unwrap();
-
-        let entry = get_entry_by_uid(0, temp_passwd.as_str()).unwrap();
-        assert_eq!(entry.dir.as_str(), "/root0");
-
-        let entry = get_entry_by_uid(1, temp_passwd.as_str()).unwrap();
-        assert_eq!(entry.dir.as_str(), "/root1");
-
-        let entry = get_entry_by_uid(2, temp_passwd.as_str()).unwrap();
-        assert_eq!(entry.dir.as_str(), "/root2");
-
-        let entry = get_entry_by_uid(3, temp_passwd.as_str()).unwrap();
-        assert_eq!(entry.dir.as_str(), "/root3");
-    }
-}

src/agent/rustjail/src/validator.rs

@@ -4,21 +4,14 @@
 //
 
 use crate::container::Config;
-use anyhow::{anyhow, Context, Error, Result};
+use anyhow::{anyhow, Result};
+use lazy_static;
 use nix::errno::Errno;
-use oci::{Linux, LinuxIdMapping, LinuxNamespace, Spec};
+use oci::{LinuxIDMapping, LinuxNamespace, Spec};
 use std::collections::HashMap;
 use std::path::{Component, PathBuf};
 
-fn einval() -> Error {
-    anyhow!(nix::Error::from_errno(Errno::EINVAL))
-}
-
-fn get_linux(oci: &Spec) -> Result<&Linux> {
-    oci.linux.as_ref().ok_or_else(einval)
-}
-
-fn contain_namespace(nses: &[LinuxNamespace], key: &str) -> bool {
+fn contain_namespace(nses: &Vec<LinuxNamespace>, key: &str) -> bool {
     for ns in nses {
         if ns.r#type.as_str() == key {
             return true;
@@ -28,28 +21,30 @@ fn contain_namespace(nses: &[LinuxNamespace], key: &str) -> bool {
     false
 }
 
-fn get_namespace_path(nses: &[LinuxNamespace], key: &str) -> Result<String> {
+fn get_namespace_path(nses: &Vec<LinuxNamespace>, key: &str) -> Result<String> {
     for ns in nses {
         if ns.r#type.as_str() == key {
             return Ok(ns.path.clone());
         }
     }
 
-    Err(einval())
+    Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)))
 }
 
 fn rootfs(root: &str) -> Result<()> {
     let path = PathBuf::from(root);
     // not absolute path or not exists
     if !path.exists() || !path.is_absolute() {
-        return Err(einval());
+        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
     }
 
     // symbolic link? ..?
     let mut stack: Vec<String> = Vec::new();
     for c in path.components() {
-        if stack.is_empty() && (c == Component::RootDir || c == Component::ParentDir) {
-            continue;
+        if stack.is_empty() {
+            if c == Component::RootDir || c == Component::ParentDir {
+                continue;
+            }
         }
 
         if c == Component::ParentDir {
@@ -57,11 +52,7 @@ fn rootfs(root: &str) -> Result<()> {
             continue;
         }
 
-        if let Some(v) = c.as_os_str().to_str() {
-            stack.push(v.to_string());
-        } else {
-            return Err(einval());
-        }
+        stack.push(c.as_os_str().to_str().unwrap().to_string());
     }
 
     let mut cleaned = PathBuf::from("/");
@@ -69,37 +60,43 @@ fn rootfs(root: &str) -> Result<()> {
         cleaned.push(e);
     }
 
-    let canon = path.canonicalize().context("canonicalize")?;
+    let canon = path.canonicalize()?;
     if cleaned != canon {
         // There is symbolic in path
-        return Err(einval());
+        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
     }
 
     Ok(())
 }
 
+fn network(_oci: &Spec) -> Result<()> {
+    Ok(())
+}
+
 fn hostname(oci: &Spec) -> Result<()> {
-    if oci.hostname.is_empty() {
+    if oci.hostname.is_empty() || oci.hostname == "".to_string() {
         return Ok(());
     }
 
-    let linux = get_linux(oci)?;
+    if oci.linux.is_none() {
+        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
+    }
+    let linux = oci.linux.as_ref().unwrap();
     if !contain_namespace(&linux.namespaces, "uts") {
-        return Err(einval());
+        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
     }
 
     Ok(())
 }
 
 fn security(oci: &Spec) -> Result<()> {
-    let linux = get_linux(oci)?;
-
-    if linux.masked_paths.is_empty() && linux.readonly_paths.is_empty() {
+    let linux = oci.linux.as_ref().unwrap();
+    if linux.masked_paths.len() == 0 && linux.readonly_paths.len() == 0 {
         return Ok(());
     }
 
     if !contain_namespace(&linux.namespaces, "mount") {
-        return Err(einval());
+        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
     }
 
     // don't care about selinux at present
@@ -107,19 +104,18 @@ fn security(oci: &Spec) -> Result<()> {
     Ok(())
 }
 
-fn idmapping(maps: &[LinuxIdMapping]) -> Result<()> {
+fn idmapping(maps: &Vec<LinuxIDMapping>) -> Result<()> {
     for map in maps {
         if map.size > 0 {
             return Ok(());
         }
     }
 
-    Err(einval())
+    Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)))
 }
 
 fn usernamespace(oci: &Spec) -> Result<()> {
-    let linux = get_linux(oci)?;
-
+    let linux = oci.linux.as_ref().unwrap();
     if contain_namespace(&linux.namespaces, "user") {
         let user_ns = PathBuf::from("/proc/self/ns/user");
         if !user_ns.exists() {
@@ -127,12 +123,12 @@ fn usernamespace(oci: &Spec) -> Result<()> {
         }
         // check if idmappings is correct, at least I saw idmaps
         // with zero size was passed to agent
-        idmapping(&linux.uid_mappings).context("idmapping uid")?;
-        idmapping(&linux.gid_mappings).context("idmapping gid")?;
+        idmapping(&linux.uid_mappings)?;
+        idmapping(&linux.gid_mappings)?;
     } else {
         // no user namespace but idmap
-        if !linux.uid_mappings.is_empty() || !linux.gid_mappings.is_empty() {
-            return Err(einval());
+        if linux.uid_mappings.len() != 0 || linux.gid_mappings.len() != 0 {
+            return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
         }
     }
 
@@ -140,8 +136,7 @@ fn usernamespace(oci: &Spec) -> Result<()> {
 }
 
 fn cgroupnamespace(oci: &Spec) -> Result<()> {
-    let linux = get_linux(oci)?;
-
+    let linux = oci.linux.as_ref().unwrap();
     if contain_namespace(&linux.namespaces, "cgroup") {
         let path = PathBuf::from("/proc/self/ns/cgroup");
         if !path.exists() {
@@ -170,42 +165,43 @@ fn check_host_ns(path: &str) -> Result<()> {
     let cpath = PathBuf::from(path);
     let hpath = PathBuf::from("/proc/self/ns/net");
 
-    let real_hpath = hpath
-        .read_link()
-        .context(format!("read link {:?}", hpath))?;
-    let meta = cpath
-        .symlink_metadata()
-        .context(format!("symlink metadata {:?}", cpath))?;
+    let real_hpath = hpath.read_link()?;
+    let meta = cpath.symlink_metadata()?;
     let file_type = meta.file_type();
 
     if !file_type.is_symlink() {
         return Ok(());
     }
-    let real_cpath = cpath
-        .read_link()
-        .context(format!("read link {:?}", cpath))?;
+    let real_cpath = cpath.read_link()?;
     if real_cpath == real_hpath {
-        return Err(einval());
+        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
     }
 
     Ok(())
 }
 
 fn sysctl(oci: &Spec) -> Result<()> {
-    let linux = get_linux(oci)?;
-
+    let linux = oci.linux.as_ref().unwrap();
     for (key, _) in linux.sysctl.iter() {
         if SYSCTLS.contains_key(key.as_str()) || key.starts_with("fs.mqueue.") {
             if contain_namespace(&linux.namespaces, "ipc") {
                 continue;
             } else {
-                return Err(einval());
+                return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
             }
         }
 
         if key.starts_with("net.") {
-            // the network ns is shared with the guest, don't expect to find it in spec
-            continue;
+            if !contain_namespace(&linux.namespaces, "network") {
+                return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
+            }
+
+            let net = get_namespace_path(&linux.namespaces, "network")?;
+            if net.is_empty() || net == "".to_string() {
+                continue;
+            }
+
+            check_host_ns(net.as_str())?;
         }
 
         if contain_namespace(&linux.namespaces, "uts") {
@@ -214,31 +210,30 @@ fn sysctl(oci: &Spec) -> Result<()> {
             }
 
             if key == "kernel.hostname" {
-                return Err(einval());
+                return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
             }
         }
 
-        return Err(einval());
+        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
     }
     Ok(())
 }
 
 fn rootless_euid_mapping(oci: &Spec) -> Result<()> {
-    let linux = get_linux(oci)?;
-
+    let linux = oci.linux.as_ref().unwrap();
     if !contain_namespace(&linux.namespaces, "user") {
-        return Err(einval());
+        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
     }
 
-    if linux.uid_mappings.is_empty() || linux.gid_mappings.is_empty() {
+    if linux.uid_mappings.len() == 0 || linux.gid_mappings.len() == 0 {
         // rootless containers requires at least one UID/GID mapping
-        return Err(einval());
+        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
     }
 
     Ok(())
 }
 
-fn has_idmapping(maps: &[LinuxIdMapping], id: u32) -> bool {
+fn has_idmapping(maps: &Vec<LinuxIDMapping>, id: u32) -> bool {
     for map in maps {
         if id >= map.container_id && id < map.container_id + map.size {
             return true;
@@ -248,7 +243,7 @@ fn has_idmapping(maps: &[LinuxIdMapping], id: u32) -> bool {
 }
 
 fn rootless_euid_mount(oci: &Spec) -> Result<()> {
-    let linux = get_linux(oci)?;
+    let linux = oci.linux.as_ref().unwrap();
 
     for mnt in oci.mounts.iter() {
         for opt in mnt.options.iter() {
@@ -256,20 +251,21 @@ fn rootless_euid_mount(oci: &Spec) -> Result<()> {
                 let fields: Vec<&str> = opt.split('=').collect();
 
                 if fields.len() != 2 {
-                    return Err(einval());
+                    return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
                 }
 
-                let id = fields[1]
-                    .trim()
-                    .parse::<u32>()
-                    .context(format!("parse field {}", &fields[1]))?;
+                let id = fields[1].trim().parse::<u32>()?;
 
-                if opt.starts_with("uid=") && !has_idmapping(&linux.uid_mappings, id) {
-                    return Err(einval());
+                if opt.starts_with("uid=") {
+                    if !has_idmapping(&linux.uid_mappings, id) {
+                        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
+                    }
                 }
 
-                if opt.starts_with("gid=") && !has_idmapping(&linux.gid_mappings, id) {
-                    return Err(einval());
+                if opt.starts_with("gid=") {
+                    if !has_idmapping(&linux.gid_mappings, id) {
+                        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
+                    }
                 }
             }
         }
@@ -278,305 +274,35 @@ fn rootless_euid_mount(oci: &Spec) -> Result<()> {
 }
 
 fn rootless_euid(oci: &Spec) -> Result<()> {
-    rootless_euid_mapping(oci).context("rootless euid mapping")?;
-    rootless_euid_mount(oci).context("rotless euid mount")?;
+    rootless_euid_mapping(oci)?;
+    rootless_euid_mount(oci)?;
     Ok(())
 }
 
 pub fn validate(conf: &Config) -> Result<()> {
     lazy_static::initialize(&SYSCTLS);
-    let oci = conf.spec.as_ref().ok_or_else(einval)?;
+    let oci = conf.spec.as_ref().unwrap();
 
     if oci.linux.is_none() {
-        return Err(einval());
+        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
     }
 
-    let root = match oci.root.as_ref() {
-        Some(v) => v.path.as_str(),
-        None => return Err(einval()),
-    };
+    if oci.root.is_none() {
+        return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL)));
+    }
+    let root = oci.root.as_ref().unwrap().path.as_str();
 
-    rootfs(root).context("rootfs")?;
-    hostname(oci).context("hostname")?;
-    security(oci).context("security")?;
-    usernamespace(oci).context("usernamespace")?;
-    cgroupnamespace(oci).context("cgroupnamespace")?;
-    sysctl(&oci).context("sysctl")?;
+    rootfs(root)?;
+    network(oci)?;
+    hostname(oci)?;
+    security(oci)?;
+    usernamespace(oci)?;
+    cgroupnamespace(oci)?;
+    sysctl(&oci)?;
 
     if conf.rootless_euid {
-        rootless_euid(oci).context("rootless euid")?;
+        rootless_euid(oci)?;
     }
 
     Ok(())
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use oci::Mount;
-
-    #[test]
-    fn test_namespace() {
-        let namespaces = [
-            LinuxNamespace {
-                r#type: "net".to_owned(),
-                path: "/sys/cgroups/net".to_owned(),
-            },
-            LinuxNamespace {
-                r#type: "uts".to_owned(),
-                path: "/sys/cgroups/uts".to_owned(),
-            },
-        ];
-
-        assert_eq!(contain_namespace(&namespaces, "net"), true);
-        assert_eq!(contain_namespace(&namespaces, "uts"), true);
-
-        assert_eq!(contain_namespace(&namespaces, ""), false);
-        assert_eq!(contain_namespace(&namespaces, "Net"), false);
-        assert_eq!(contain_namespace(&namespaces, "ipc"), false);
-
-        assert_eq!(
-            get_namespace_path(&namespaces, "net").unwrap(),
-            "/sys/cgroups/net"
-        );
-        assert_eq!(
-            get_namespace_path(&namespaces, "uts").unwrap(),
-            "/sys/cgroups/uts"
-        );
-
-        get_namespace_path(&namespaces, "").unwrap_err();
-        get_namespace_path(&namespaces, "Uts").unwrap_err();
-        get_namespace_path(&namespaces, "ipc").unwrap_err();
-    }
-
-    #[test]
-    fn test_rootfs() {
-        rootfs("/_no_exit_fs_xxxxxxxxxxx").unwrap_err();
-        rootfs("sys").unwrap_err();
-        rootfs("/proc/self/root").unwrap_err();
-        rootfs("/proc/self/root/sys").unwrap_err();
-
-        rootfs("/proc/self").unwrap_err();
-        rootfs("/./proc/self").unwrap_err();
-        rootfs("/proc/././self").unwrap_err();
-        rootfs("/proc/.././self").unwrap_err();
-
-        rootfs("/proc/uptime").unwrap();
-        rootfs("/../proc/uptime").unwrap();
-        rootfs("/../../proc/uptime").unwrap();
-        rootfs("/proc/../proc/uptime").unwrap();
-        rootfs("/proc/../../proc/uptime").unwrap();
-    }
-
-    #[test]
-    fn test_hostname() {
-        let mut spec = Spec::default();
-
-        hostname(&spec).unwrap();
-
-        spec.hostname = "a.test.com".to_owned();
-        hostname(&spec).unwrap_err();
-
-        let mut linux = Linux::default();
-        linux.namespaces = vec![
-            LinuxNamespace {
-                r#type: "net".to_owned(),
-                path: "/sys/cgroups/net".to_owned(),
-            },
-            LinuxNamespace {
-                r#type: "uts".to_owned(),
-                path: "/sys/cgroups/uts".to_owned(),
-            },
-        ];
-        spec.linux = Some(linux);
-        hostname(&spec).unwrap();
-    }
-
-    #[test]
-    fn test_security() {
-        let mut spec = Spec::default();
-
-        let linux = Linux::default();
-        spec.linux = Some(linux);
-        security(&spec).unwrap();
-
-        let mut linux = Linux::default();
-        linux.masked_paths.push("/test".to_owned());
-        linux.namespaces = vec![
-            LinuxNamespace {
-                r#type: "net".to_owned(),
-                path: "/sys/cgroups/net".to_owned(),
-            },
-            LinuxNamespace {
-                r#type: "uts".to_owned(),
-                path: "/sys/cgroups/uts".to_owned(),
-            },
-        ];
-        spec.linux = Some(linux);
-        security(&spec).unwrap_err();
-
-        let mut linux = Linux::default();
-        linux.masked_paths.push("/test".to_owned());
-        linux.namespaces = vec![
-            LinuxNamespace {
-                r#type: "net".to_owned(),
-                path: "/sys/cgroups/net".to_owned(),
-            },
-            LinuxNamespace {
-                r#type: "mount".to_owned(),
-                path: "/sys/cgroups/mount".to_owned(),
-            },
-        ];
-        spec.linux = Some(linux);
-        security(&spec).unwrap();
-    }
-
-    #[test]
-    fn test_usernamespace() {
-        let mut spec = Spec::default();
-        usernamespace(&spec).unwrap_err();
-
-        let linux = Linux::default();
-        spec.linux = Some(linux);
-        usernamespace(&spec).unwrap();
-
-        let mut linux = Linux::default();
-        linux.uid_mappings = vec![LinuxIdMapping {
-            container_id: 0,
-            host_id: 1000,
-            size: 0,
-        }];
-        spec.linux = Some(linux);
-        usernamespace(&spec).unwrap_err();
-
-        let mut linux = Linux::default();
-        linux.uid_mappings = vec![LinuxIdMapping {
-            container_id: 0,
-            host_id: 1000,
-            size: 100,
-        }];
-        spec.linux = Some(linux);
-        usernamespace(&spec).unwrap_err();
-    }
-
-    #[test]
-    fn test_rootless_euid() {
-        let mut spec = Spec::default();
-
-        // Test case: without linux
-        rootless_euid_mapping(&spec).unwrap_err();
-        rootless_euid_mount(&spec).unwrap_err();
-
-        // Test case: without user namespace
-        let linux = Linux::default();
-        spec.linux = Some(linux);
-        rootless_euid_mapping(&spec).unwrap_err();
-
-        // Test case: without user namespace
-        let linux = spec.linux.as_mut().unwrap();
-        linux.namespaces = vec![
-            LinuxNamespace {
-                r#type: "net".to_owned(),
-                path: "/sys/cgroups/net".to_owned(),
-            },
-            LinuxNamespace {
-                r#type: "uts".to_owned(),
-                path: "/sys/cgroups/uts".to_owned(),
-            },
-        ];
-        rootless_euid_mapping(&spec).unwrap_err();
-
-        let linux = spec.linux.as_mut().unwrap();
-        linux.namespaces = vec![
-            LinuxNamespace {
-                r#type: "net".to_owned(),
-                path: "/sys/cgroups/net".to_owned(),
-            },
-            LinuxNamespace {
-                r#type: "user".to_owned(),
-                path: "/sys/cgroups/user".to_owned(),
-            },
-        ];
-        linux.uid_mappings = vec![LinuxIdMapping {
-            container_id: 0,
-            host_id: 1000,
-            size: 1000,
-        }];
-        linux.gid_mappings = vec![LinuxIdMapping {
-            container_id: 0,
-            host_id: 1000,
-            size: 1000,
-        }];
-        rootless_euid_mapping(&spec).unwrap();
-
-        spec.mounts.push(Mount {
-            destination: "/app".to_owned(),
-            r#type: "tmpfs".to_owned(),
-            source: "".to_owned(),
-            options: vec!["uid=10000".to_owned()],
-        });
-        rootless_euid_mount(&spec).unwrap_err();
-
-        spec.mounts = vec![
-            (Mount {
-                destination: "/app".to_owned(),
-                r#type: "tmpfs".to_owned(),
-                source: "".to_owned(),
-                options: vec!["uid=500".to_owned(), "gid=500".to_owned()],
-            }),
-        ];
-        rootless_euid(&spec).unwrap();
-    }
-
-    #[test]
-    fn test_check_host_ns() {
-        check_host_ns("/proc/self/ns/net").unwrap_err();
-        check_host_ns("/proc/sys/net/ipv4/tcp_sack").unwrap();
-    }
-
-    #[test]
-    fn test_sysctl() {
-        let mut spec = Spec::default();
-
-        let mut linux = Linux::default();
-        linux.namespaces = vec![LinuxNamespace {
-            r#type: "net".to_owned(),
-            path: "/sys/cgroups/net".to_owned(),
-        }];
-        linux
-            .sysctl
-            .insert("kernel.domainname".to_owned(), "test.com".to_owned());
-        spec.linux = Some(linux);
-        sysctl(&spec).unwrap_err();
-
-        spec.linux
-            .as_mut()
-            .unwrap()
-            .namespaces
-            .push(LinuxNamespace {
-                r#type: "uts".to_owned(),
-                path: "/sys/cgroups/uts".to_owned(),
-            });
-        sysctl(&spec).unwrap();
-    }
-
-    #[test]
-    fn test_validate() {
-        let spec = Spec::default();
-        let mut config = Config {
-            cgroup_name: "container1".to_owned(),
-            use_systemd_cgroup: false,
-            no_pivot_root: true,
-            no_new_keyring: true,
-            rootless_euid: false,
-            rootless_cgroup: false,
-            spec: Some(spec),
-        };
-
-        validate(&config).unwrap_err();
-
-        let linux = Linux::default();
-        config.spec.as_mut().unwrap().linux = Some(linux);
-        validate(&config).unwrap_err();
-    }
-}

src/agent/src/config.rs

@@ -10,7 +10,6 @@ use std::time;
 const DEBUG_CONSOLE_FLAG: &str = "agent.debug_console";
 const DEV_MODE_FLAG: &str = "agent.devmode";
 const LOG_LEVEL_OPTION: &str = "agent.log";
-const SERVER_ADDR_OPTION: &str = "agent.server_addr";
 const HOTPLUG_TIMOUT_OPTION: &str = "agent.hotplug_timeout";
 const DEBUG_CONSOLE_VPORT_OPTION: &str = "agent.debug_console_vport";
 const LOG_VPORT_OPTION: &str = "agent.log_vport";
@@ -22,29 +21,14 @@ const DEFAULT_HOTPLUG_TIMEOUT: time::Duration = time::Duration::from_secs(3);
 const DEFAULT_CONTAINER_PIPE_SIZE: i32 = 0;
 const VSOCK_ADDR: &str = "vsock://-1";
 const VSOCK_PORT: u16 = 1024;
-
-// Environment variables used for development and testing
 const SERVER_ADDR_ENV_VAR: &str = "KATA_AGENT_SERVER_ADDR";
-const LOG_LEVEL_ENV_VAR: &str = "KATA_AGENT_LOG_LEVEL";
 
-const ERR_INVALID_LOG_LEVEL: &str = "invalid log level";
-const ERR_INVALID_LOG_LEVEL_PARAM: &str = "invalid log level parameter";
-const ERR_INVALID_GET_VALUE_PARAM: &str = "expected name=value";
-const ERR_INVALID_GET_VALUE_NO_NAME: &str = "name=value parameter missing name";
-const ERR_INVALID_GET_VALUE_NO_VALUE: &str = "name=value parameter missing value";
-const ERR_INVALID_LOG_LEVEL_KEY: &str = "invalid log level key name";
-
-const ERR_INVALID_HOTPLUG_TIMEOUT: &str = "invalid hotplug timeout parameter";
-const ERR_INVALID_HOTPLUG_TIMEOUT_PARAM: &str = "unable to parse hotplug timeout";
-const ERR_INVALID_HOTPLUG_TIMEOUT_KEY: &str = "invalid hotplug timeout key name";
-
-const ERR_INVALID_CONTAINER_PIPE_SIZE: &str = "invalid container pipe size parameter";
-const ERR_INVALID_CONTAINER_PIPE_SIZE_PARAM: &str = "unable to parse container pipe size";
-const ERR_INVALID_CONTAINER_PIPE_SIZE_KEY: &str = "invalid container pipe size key name";
-const ERR_INVALID_CONTAINER_PIPE_NEGATIVE: &str = "container pipe size should not be negative";
+// FIXME: unused
+const TRACE_MODE_FLAG: &str = "agent.trace";
+const USE_VSOCK_FLAG: &str = "agent.use_vsock";
 
 #[derive(Debug)]
-pub struct AgentConfig {
+pub struct agentConfig {
     pub debug_console: bool,
     pub dev_mode: bool,
     pub log_level: slog::Level,
@@ -86,9 +70,9 @@ macro_rules! parse_cmdline_param {
     };
 }
 
-impl AgentConfig {
-    pub fn new() -> AgentConfig {
-        AgentConfig {
+impl agentConfig {
+    pub fn new() -> agentConfig {
+        agentConfig {
             debug_console: false,
             dev_mode: false,
             log_level: DEFAULT_LOG_LEVEL,
@@ -111,12 +95,6 @@ impl AgentConfig {
 
             // parse cmdline options
             parse_cmdline_param!(param, LOG_LEVEL_OPTION, self.log_level, get_log_level);
-            parse_cmdline_param!(
-                param,
-                SERVER_ADDR_OPTION,
-                self.server_addr,
-                get_string_value
-            );
 
             // ensure the timeout is a positive value
             parse_cmdline_param!(
@@ -124,7 +102,7 @@ impl AgentConfig {
                 HOTPLUG_TIMOUT_OPTION,
                 self.hotplug_timeout,
                 get_hotplug_timeout,
-                |hotplug_timeout: time::Duration| hotplug_timeout.as_secs() > 0
+                |hotplugTimeout: time::Duration| hotplugTimeout.as_secs() > 0
             );
 
             // vsock port should be positive values
@@ -161,18 +139,12 @@ impl AgentConfig {
             self.server_addr = addr;
         }
 
-        if let Ok(addr) = env::var(LOG_LEVEL_ENV_VAR) {
-            if let Ok(level) = logrus_to_slog_level(&addr) {
-                self.log_level = level;
-            }
-        }
-
         Ok(())
     }
 }
 
 fn get_vsock_port(p: &str) -> Result<i32> {
-    let fields: Vec<&str> = p.split('=').collect();
+    let fields: Vec<&str> = p.split("=").collect();
     if fields.len() != 2 {
         return Err(anyhow!("invalid port parameter"));
     }
@@ -200,7 +172,7 @@ fn logrus_to_slog_level(logrus_level: &str) -> Result<slog::Level> {
         "trace" => slog::Level::Trace,
 
         _ => {
-            return Err(anyhow!(ERR_INVALID_LOG_LEVEL));
+            return Err(anyhow!("invalid log level"));
         }
     };
 
@@ -208,41 +180,41 @@ fn logrus_to_slog_level(logrus_level: &str) -> Result<slog::Level> {
 }
 
 fn get_log_level(param: &str) -> Result<slog::Level> {
-    let fields: Vec<&str> = param.split('=').collect();
+    let fields: Vec<&str> = param.split("=").collect();
 
     if fields.len() != 2 {
-        return Err(anyhow!(ERR_INVALID_LOG_LEVEL_PARAM));
+        return Err(anyhow!("invalid log level parameter"));
     }
 
     if fields[0] != LOG_LEVEL_OPTION {
-        Err(anyhow!(ERR_INVALID_LOG_LEVEL_KEY))
+        Err(anyhow!("invalid log level key name"))
     } else {
         Ok(logrus_to_slog_level(fields[1])?)
     }
 }
 
 fn get_hotplug_timeout(param: &str) -> Result<time::Duration> {
-    let fields: Vec<&str> = param.split('=').collect();
+    let fields: Vec<&str> = param.split("=").collect();
 
     if fields.len() != 2 {
-        return Err(anyhow!(ERR_INVALID_HOTPLUG_TIMEOUT));
+        return Err(anyhow!("invalid hotplug timeout parameter"));
     }
 
     let key = fields[0];
     if key != HOTPLUG_TIMOUT_OPTION {
-        return Err(anyhow!(ERR_INVALID_HOTPLUG_TIMEOUT_KEY));
+        return Err(anyhow!("invalid hotplug timeout key name"));
     }
 
     let value = fields[1].parse::<u64>();
     if value.is_err() {
-        return Err(anyhow!(ERR_INVALID_HOTPLUG_TIMEOUT_PARAM));
+        return Err(anyhow!("unable to parse hotplug timeout"));
     }
 
     Ok(time::Duration::from_secs(value.unwrap()))
 }
 
 fn get_bool_value(param: &str) -> Result<bool> {
-    let fields: Vec<&str> = param.split('=').collect();
+    let fields: Vec<&str> = param.split("=").collect();
 
     if fields.len() != 2 {
         return Ok(false);
@@ -253,58 +225,36 @@ fn get_bool_value(param: &str) -> Result<bool> {
     // first try to parse as bool value
     v.parse::<bool>().or_else(|_err1| {
         // then try to parse as integer value
-        v.parse::<u64>().or(Ok(0)).map(|v| !matches!(v, 0))
+        v.parse::<u64>().or_else(|_err2| Ok(0)).and_then(|v| {
+            // only `0` returns false, otherwise returns true
+            Ok(match v {
+                0 => false,
+                _ => true,
+            })
+        })
     })
 }
 
-// Return the value from a "name=value" string.
-//
-// Note:
-//
-// - A name *and* a value is required.
-// - A value can contain any number of equal signs.
-// - We could/should maybe check if the name is pure whitespace
-//   since this is considered to be invalid.
-fn get_string_value(param: &str) -> Result<String> {
-    let fields: Vec<&str> = param.split('=').collect();
-
-    if fields.len() < 2 {
-        return Err(anyhow!(ERR_INVALID_GET_VALUE_PARAM));
-    }
-
-    // We need name (but the value can be blank)
-    if fields[0].is_empty() {
-        return Err(anyhow!(ERR_INVALID_GET_VALUE_NO_NAME));
-    }
-
-    let value = fields[1..].join("=");
-    if value.is_empty() {
-        return Err(anyhow!(ERR_INVALID_GET_VALUE_NO_VALUE));
-    }
-
-    Ok(value)
-}
-
 fn get_container_pipe_size(param: &str) -> Result<i32> {
-    let fields: Vec<&str> = param.split('=').collect();
+    let fields: Vec<&str> = param.split("=").collect();
 
     if fields.len() != 2 {
-        return Err(anyhow!(ERR_INVALID_CONTAINER_PIPE_SIZE));
+        return Err(anyhow!("invalid container pipe size parameter"));
     }
 
     let key = fields[0];
     if key != CONTAINER_PIPE_SIZE_OPTION {
-        return Err(anyhow!(ERR_INVALID_CONTAINER_PIPE_SIZE_KEY));
+        return Err(anyhow!("invalid container pipe size key name"));
     }
 
     let res = fields[1].parse::<i32>();
     if res.is_err() {
-        return Err(anyhow!(ERR_INVALID_CONTAINER_PIPE_SIZE_PARAM));
+        return Err(anyhow!("unable to parse container pipe size"));
     }
 
     let value = res.unwrap();
     if value < 0 {
-        return Err(anyhow!(ERR_INVALID_CONTAINER_PIPE_NEGATIVE));
+        return Err(anyhow!("container pipe size should not be negative"));
     }
 
     Ok(value)
@@ -319,6 +269,19 @@ mod tests {
     use std::time;
     use tempfile::tempdir;
 
+    const ERR_INVALID_LOG_LEVEL: &str = "invalid log level";
+    const ERR_INVALID_LOG_LEVEL_PARAM: &str = "invalid log level parameter";
+    const ERR_INVALID_LOG_LEVEL_KEY: &str = "invalid log level key name";
+
+    const ERR_INVALID_HOTPLUG_TIMEOUT: &str = "invalid hotplug timeout parameter";
+    const ERR_INVALID_HOTPLUG_TIMEOUT_PARAM: &str = "unable to parse hotplug timeout";
+    const ERR_INVALID_HOTPLUG_TIMEOUT_KEY: &str = "invalid hotplug timeout key name";
+
+    const ERR_INVALID_CONTAINER_PIPE_SIZE: &str = "invalid container pipe size parameter";
+    const ERR_INVALID_CONTAINER_PIPE_SIZE_PARAM: &str = "unable to parse container pipe size";
+    const ERR_INVALID_CONTAINER_PIPE_SIZE_KEY: &str = "invalid container pipe size key name";
+    const ERR_INVALID_CONTAINER_PIPE_NEGATIVE: &str = "container pipe size should not be negative";
+
     // helper function to make errors less crazy-long
     fn make_err(desc: &str) -> Error {
         anyhow!(desc.to_string())
@@ -334,25 +297,22 @@ mod tests {
             if $expected_result.is_ok() {
                 let expected_level = $expected_result.as_ref().unwrap();
                 let actual_level = $actual_result.unwrap();
-                assert!(*expected_level == actual_level, "{}", $msg);
+                assert!(*expected_level == actual_level, $msg);
             } else {
                 let expected_error = $expected_result.as_ref().unwrap_err();
+                let actual_error = $actual_result.unwrap_err();
+
                 let expected_error_msg = format!("{:?}", expected_error);
+                let actual_error_msg = format!("{:?}", actual_error);
 
-                if let Err(actual_error) = $actual_result {
-                    let actual_error_msg = format!("{:?}", actual_error);
-
-                    assert!(expected_error_msg == actual_error_msg, "{}", $msg);
-                } else {
-                    assert!(expected_error_msg == "expected error, got OK", "{}", $msg);
-                }
+                assert!(expected_error_msg == actual_error_msg, $msg);
             }
         };
     }
 
     #[test]
     fn test_new() {
-        let config = AgentConfig::new();
+        let config = agentConfig::new();
         assert_eq!(config.debug_console, false);
         assert_eq!(config.dev_mode, false);
         assert_eq!(config.log_level, DEFAULT_LOG_LEVEL);
@@ -361,550 +321,297 @@ mod tests {
 
     #[test]
     fn test_parse_cmdline() {
-        const TEST_SERVER_ADDR: &str = "vsock://-1:1024";
-
         #[derive(Debug)]
         struct TestData<'a> {
             contents: &'a str,
-            env_vars: Vec<&'a str>,
             debug_console: bool,
             dev_mode: bool,
             log_level: slog::Level,
             hotplug_timeout: time::Duration,
             container_pipe_size: i32,
-            server_addr: &'a str,
             unified_cgroup_hierarchy: bool,
         }
 
         let tests = &[
             TestData {
                 contents: "agent.debug_consolex agent.devmode",
-                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "agent.debug_console agent.devmodex",
-                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "agent.logx=debug",
-                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "agent.log=debug",
-                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: false,
                 log_level: slog::Level::Debug,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
-                unified_cgroup_hierarchy: false,
-            },
-            TestData {
-                contents: "agent.log=debug",
-                env_vars: vec!["KATA_AGENT_LOG_LEVEL=trace"],
-                debug_console: false,
-                dev_mode: false,
-                log_level: slog::Level::Trace,
-                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
-                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "",
-                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "foo",
-                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "foo bar",
-                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "foo bar",
-                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "foo agent bar",
-                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "foo debug_console agent bar devmode",
-                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "agent.debug_console",
-                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "   agent.debug_console ",
-                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "agent.debug_console foo",
-                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: " agent.debug_console foo",
-                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "foo agent.debug_console bar",
-                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "foo agent.debug_console",
-                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "foo agent.debug_console ",
-                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: false,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "agent.devmode",
-                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "   agent.devmode ",
-                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "agent.devmode foo",
-                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: " agent.devmode foo",
-                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "foo agent.devmode bar",
-                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "foo agent.devmode",
-                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "foo agent.devmode ",
-                env_vars: Vec::new(),
                 debug_console: false,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "agent.devmode agent.debug_console",
-                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "agent.devmode agent.debug_console agent.hotplug_timeout=100 agent.unified_cgroup_hierarchy=a",
-                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: time::Duration::from_secs(100),
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "agent.devmode agent.debug_console agent.hotplug_timeout=0 agent.unified_cgroup_hierarchy=11",
-                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: true,
             },
             TestData {
                 contents: "agent.devmode agent.debug_console agent.container_pipe_size=2097152 agent.unified_cgroup_hierarchy=false",
-                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: 2097152,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "agent.devmode agent.debug_console agent.container_pipe_size=100 agent.unified_cgroup_hierarchy=true",
-                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: 100,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: true,
             },
             TestData {
                 contents: "agent.devmode agent.debug_console agent.container_pipe_size=0 agent.unified_cgroup_hierarchy=0",
-                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: false,
             },
             TestData {
                 contents: "agent.devmode agent.debug_console agent.container_pip_siz=100 agent.unified_cgroup_hierarchy=1",
-                env_vars: Vec::new(),
                 debug_console: true,
                 dev_mode: true,
                 log_level: DEFAULT_LOG_LEVEL,
                 hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
                 container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
                 unified_cgroup_hierarchy: true,
             },
-            TestData {
-                contents: "",
-                env_vars: Vec::new(),
-                debug_console: false,
-                dev_mode: false,
-                log_level: DEFAULT_LOG_LEVEL,
-                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
-                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
-                unified_cgroup_hierarchy: false,
-            },
-            TestData {
-                contents: "",
-                env_vars: vec!["KATA_AGENT_SERVER_ADDR=foo"],
-                debug_console: false,
-                dev_mode: false,
-                log_level: DEFAULT_LOG_LEVEL,
-                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
-                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: "foo",
-                unified_cgroup_hierarchy: false,
-            },
-            TestData {
-                contents: "",
-                env_vars: vec!["KATA_AGENT_SERVER_ADDR=="],
-                debug_console: false,
-                dev_mode: false,
-                log_level: DEFAULT_LOG_LEVEL,
-                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
-                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: "=",
-                unified_cgroup_hierarchy: false,
-            },
-            TestData {
-                contents: "",
-                env_vars: vec!["KATA_AGENT_SERVER_ADDR==foo"],
-                debug_console: false,
-                dev_mode: false,
-                log_level: DEFAULT_LOG_LEVEL,
-                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
-                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: "=foo",
-                unified_cgroup_hierarchy: false,
-            },
-            TestData {
-                contents: "",
-                env_vars: vec!["KATA_AGENT_SERVER_ADDR=foo=bar=baz="],
-                debug_console: false,
-                dev_mode: false,
-                log_level: DEFAULT_LOG_LEVEL,
-                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
-                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: "foo=bar=baz=",
-                unified_cgroup_hierarchy: false,
-            },
-            TestData {
-                contents: "",
-                env_vars: vec!["KATA_AGENT_SERVER_ADDR=unix:///tmp/foo.socket"],
-                debug_console: false,
-                dev_mode: false,
-                log_level: DEFAULT_LOG_LEVEL,
-                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
-                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: "unix:///tmp/foo.socket",
-                unified_cgroup_hierarchy: false,
-            },
-            TestData {
-                contents: "",
-                env_vars: vec!["KATA_AGENT_SERVER_ADDR=unix://@/tmp/foo.socket"],
-                debug_console: false,
-                dev_mode: false,
-                log_level: DEFAULT_LOG_LEVEL,
-                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
-                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: "unix://@/tmp/foo.socket",
-                unified_cgroup_hierarchy: false,
-            },
-            TestData {
-                contents: "",
-                env_vars: vec!["KATA_AGENT_LOG_LEVEL="],
-                debug_console: false,
-                dev_mode: false,
-                log_level: DEFAULT_LOG_LEVEL,
-                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
-                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
-                unified_cgroup_hierarchy: false,
-            },
-            TestData {
-                contents: "",
-                env_vars: vec!["KATA_AGENT_LOG_LEVEL=invalid"],
-                debug_console: false,
-                dev_mode: false,
-                log_level: DEFAULT_LOG_LEVEL,
-                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
-                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
-                unified_cgroup_hierarchy: false,
-            },
-            TestData {
-                contents: "",
-                env_vars: vec!["KATA_AGENT_LOG_LEVEL=debug"],
-                debug_console: false,
-                dev_mode: false,
-                log_level: slog::Level::Debug,
-                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
-                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
-                unified_cgroup_hierarchy: false,
-            },
-            TestData {
-                contents: "",
-                env_vars: vec!["KATA_AGENT_LOG_LEVEL=debugger"],
-                debug_console: false,
-                dev_mode: false,
-                log_level: DEFAULT_LOG_LEVEL,
-                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
-                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
-                unified_cgroup_hierarchy: false,
-            },
-            TestData {
-                contents: "server_addr=unix:///tmp/foo.socket",
-                env_vars: Vec::new(),
-                debug_console: false,
-                dev_mode: false,
-                log_level: DEFAULT_LOG_LEVEL,
-                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
-                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
-                unified_cgroup_hierarchy: false,
-            },
-            TestData {
-                contents: "agent.server_address=unix:///tmp/foo.socket",
-                env_vars: Vec::new(),
-                debug_console: false,
-                dev_mode: false,
-                log_level: DEFAULT_LOG_LEVEL,
-                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
-                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: TEST_SERVER_ADDR,
-                unified_cgroup_hierarchy: false,
-            },
-            TestData {
-                contents: "agent.server_addr=unix:///tmp/foo.socket",
-                env_vars: Vec::new(),
-                debug_console: false,
-                dev_mode: false,
-                log_level: DEFAULT_LOG_LEVEL,
-                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
-                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: "unix:///tmp/foo.socket",
-                unified_cgroup_hierarchy: false,
-            },
-            TestData {
-                contents: " agent.server_addr=unix:///tmp/foo.socket",
-                env_vars: Vec::new(),
-                debug_console: false,
-                dev_mode: false,
-                log_level: DEFAULT_LOG_LEVEL,
-                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
-                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: "unix:///tmp/foo.socket",
-                unified_cgroup_hierarchy: false,
-            },
-            TestData {
-                contents: " agent.server_addr=unix:///tmp/foo.socket a",
-                env_vars: Vec::new(),
-                debug_console: false,
-                dev_mode: false,
-                log_level: DEFAULT_LOG_LEVEL,
-                hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
-                container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-                server_addr: "unix:///tmp/foo.socket",
-                unified_cgroup_hierarchy: false,
-            },
         ];
 
         let dir = tempdir().expect("failed to create tmpdir");
@@ -914,12 +621,11 @@ mod tests {
 
         let filename = file_path.to_str().expect("failed to create filename");
 
-        let mut config = AgentConfig::new();
+        let mut config = agentConfig::new();
         let result = config.parse_cmdline(&filename.to_owned());
         assert!(result.is_err());
 
-        // Now, test various combinations of file contents and environment
-        // variables.
+        // Now, test various combinations of file contents
         for (i, d) in tests.iter().enumerate() {
             let msg = format!("test[{}]: {:?}", i, d);
 
@@ -928,25 +634,12 @@ mod tests {
             let filename = file_path.to_str().expect("failed to create filename");
 
             let mut file =
-                File::create(filename).unwrap_or_else(|_| panic!("{}: failed to create file", msg));
+                File::create(filename).expect(&format!("{}: failed to create file", msg));
 
             file.write_all(d.contents.as_bytes())
-                .unwrap_or_else(|_| panic!("{}: failed to write file contents", msg));
+                .expect(&format!("{}: failed to write file contents", msg));
 
-            let mut vars_to_unset = Vec::new();
-
-            for v in &d.env_vars {
-                let fields: Vec<&str> = v.split('=').collect();
-
-                let name = fields[0];
-                let value = fields[1..].join("=");
-
-                env::set_var(name, value);
-
-                vars_to_unset.push(name);
-            }
-
-            let mut config = AgentConfig::new();
+            let mut config = agentConfig::new();
             assert_eq!(config.debug_console, false, "{}", msg);
             assert_eq!(config.dev_mode, false, "{}", msg);
             assert_eq!(config.unified_cgroup_hierarchy, false, "{}", msg);
@@ -957,7 +650,6 @@ mod tests {
                 msg
             );
             assert_eq!(config.container_pipe_size, 0, "{}", msg);
-            assert_eq!(config.server_addr, TEST_SERVER_ADDR, "{}", msg);
 
             let result = config.parse_cmdline(filename);
             assert!(result.is_ok(), "{}", msg);
@@ -972,11 +664,6 @@ mod tests {
             assert_eq!(d.log_level, config.log_level, "{}", msg);
             assert_eq!(d.hotplug_timeout, config.hotplug_timeout, "{}", msg);
             assert_eq!(d.container_pipe_size, config.container_pipe_size, "{}", msg);
-            assert_eq!(d.server_addr, config.server_addr, "{}", msg);
-
-            for v in vars_to_unset {
-                env::remove_var(v);
-            }
         }
     }
 
@@ -1050,7 +737,7 @@ mod tests {
 
             let msg = format!("{}: result: {:?}", msg, result);
 
-            assert_result!(d.result, result, msg);
+            assert_result!(d.result, result, format!("{}", msg));
         }
     }
 
@@ -1144,7 +831,7 @@ mod tests {
 
             let msg = format!("{}: result: {:?}", msg, result);
 
-            assert_result!(d.result, result, msg);
+            assert_result!(d.result, result, format!("{}", msg));
         }
     }
 
@@ -1214,7 +901,7 @@ mod tests {
 
             let msg = format!("{}: result: {:?}", msg, result);
 
-            assert_result!(d.result, result, msg);
+            assert_result!(d.result, result, format!("{}", msg));
         }
     }
 
@@ -1288,85 +975,7 @@ mod tests {
 
             let msg = format!("{}: result: {:?}", msg, result);
 
-            assert_result!(d.result, result, msg);
-        }
-    }
-
-    #[test]
-    fn test_get_string_value() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            param: &'a str,
-            result: Result<String>,
-        }
-
-        let tests = &[
-            TestData {
-                param: "",
-                result: Err(make_err(ERR_INVALID_GET_VALUE_PARAM)),
-            },
-            TestData {
-                param: "=",
-                result: Err(make_err(ERR_INVALID_GET_VALUE_NO_NAME)),
-            },
-            TestData {
-                param: "==",
-                result: Err(make_err(ERR_INVALID_GET_VALUE_NO_NAME)),
-            },
-            TestData {
-                param: "x=",
-                result: Err(make_err(ERR_INVALID_GET_VALUE_NO_VALUE)),
-            },
-            TestData {
-                param: "x==",
-                result: Ok("=".into()),
-            },
-            TestData {
-                param: "x===",
-                result: Ok("==".into()),
-            },
-            TestData {
-                param: "x==x",
-                result: Ok("=x".into()),
-            },
-            TestData {
-                param: "x=x",
-                result: Ok("x".into()),
-            },
-            TestData {
-                param: "x=x=",
-                result: Ok("x=".into()),
-            },
-            TestData {
-                param: "x=x=x",
-                result: Ok("x=x".into()),
-            },
-            TestData {
-                param: "foo=bar",
-                result: Ok("bar".into()),
-            },
-            TestData {
-                param: "x= =",
-                result: Ok(" =".into()),
-            },
-            TestData {
-                param: "x= =",
-                result: Ok(" =".into()),
-            },
-            TestData {
-                param: "x= = ",
-                result: Ok(" = ".into()),
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let result = get_string_value(d.param);
-
-            let msg = format!("{}: result: {:?}", msg, result);
-
-            assert_result!(d.result, result, msg);
+            assert_result!(d.result, result, format!("{}", msg));
         }
     }
 }