Merge pull request #4591 from fidencio/2.5.0-rc0-branch-bump

# Kata Containers 2.5.0-rc0

.github/workflows/add-pr-sizing-label.yaml

@@ -33,6 +33,8 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.KATA_GITHUB_ACTIONS_PR_SIZE_TOKEN }}
         run: |
           pr=${{ github.event.number }}
+          # Removing man-db, workflow kept failing, fixes: #4480
+          sudo apt -y remove --purge man-db
           sudo apt -y install diffstat patchutils
 
           pr-add-size-label.sh -p "$pr"

.github/workflows/commit-message-check.yaml

@@ -63,7 +63,8 @@ jobs:
         #   the entire commit message.
         #
         # - Body lines *can* be longer than the maximum if they start
-        #   with a non-alphabetic character.
+        #   with a non-alphabetic character or if there is no whitespace in
+        #   the line.
         #
         #   This allows stack traces, log files snippets, emails, long URLs,
         #   etc to be specified. Some of these naturally "work" as they start
@@ -74,7 +75,7 @@ jobs:
         #
         # - A SoB comment can be any length (as it is unreasonable to penalise
         #   people with long names/email addresses :)
-        pattern: '^.+(\n([a-zA-Z].{0,149}|[^a-zA-Z\n].*|Signed-off-by:.*|))+$'
+        pattern: '^.+(\n([a-zA-Z].{0,72}|[^a-zA-Z\n].*|[^\s\n]*|Signed-off-by:.*|))+$'
         error: 'Body line too long (max 72)'
         post_error: ${{ env.error_msg }}
 

.github/workflows/docs-url-alive-check.yaml

@@ -14,31 +14,31 @@ jobs:
       target_branch: ${{ github.base_ref }}
     steps:
     - name: Install Go
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      if: github.repository_owner == 'kata-containers'
       uses: actions/setup-go@v2
       with:
         go-version: ${{ matrix.go-version }}
       env:
         GOPATH: ${{ runner.workspace }}/kata-containers
     - name: Set env
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      if: github.repository_owner == 'kata-containers'
       run: |
         echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
         echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
     - name: Checkout code
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      if: github.repository_owner == 'kata-containers'
       uses: actions/checkout@v2
       with:
         fetch-depth: 0
         path: ./src/github.com/${{ github.repository }}
     - name: Setup
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      if: github.repository_owner == 'kata-containers'
       run: |
         cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
       env:
         GOPATH: ${{ runner.workspace }}/kata-containers
     # docs url alive check
     - name: Docs URL Alive Check
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      if: github.repository_owner == 'kata-containers'
       run: |
         cd ${GOPATH}/src/github.com/${{ github.repository }} && make docs-url-alive-check

.github/workflows/kata-deploy-push.yaml

@@ -24,6 +24,7 @@ jobs:
           - firecracker
           - rootfs-image
           - rootfs-initrd
+          - virtiofsd
     steps:
       - uses: actions/checkout@v2
       - name: Install docker

.github/workflows/kata-deploy-test.yaml

@@ -1,4 +1,5 @@
 on:
+  workflow_dispatch: # this is used to trigger the workflow on non-main branches
   issue_comment:
     types: [created, edited]
 
@@ -47,6 +48,7 @@ jobs:
           - rootfs-image
           - rootfs-initrd
           - shim-v2
+          - virtiofsd
     steps:
       - name: get-PR-ref
         id: get-PR-ref

.github/workflows/release.yaml

@@ -17,6 +17,7 @@ jobs:
           - rootfs-image
           - rootfs-initrd
           - shim-v2
+          - virtiofsd
     steps:
       - uses: actions/checkout@v2
       - name: Install docker

.github/workflows/snap-release.yaml

@@ -19,6 +19,8 @@ jobs:
 
       - name: Build snap
         run: |
+          # Removing man-db, workflow kept failing, fixes: #4480
+          sudo apt -y remove --purge man-db
           sudo apt-get install -y git git-extras
           kata_url="https://github.com/kata-containers/kata-containers"
           latest_version=$(git ls-remote --tags ${kata_url}  | egrep -o "refs.*" | egrep -v "\-alpha|\-rc|{}" | egrep -o "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+" | sort -V -r | head -1)
@@ -26,7 +28,7 @@ jobs:
           # Check semantic versioning format (x.y.z) and if the current tag is the latest tag
           if echo "${current_version}" | grep -q "^[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+$" && echo -e "$latest_version\n$current_version" | sort -C -V; then
             # Current version is the latest version, build it
-            snapcraft -d snap --destructive-mode
+            snapcraft snap --debug --destructive-mode
           fi
 
       - name: Upload snap

.github/workflows/snap.yaml

@@ -24,4 +24,4 @@ jobs:
       - name: Build snap
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         run: |
-          snapcraft -d snap --destructive-mode
+          snapcraft snap --debug --destructive-mode

.gitignore

@@ -10,4 +10,5 @@ src/agent/kata-agent.service
 src/agent/protocols/src/*.rs
 !src/agent/protocols/src/lib.rs
 build
+src/tools/log-parser/kata-log-parser
 

Makefile

@@ -15,6 +15,7 @@ TOOLS =
 TOOLS += agent-ctl
 TOOLS += trace-forwarder
 TOOLS += runk
+TOOLS += log-parser
 
 STANDARD_TARGETS = build check clean install test vendor
 

README.md

@@ -140,7 +140,7 @@ The table below lists the remaining parts of the project:
 
 Kata Containers is now
 [available natively for most distributions](docs/install/README.md#packaged-installation-methods).
-However, packaging scripts and metadata are still used to generate snap and GitHub releases. See
+However, packaging scripts and metadata are still used to generate [snap](snap/local) and GitHub releases. See
 the [components](#components) section for further details.
 
 ## Glossary of Terms

VERSION

@@ -1 +1 @@
-2.5.0-alpha1
+2.5.0-rc0

ci/darwin-test.sh

@@ -11,10 +11,10 @@ runtimedir=$cidir/../src/runtime
 
 build_working_packages() {
 	# working packages:
-	device_api=$runtimedir/virtcontainers/device/api
-	device_config=$runtimedir/virtcontainers/device/config
-	device_drivers=$runtimedir/virtcontainers/device/drivers
-	device_manager=$runtimedir/virtcontainers/device/manager
+	device_api=$runtimedir/pkg/device/api
+	device_config=$runtimedir/pkg/device/config
+	device_drivers=$runtimedir/pkg/device/drivers
+	device_manager=$runtimedir/pkg/device/manager
 	rc_pkg_dir=$runtimedir/pkg/resourcecontrol/
 	utils_pkg_dir=$runtimedir/virtcontainers/utils
 

ci/go-test.sh

@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2020 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-
-set -e
-
-cidir=$(dirname "$0")
-source "${cidir}/lib.sh"
-
-run_go_test

ci/lib.sh

@@ -18,6 +18,13 @@ clone_tests_repo()
 {
 	if [ -d "$tests_repo_dir" ]; then
 		[ -n "${CI:-}" ] && return
+		# git config --global --add safe.directory will always append
+		# the target to .gitconfig without checking the existence of
+		# the target, so it's better to check it before adding the target repo.
+		local sd="$(git config --global --get safe.directory ${tests_repo_dir} || true)"
+		if [ -z "${sd}" ]; then
+			git config --global --add safe.directory ${tests_repo_dir}
+		fi
 		pushd "${tests_repo_dir}"
 		git checkout "${branch}"
 		git pull
@@ -39,12 +46,6 @@ run_static_checks()
 	bash "$tests_repo_dir/.ci/static-checks.sh" "$@"
 }
 
-run_go_test()
-{
-	clone_tests_repo
-	bash "$tests_repo_dir/.ci/go-test.sh"
-}
-
 run_docs_url_alive_check()
 {
 	clone_tests_repo

docs/Developer-Guide.md

@@ -116,7 +116,7 @@ detailed below.
 The Kata logs appear in the `containerd` log files, along with logs from `containerd` itself.
 
 For more information about `containerd` debug, please see the
-[`containerd` documentation](https://github.com/containerd/containerd/blob/master/docs/getting-started.md).
+[`containerd` documentation](https://github.com/containerd/containerd/blob/main/docs/getting-started.md).
 
 #### Enabling full `containerd` debug
 
@@ -465,7 +465,7 @@ script and paste its output directly into a
 > [runtime](../src/runtime) repository.
 
 To perform analysis on Kata logs, use the
-[`kata-log-parser`](https://github.com/kata-containers/tests/tree/main/cmd/log-parser)
+[`kata-log-parser`](../src/tools/log-parser)
 tool, which can convert the logs into formats (e.g. JSON, TOML, XML, and YAML).
 
 See [Set up a debug console](#set-up-a-debug-console).
@@ -700,11 +700,11 @@ options to have the kernel boot messages logged into the system journal.
 For generic information on enabling debug in the configuration file, see the
 [Enable full debug](#enable-full-debug) section.
 
-The kernel boot messages will appear in the `containerd` or `CRI-O` log appropriately,
+The kernel boot messages will appear in the `kata` logs (and in the `containerd` or `CRI-O` log appropriately).
 such as:
 
 ```bash
-$ sudo journalctl -t containerd
+$ sudo journalctl -t kata
 -- Logs begin at Thu 2020-02-13 16:20:40 UTC, end at Thu 2020-02-13 16:30:23 UTC. --
 ...
 time="2020-09-15T14:56:23.095113803+08:00" level=debug msg="reading guest console" console-protocol=unix console-url=/run/vc/vm/ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791/console.sock pid=107642 sandbox=ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791 source=virtcontainers subsystem=sandbox vmconsole="[    0.395399] brd: module loaded"
@@ -714,3 +714,4 @@ time="2020-09-15T14:56:23.105268162+08:00" level=debug msg="reading guest consol
 time="2020-09-15T14:56:23.121121598+08:00" level=debug msg="reading guest console" console-protocol=unix console-url=/run/vc/vm/ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791/console.sock pid=107642 sandbox=ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791 source=virtcontainers subsystem=sandbox vmconsole="[    0.421324] memmap_init_zone_device initialised 32768 pages in 12ms"
 ...
 ```
+Refer to the [kata-log-parser documentation](../src/tools/log-parser/README.md) which is useful to fetch these.

docs/Release-Process.md

@@ -4,11 +4,11 @@
 ## Requirements
 
 - [hub](https://github.com/github/hub)
-  * Using an [application token](https://github.com/settings/tokens) is required for hub.
+  * Using an [application token](https://github.com/settings/tokens) is required for hub (set to a GITHUB_TOKEN environment variable).
 
 - GitHub permissions to push tags and create releases in Kata repositories.
 
-- GPG configured to sign git tags. https://help.github.com/articles/generating-a-new-gpg-key/
+- GPG configured to sign git tags. https://docs.github.com/en/authentication/managing-commit-signature-verification/generating-a-new-gpg-key
 
 - You should configure your GitHub to use your ssh keys (to push to branches). See https://help.github.com/articles/adding-a-new-ssh-key-to-your-github-account/.
     * As an alternative, configure hub to push and fork with HTTPS, `git config --global hub.protocol https` (Not tested yet) *
@@ -48,7 +48,7 @@
 ### Merge all bump version Pull requests
 
   - The above step will create a GitHub pull request in the Kata projects. Trigger the CI using `/test` command on each bump Pull request.
-  - Trigger the test-kata-deploy workflow on the kata-containers repository bump Pull request using `/test_kata_deploy` (monitor under the "action" tab).
+  - Trigger the `test-kata-deploy` workflow which is under the `Actions` tab on the repository GitHub page (make sure to select the correct branch and validate it passes).
   - Check any failures and fix if needed.
   - Work with the Kata approvers to verify that the CI works and the pull requests are merged.
 

docs/Unit-Test-Advice.md

@@ -320,7 +320,7 @@ mod tests {
 
 ## Test user
 
-[Unit tests are run *twice*](https://github.com/kata-containers/tests/blob/main/.ci/go-test.sh):
+[Unit tests are run *twice*](../src/runtime/go-test.sh):
 
 - as the current user
 - as the `root` user (if different to the current user)

docs/code-pr-advice.md

@@ -79,7 +79,7 @@ a "`BUG: feature X not implemented see {bug-url}`" type error.
 - Don't use multiple log calls when a single log call could be used.
 
 - Use structured logging where possible to allow
-  [standard tooling](https://github.com/kata-containers/tests/tree/main/cmd/log-parser)
+  [standard tooling](../src/tools/log-parser)
   be able to extract the log fields.
 
 ### Names

docs/design/README.md

@@ -12,7 +12,7 @@ Kata Containers design documents:
 - [Metrics(Kata 2.0)](kata-2-0-metrics.md)
 - [Design for Kata Containers `Lazyload` ability with `nydus`](kata-nydus-design.md)
 - [Design for direct-assigned volume](direct-blk-device-assignment.md)
-
+- [Design for core-scheduling](core-scheduling.md)
 ---
 
 - [Design proposals](proposals)

docs/design/architecture/kubernetes.md

@@ -17,7 +17,7 @@ Kubelet instance is responsible for managing the lifecycle of pods
 within the nodes and eventually relies on a container runtime to
 handle execution. The Kubelet architecture decouples lifecycle
 management from container execution through a dedicated gRPC based
-[Container Runtime Interface (CRI)](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/node/container-runtime-interface-v1.md).
+[Container Runtime Interface (CRI)](https://github.com/kubernetes/design-proposals-archive/blob/main/node/container-runtime-interface-v1.md).
 
 In other words, a Kubelet is a CRI client and expects a CRI
 implementation to handle the server side of the interface.

docs/design/architecture/storage.md

@@ -1,5 +1,17 @@
 # Storage
 
+## Limits
+
+Kata Containers is [compatible](README.md#compatibility) with existing
+standards and runtime. From the perspective of storage, this means no
+limits are placed on the amount of storage a container
+[workload](README.md#workload) may use.
+
+Since cgroups are not able to set limits on storage allocation, if you
+wish to constrain the amount of storage a container uses, consider
+using an existing facility such as `quota(1)` limits or
+[device mapper](#devicemapper) limits.
+
 ## virtio SCSI
 
 If a block-based graph driver is [configured](README.md#configuration),
@@ -20,7 +32,7 @@ For virtio-fs, the [runtime](README.md#runtime) starts one `virtiofsd` daemon
 ## Devicemapper
 
 The
-[devicemapper `snapshotter`](https://github.com/containerd/containerd/tree/master/snapshots/devmapper)
+[devicemapper `snapshotter`](https://github.com/containerd/containerd/tree/main/snapshots/devmapper)
 is a special case. The `snapshotter` uses dedicated block devices
 rather than formatted filesystems, and operates at the block level
 rather than the file level. This knowledge is used to directly use the

docs/design/core-scheduling.md

@@ -0,0 +1,12 @@
+# Core scheduling
+
+Core scheduling is a Linux kernel feature that allows only trusted tasks to run concurrently on
+CPUs sharing compute resources (for example, hyper-threads on a core).
+
+Containerd versions >= 1.6.4 leverage this to treat all of the processes associated with a
+given pod or container to be a single group of trusted tasks. To indicate this should be carried
+out, containerd sets the `SCHED_CORE` environment variable for each shim it spawns. When this is
+set, the Kata Containers shim implementation uses the `prctl` syscall to create a new core scheduling
+domain for the shim process itself as well as future VMM processes it will start.
+
+For more details on the core scheduling feature, see the [Linux documentation](https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/core-scheduling.html).

docs/how-to/containerd-kata.md

@@ -72,7 +72,6 @@ $ command -v containerd
 
 ### Install CNI plugins
 
-> **Note:** You do not need to install CNI plugins if you do not want to use containerd with Kubernetes.
 > If you have installed Kubernetes with `kubeadm`, you might have already installed the CNI plugins.
 
 You can manually install CNI plugins as follows:
@@ -94,8 +93,8 @@ $ popd
 You can install the `cri-tools` from source code:
 
 ```bash
-$ go get github.com/kubernetes-incubator/cri-tools
-$ pushd $GOPATH/src/github.com/kubernetes-incubator/cri-tools
+$ go get github.com/kubernetes-sigs/cri-tools
+$ pushd $GOPATH/src/github.com/kubernetes-sigs/cri-tools
 $ make
 $ sudo -E make install
 $ popd
@@ -131,74 +130,42 @@ For
 
 The `RuntimeClass` is suggested.
 
-The following configuration includes three runtime classes:
+The following configuration includes two runtime classes:
 - `plugins.cri.containerd.runtimes.runc`: the runc, and it is the default runtime.
 - `plugins.cri.containerd.runtimes.kata`: The function in containerd (reference [the document here](https://github.com/containerd/containerd/tree/master/runtime/v2#binary-naming)) 
   where the dot-connected string `io.containerd.kata.v2` is translated to `containerd-shim-kata-v2` (i.e. the 
   binary name of the Kata implementation of [Containerd Runtime V2 (Shim API)](https://github.com/containerd/containerd/tree/master/runtime/v2)).
-- `plugins.cri.containerd.runtimes.katacli`: the `containerd-shim-runc-v1` calls `kata-runtime`, which is the legacy process.
 
 ```toml
     [plugins.cri.containerd]
       no_pivot = false
     [plugins.cri.containerd.runtimes]
-      [plugins.cri.containerd.runtimes.runc]
-         runtime_type = "io.containerd.runc.v1"
-         [plugins.cri.containerd.runtimes.runc.options]
-           NoPivotRoot = false
-           NoNewKeyring = false
-           ShimCgroup = ""
-           IoUid = 0
-           IoGid = 0
-           BinaryName = "runc"
-           Root = ""
-           CriuPath = ""
-           SystemdCgroup = false
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+         privileged_without_host_devices = false
+         runtime_type = "io.containerd.runc.v2"
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+            BinaryName = ""
+            CriuImagePath = ""
+            CriuPath = ""
+            CriuWorkPath = ""
+            IoGid = 0
       [plugins.cri.containerd.runtimes.kata]
          runtime_type = "io.containerd.kata.v2"
-      [plugins.cri.containerd.runtimes.katacli]
-         runtime_type = "io.containerd.runc.v1"
-         [plugins.cri.containerd.runtimes.katacli.options]
-           NoPivotRoot = false
-           NoNewKeyring = false
-           ShimCgroup = ""
-           IoUid = 0
-           IoGid = 0
-           BinaryName = "/usr/bin/kata-runtime"
-           Root = ""
-           CriuPath = ""
-           SystemdCgroup = false
-```
-
-From Containerd v1.2.4 and Kata v1.6.0, there is a new runtime option supported, which allows you to specify a specific Kata configuration file as follows:
-
-```toml
-      [plugins.cri.containerd.runtimes.kata]
-         runtime_type = "io.containerd.kata.v2"
-	 privileged_without_host_devices = true
-	 [plugins.cri.containerd.runtimes.kata.options]
-	   ConfigPath = "/etc/kata-containers/config.toml"
+         privileged_without_host_devices = true
+         pod_annotations = ["io.katacontainers.*"]
+         container_annotations = ["io.katacontainers.*"]
+         [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options]
+            ConfigPath = "/opt/kata/share/defaults/kata-containers/configuration.toml"
 ```
 
 `privileged_without_host_devices` tells containerd that a privileged Kata container should not have direct access to all host devices. If unset, containerd will pass all host devices to Kata container, which may cause security issues.
 
+`pod_annotations` is the list of pod annotations passed to both the pod sandbox as well as container through the OCI config.
+
+`container_annotations` is the list of container annotations passed through to the OCI config of the containers.
+
 This `ConfigPath` option is optional. If you do not specify it, shimv2 first tries to get the configuration file from the environment variable `KATA_CONF_FILE`. If neither are set, shimv2 will use the default Kata configuration file paths (`/etc/kata-containers/configuration.toml` and `/usr/share/defaults/kata-containers/configuration.toml`).
 
-If you use Containerd older than v1.2.4 or a version of Kata older than v1.6.0  and also want to specify a configuration file, you can use the following workaround, since the shimv2 accepts an environment variable, `KATA_CONF_FILE` for the configuration file path. Then, you can create a
-shell script with the following:
-
-```bash
-#!/usr/bin/env bash
-KATA_CONF_FILE=/etc/kata-containers/firecracker.toml containerd-shim-kata-v2 $@
-```
-
-Name it as `/usr/local/bin/containerd-shim-katafc-v2` and reference it in the configuration of containerd:
-
-```toml
-      [plugins.cri.containerd.runtimes.kata-firecracker]
-         runtime_type = "io.containerd.katafc.v2"
-```
-
 #### Kata Containers as the runtime for untrusted workload
 
 For cases without `RuntimeClass` support, we can use the legacy annotation method to support using Kata Containers 
@@ -218,28 +185,8 @@ and then, run an untrusted workload with Kata Containers:
       runtime_type = "io.containerd.kata.v2"
 ```
 
-For the earlier versions of Kata Containers and containerd that do not support Runtime V2 (Shim API), you can use the following alternative configuration:
-
-```toml
-    [plugins.cri.containerd]
-  
-    # "plugins.cri.containerd.default_runtime" is the runtime to use in containerd.
-    [plugins.cri.containerd.default_runtime]
-      # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
-      runtime_type = "io.containerd.runtime.v1.linux"
-
-    # "plugins.cri.containerd.untrusted_workload_runtime" is a runtime to run untrusted workloads on it.
-    [plugins.cri.containerd.untrusted_workload_runtime]
-      # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
-      runtime_type = "io.containerd.runtime.v1.linux"
-
-      # runtime_engine is the name of the runtime engine used by containerd.
-      runtime_engine = "/usr/bin/kata-runtime"
-```
-
 You can find more information on the [Containerd config documentation](https://github.com/containerd/cri/blob/master/docs/config.md)
 
-
 #### Kata Containers as the default runtime
 
 If you want to set Kata Containers as the only runtime in the deployment, you can simply configure as follows:
@@ -250,15 +197,6 @@ If you want to set Kata Containers as the only runtime in the deployment, you ca
       runtime_type = "io.containerd.kata.v2"
 ```
 
-Alternatively, for the earlier versions of Kata Containers and containerd that do not support Runtime V2 (Shim API), you can use the following alternative configuration:
-
-```toml
-    [plugins.cri.containerd]
-    [plugins.cri.containerd.default_runtime]
-      runtime_type = "io.containerd.runtime.v1.linux"
-      runtime_engine = "/usr/bin/kata-runtime"
-```
-
 ### Configuration for `cri-tools`
 
 > **Note:** If you skipped the [Install `cri-tools`](#install-cri-tools) section, you can skip this section too.
@@ -312,10 +250,12 @@ To run a container with Kata Containers through the containerd command line, you
 
 ```bash
 $ sudo ctr image pull docker.io/library/busybox:latest
-$ sudo ctr run --runtime io.containerd.run.kata.v2 -t --rm docker.io/library/busybox:latest hello sh
+$ sudo ctr run --cni --runtime io.containerd.run.kata.v2 -t --rm docker.io/library/busybox:latest hello sh
 ```
 
 This launches a BusyBox container named `hello`, and it will be removed by `--rm` after it quits.
+The `--cni` flag enables CNI networking for the container. Without this flag, a container with just a
+loopback interface is created.
 
 ### Launch Pods with `crictl` command line
 

docs/how-to/how-to-import-kata-logs-with-fluentd.md

@@ -68,7 +68,7 @@ the Kata logs import to the EFK stack.
 > stack they are able to utilise in order to modify and test as necessary.
 
 Minikube by default
-[configures](https://github.com/kubernetes/minikube/blob/master/deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/etc/systemd/journald.conf)
+[configures](https://github.com/kubernetes/minikube/blob/master/deploy/iso/minikube-iso/board/minikube/x86_64/rootfs-overlay/etc/systemd/journald.conf)
  the `systemd-journald` with the
 [`Storage=volatile`](https://www.freedesktop.org/software/systemd/man/journald.conf.html) option,
 which results in the journal being stored in `/run/log/journal`. Unfortunately, the Minikube EFK
@@ -163,7 +163,7 @@ sub-filter on, for instance, the `SYSLOG_IDENTIFIER` to differentiate the Kata c
 on the `PRIORITY` to filter out critical issues etc.
 
 Kata generates a significant amount of Kata specific information, which can be seen as
-[`logfmt`](https://github.com/kata-containers/tests/tree/main/cmd/log-parser#logfile-requirements).
+[`logfmt`](../../src/tools/log-parser/README.md#logfile-requirements).
 data contained in the `MESSAGE` field. Imported as-is, there is no easy way to filter on that data
 in Kibana:
 

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -91,6 +91,7 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.virtio_fs_daemon` | string | virtio-fs `vhost-user` daemon path |
 | `io.katacontainers.config.hypervisor.virtio_fs_extra_args` | string | extra options passed to `virtiofs` daemon |
 | `io.katacontainers.config.hypervisor.enable_guest_swap` | `boolean` | enable swap in the guest |
+| `io.katacontainers.config.hypervisor.use_legacy_serial` | `boolean` | uses legacy serial device for guest's console (QEMU) |
 
 ## Container Options
 | Key | Value Type | Comments |
@@ -172,7 +173,7 @@ kind: Pod
 metadata:
   name: pod2
   annotations:
-    io.katacontainers.config.runtime.disable_guest_seccomp: false
+    io.katacontainers.config.runtime.disable_guest_seccomp: "false"
 spec:
   runtimeClassName: kata
   containers:

docs/how-to/privileged.md

@@ -31,7 +31,7 @@ See below example config:
   [plugins.cri]
     [plugins.cri.containerd]
        [plugins.cri.containerd.runtimes.runc]
-         runtime_type = "io.containerd.runc.v1"
+         runtime_type = "io.containerd.runc.v2"
          privileged_without_host_devices = false
        [plugins.cri.containerd.runtimes.kata]
          runtime_type = "io.containerd.kata.v2"

docs/how-to/run-kata-with-k8s.md

@@ -99,7 +99,18 @@ $ sudo systemctl restart kubelet
 $ sudo kubeadm init --ignore-preflight-errors=all --cri-socket /var/run/crio/crio.sock --pod-network-cidr=10.244.0.0/16
 
 # If using containerd
-$ sudo kubeadm init --ignore-preflight-errors=all --cri-socket /run/containerd/containerd.sock --pod-network-cidr=10.244.0.0/16
+$ cat <<EOF | tee kubeadm-config.yaml
+apiVersion: kubeadm.k8s.io/v1beta3
+kind: InitConfiguration
+nodeRegistration:
+  criSocket: "/run/containerd/containerd.sock"
+---
+kind: KubeletConfiguration
+apiVersion: kubelet.config.k8s.io/v1beta1
+cgroupDriver: cgroupfs
+podCIDR: "10.244.0.0/16"
+EOF
+$ sudo kubeadm init --ignore-preflight-errors=all --config kubeadm-config.yaml
 
 $ export KUBECONFIG=/etc/kubernetes/admin.conf
 ```

docs/use-cases/NVIDIA-GPU-passthrough-and-Kata.md

@@ -2,20 +2,20 @@
 
 An NVIDIA GPU device can be passed to a Kata Containers container using GPU
 passthrough (NVIDIA GPU pass-through mode) as well as GPU mediated passthrough
-(NVIDIA vGPU mode).
+(NVIDIA `vGPU` mode).
 
 NVIDIA GPU pass-through mode, an entire physical GPU is directly assigned to one
 VM, bypassing the NVIDIA Virtual GPU Manager. In this mode of operation, the GPU
 is accessed exclusively by the NVIDIA driver running in the VM to which it is
 assigned. The GPU is not shared among VMs.
 
-NVIDIA Virtual GPU (vGPU) enables multiple virtual machines (VMs) to have
+NVIDIA Virtual GPU (`vGPU`) enables multiple virtual machines (VMs) to have
 simultaneous, direct access to a single physical GPU, using the same NVIDIA
 graphics drivers that are deployed on non-virtualized operating systems. By
-doing this, NVIDIA vGPU provides VMs with unparalleled graphics performance,
+doing this, NVIDIA `vGPU` provides VMs with unparalleled graphics performance,
 compute performance, and application compatibility, together with the
 cost-effectiveness and scalability brought about by sharing a GPU among multiple
-workloads. A vGPU can be either time-sliced or Multi-Instance GPU (MIG)-backed
+workloads. A `vGPU` can be either time-sliced or Multi-Instance GPU (MIG)-backed
 with [MIG-slices](https://docs.nvidia.com/datacenter/tesla/mig-user-guide/).
 
 | Technology | Description | Behavior | Detail |
@@ -46,14 +46,14 @@ $ lspci -s d0:00.0 -vv | grep Region
 For large BARs devices, MMIO mapping above 4G address space should be `enabled`
 in the PCI configuration of the BIOS.
 
-Some hardware vendors use different name in BIOS, such as:
+Some hardware vendors use a different name in BIOS, such as:
 
 - Above 4G Decoding
 - Memory Hole for PCI MMIO
 - Memory Mapped I/O above 4GB
 
 If one is using a GPU based on the Ampere architecture and later additionally
-SR-IOV needs to be enabled for the vGPU use-case.
+SR-IOV needs to be enabled for the `vGPU` use-case.
 
 The following steps outline the workflow for using an NVIDIA GPU with Kata.
 
@@ -154,7 +154,7 @@ $ ./build-kernel.sh -v 5.15.23 -g nvidia build
 $ sudo -E ./build-kernel.sh -v 5.15.23 -g nvidia install
 ```
 
-To build NVIDIA Driver in Kata container, `linux-headers` is required.
+To build NVIDIA Driver in Kata container, `linux-headers` are required.
 This is a way to generate deb packages for `linux-headers`:
 
 > **Note**:
@@ -177,7 +177,7 @@ kernel = "/usr/share/kata-containers/vmlinuz-nvidia-gpu.container"
 
 Use the following steps to pass an NVIDIA GPU device in pass-through mode with Kata:
 
-1. Find the Bus-Device-Function (BDF) for GPU device on host:
+1. Find the Bus-Device-Function (BDF) for the GPU device on the host:
 
    ```sh
    $ sudo lspci -nn -D | grep -i nvidia
@@ -219,7 +219,7 @@ Use the following steps to pass an NVIDIA GPU device in pass-through mode with K
    crw-rw-rw- 1 root    root     10, 196 Mar 18 02:27 vfio
    ```
 
-4. Start a Kata container with GPU device:
+4. Start a Kata container with the GPU device:
 
    ```sh
    # You may need to `modprobe vhost-vsock` if you get
@@ -246,9 +246,228 @@ Use the following steps to pass an NVIDIA GPU device in pass-through mode with K
 ## NVIDIA vGPU mode with Kata Containers
 
 NVIDIA vGPU is a licensed product on all supported GPU boards. A software license
-is required to enable all vGPU features within the guest VM.
+is required to enable all vGPU features within the guest VM. NVIDIA vGPU manager
+needs to be installed on the host to configure GPUs in vGPU mode. See [NVIDIA Virtual GPU Software Documentation v14.0 through 14.1](https://docs.nvidia.com/grid/14.0/) for more details.
 
-> **TODO**: Will follow up with instructions
+### NVIDIA vGPU time-sliced
+
+In the time-sliced mode, the GPU is not partitioned and the workload uses the
+whole GPU and shares access to the GPU engines. Processes are scheduled in
+series. The best effort scheduler is the default one and can be exchanged by
+other scheduling policies see the documentation above how to do that.
+
+Beware if you had `MIG` enabled before to disable `MIG` on the GPU if you want
+to use `time-sliced` `vGPU`.
+
+```sh
+$ sudo nvidia-smi -mig 0
+```
+
+Enable the virtual functions for the physical GPU in the `sysfs` file system.
+
+```sh
+$ sudo /usr/lib/nvidia/sriov-manage -e 0000:41:00.0
+```
+
+Get the `BDF` of the available virtual function on the GPU, and choose one for the
+following steps.
+
+```sh
+$ cd /sys/bus/pci/devices/0000:41:00.0/
+$ ls -l |  grep virtfn
+```
+
+#### List all available vGPU instances
+
+The following shell snippet will walk the `sysfs` and only print instances
+that are available, that can be created.
+
+```sh
+# The 00.0 is often the PF of the device the VFs will have the funciont in the
+# BDF incremented by some values so e.g. the very first VF is 0000:41:00.4
+
+cd /sys/bus/pci/devices/0000:41:00.0/
+
+for vf in $(ls -d virtfn*)
+do
+        BDF=$(basename $(readlink -f $vf))
+        for md in $(ls -d $vf/mdev_supported_types/*)
+        do
+                AVAIL=$(cat $md/available_instances)
+                NAME=$(cat $md/name)
+                DIR=$(basename $md)
+
+                if [ $AVAIL -gt 0 ]; then
+                        echo "| BDF          | INSTANCES | NAME           | DIR        |"
+                        echo "+--------------+-----------+----------------+------------+"
+                        printf "| %12s |%10d |%15s | %10s |\n\n" "$BDF" "$AVAIL" "$NAME" "$DIR"
+                fi
+
+        done
+done
+```
+
+If there are available instances you get something like this (for the first VF),
+beware that the output is highly dependent on the GPU you have, if there is no
+output check again if `MIG` is really disabled.
+
+```sh
+| BDF          | INSTANCES | NAME           | DIR        |
++--------------+-----------+----------------+------------+
+| 0000:41:00.4 |         1 |  GRID A100D-4C | nvidia-692 |
+
+| BDF          | INSTANCES | NAME           | DIR        |
++--------------+-----------+----------------+------------+
+| 0000:41:00.4 |         1 |  GRID A100D-8C | nvidia-693 |
+
+| BDF          | INSTANCES | NAME           | DIR        |
++--------------+-----------+----------------+------------+
+| 0000:41:00.4 |         1 | GRID A100D-10C | nvidia-694 |
+
+| BDF          | INSTANCES | NAME           | DIR        |
++--------------+-----------+----------------+------------+
+| 0000:41:00.4 |         1 | GRID A100D-16C | nvidia-695 |
+
+| BDF          | INSTANCES | NAME           | DIR        |
++--------------+-----------+----------------+------------+
+| 0000:41:00.4 |         1 | GRID A100D-20C | nvidia-696 |
+
+| BDF          | INSTANCES | NAME           | DIR        |
++--------------+-----------+----------------+------------+
+| 0000:41:00.4 |         1 | GRID A100D-40C | nvidia-697 |
+
+| BDF          | INSTANCES | NAME           | DIR        |
++--------------+-----------+----------------+------------+
+| 0000:41:00.4 |         1 | GRID A100D-80C | nvidia-698 |
+
+```
+
+Change to the `mdev_supported_types` directory for the virtual function on which
+you want to create the `vGPU`. Taking the first output as an example:
+
+```sh
+$ cd virtfn0/mdev_supported_types/nvidia-692
+$ UUIDGEN=$(uuidgen)
+$ sudo bash -c "echo $UUIDGEN > create"
+```
+
+Confirm that the `vGPU` was created. You should see the `UUID` pointing to a
+subdirectory of the `sysfs` space.
+
+```sh
+$ ls -l /sys/bus/mdev/devices/
+```
+
+Get the `IOMMU` group number and verify there is a `VFIO` device created to use
+with Kata.
+
+```sh
+$ ls -l /sys/bus/mdev/devices/*/
+$ ls -l /dev/vfio
+```
+
+Use the `VFIO` device created in the same way as in the pass-through use-case.
+Beware that the guest needs the NVIDIA guest drivers, so one would need to build
+a new guest `OS` image.
+
+### NVIDIA vGPU MIG-backed
+
+We're not going into detail what `MIG` is but briefly it is a technology to
+partition the hardware into independent instances with guaranteed quality of
+service. For more details see [NVIDIA Multi-Instance GPU User Guide](https://docs.nvidia.com/datacenter/tesla/mig-user-guide/).
+
+First enable `MIG` mode for a GPU, depending on the platform you're running
+a reboot would be necessary. Some platforms support GPU reset.
+
+```sh
+$ sudo nvidia-smi -mig 1
+```
+
+If the platform supports a GPU reset one can run, otherwise you will get a
+warning to reboot the server.
+
+```sh
+$ sudo nvidia-smi --gpu-reset
+```
+
+The driver per default provides a number of profiles that users can opt-in when
+configuring the MIG feature.
+
+```sh
+$ sudo nvidia-smi mig -lgip
++-----------------------------------------------------------------------------+
+| GPU instance profiles:                                                      |
+| GPU   Name             ID    Instances   Memory     P2P    SM    DEC   ENC  |
+|                              Free/Total   GiB              CE    JPEG  OFA  |
+|=============================================================================|
+|   0  MIG 1g.10gb       19     7/7        9.50       No     14     0     0   |
+|                                                             1     0     0   |
++-----------------------------------------------------------------------------+
+|   0  MIG 1g.10gb+me    20     1/1        9.50       No     14     1     0   |
+|                                                             1     1     1   |
++-----------------------------------------------------------------------------+
+|   0  MIG 2g.20gb       14     3/3        19.50      No     28     1     0   |
+|                                                             2     0     0   |
++-----------------------------------------------------------------------------+
+                              ...
+```
+
+Create the GPU instances that correspond to the `vGPU` types of the `MIG-backed`
+`vGPUs` that you will create [NVIDIA A100 PCIe 80GB Virtual GPU Types](https://docs.nvidia.com/grid/13.0/grid-vgpu-user-guide/index.html#vgpu-types-nvidia-a100-pcie-80gb).
+
+```sh
+# MIG 1g.10gb --> vGPU A100D-1-10C
+$ sudo nvidia-smi mig -cgi 19
+```
+
+List the GPU instances and get the GPU instance id to create the compute
+instance.
+
+```sh
+$ sudo nvidia-smi mig -lgi # list the created GPU instances
+$ sudo nvidia-smi mig -cci -gi 9 # each GPU instance can have several compute
+                                 # instances. Instance -> Workload
+```
+
+Verify that the compute instances were created within the GPU instance
+
+```sh
+$ nvidia-smi
+                              ... snip ...
++-----------------------------------------------------------------------------+
+| MIG devices:                                                                |
++------------------+----------------------+-----------+-----------------------+
+| GPU  GI  CI  MIG |         Memory-Usage |        Vol|         Shared        |
+|      ID  ID  Dev |           BAR1-Usage | SM     Unc| CE  ENC  DEC  OFA  JPG|
+|                  |                      |        ECC|                       |
+|==================+======================+===========+=======================|
+|  0    9   0   0  |      0MiB /  9728MiB | 14      0 |  1   0    0    0    0 |
+|                  |      0MiB /  4095MiB |           |                       |
++------------------+----------------------+-----------+-----------------------+
+                              ... snip ...
+```
+
+We can use the [snippet](#list-all-available-vgpu-instances) from before to list
+the available `vGPU` instances, this time `MIG-backed`.
+
+```sh
+| BDF          | INSTANCES | NAME           | DIR        |
++--------------+-----------+----------------+------------+
+| 0000:41:00.4 |         1 |GRID A100D-1-10C | nvidia-699 |
+
+| BDF          | INSTANCES | NAME           | DIR        |
++--------------+-----------+----------------+------------+
+| 0000:41:00.5 |         1 |GRID A100D-1-10C | nvidia-699 |
+
+| BDF          | INSTANCES | NAME           | DIR        |
++--------------+-----------+----------------+------------+
+| 0000:41:01.6 |         1 |GRID A100D-1-10C | nvidia-699 |
+                       ... snip ...
+```
+
+Repeat the steps after the [snippet](#list-all-available-vgpu-instances) listing
+to create the corresponding `mdev` device and use the guest `OS` created in the
+previous section with `time-sliced` `vGPUs`.
 
 ## Install NVIDIA Driver + Toolkit in Kata Containers Guest OS
 
@@ -263,7 +482,7 @@ export EXTRA_PKGS="gcc make curl gnupg"
 ```
 
 Having the `$ROOTFS_DIR` exported in the previous step we can now install all the
-need parts in the guest OS. In this case we have an Ubuntu based rootfs.
+needed parts in the guest OS. In this case, we have an Ubuntu based rootfs.
 
 First off all mount the special filesystems into the rootfs
 
@@ -281,9 +500,9 @@ Now we can enter `chroot`
 $ sudo chroot ${ROOTFS_DIR}
 ```
 
-Inside the rootfs one is going to install the drivers and toolkit to enable easy
-creation of GPU containers with Kata. We can also use this rootfs for any other
-container not specifically only for GPUs.
+Inside the rootfs one is going to install the drivers and toolkit to enable the
+easy creation of GPU containers with Kata. We can also use this rootfs for any
+other container not specifically only for GPUs.
 
 As a prerequisite install the copied kernel development packages
 
@@ -304,6 +523,7 @@ $ ./NVIDIA-Linux-x86_64-510.54.run -x
 $ cd NVIDIA-Linux-x86_64-510.54
 $ ./nvidia-installer -k 5.15.23-nvidia-gpu
 ```
+
 Having the drivers installed we need to install the toolkit which will take care
 of providing the right bits into the container.
 
@@ -325,7 +545,7 @@ Create the hook execution file for Kata:
 /usr/bin/nvidia-container-toolkit -debug $@
 ```
 
-As a last step one can do some cleanup of files or package caches. Build the
+As the last step one can do some cleanup of files or package caches. Build the
 rootfs and configure it for use with Kata according to the development guide.
 
 Enable the `guest_hook_path` in Kata's `configuration.toml`
@@ -334,7 +554,7 @@ Enable the `guest_hook_path` in Kata's `configuration.toml`
 guest_hook_path = "/usr/share/oci/hooks"
 ```
 
-One has build a NVIDIA rootfs, kernel and now we can run any GPU container
+One has built a NVIDIA rootfs, kernel and now we can run any GPU container
 without installing the drivers into the container. Check NVIDIA device status
 with `nvidia-smi`
 
@@ -362,7 +582,7 @@ Fri Mar 18 10:36:59 2022
 +-----------------------------------------------------------------------------+
 ```
 
-As a last step one can remove the additional packages and files that were added
+As the last step one can remove the additional packages and files that were added
 to the `$ROOTFS_DIR` to keep it as small as possible.
 
 ## References

docs/use-cases/using-Intel-QAT-and-kata.md

@@ -312,7 +312,7 @@ working properly with the Kata Containers VM.
 
 ### Build OpenSSL Intel® QAT engine container
 
-Use the OpenSSL Intel® QAT [Dockerfile](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/master/demo/openssl-qat-engine) 
+Use the OpenSSL Intel® QAT [Dockerfile](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/main/demo/openssl-qat-engine) 
 to build a container image with an optimized OpenSSL engine for 
 Intel® QAT. Using `docker build` with the Kata Containers runtime can sometimes
 have issues. Therefore, make sure that `runc` is the default Docker container 
@@ -444,7 +444,7 @@ $ sudo docker save -o openssl-qat-engine.tar openssl-qat-engine:latest
 $ sudo ctr -n=k8s.io images import openssl-qat-engine.tar
 ```
 
-The [Intel® QAT Plugin](https://github.com/intel/intel-device-plugins-for-kubernetes/blob/master/cmd/qat_plugin/README.md)
+The [Intel® QAT Plugin](https://github.com/intel/intel-device-plugins-for-kubernetes/blob/main/cmd/qat_plugin/README.md)
 needs to be started so that the virtual functions can be discovered and
 used by Kubernetes. 
 

snap/local/README.md

@@ -22,21 +22,35 @@ $ sudo snap install kata-containers --classic
 
 ## Build and install snap image
 
-Run next command at the root directory of the packaging repository.
+Run the command below which will use the packaging Makefile to build the snap image:
 
 ```sh
-$ make snap
+$ make -C tools/packaging snap
 ```
 
+> **Warning:**
+>
+> By default, `snapcraft` will create a clean virtual machine
+> environment to build the snap in using the `multipass` tool.
+>
+> However, `multipass` is silently disabled when `--destructive-mode` is
+> used.
+>
+> Since building the Kata Containers package currently requires
+> `--destructive-mode`, the snap will be built using the host
+> environment. To avoid parts of the build auto-detecting additional
+> features to enable (for example for QEMU), we recommend that you
+> only run the snap build in a minimal host environment.
+
 To install the resulting snap image, snap must be put in [classic mode][3] and the
-security confinement must be disabled (*--classic*). Also since the resulting snap
-has not been signed the verification of signature must be omitted (*--dangerous*).
+security confinement must be disabled (`--classic`). Also since the resulting snap
+has not been signed the verification of signature must be omitted (`--dangerous`).
 
 ```sh
-$ sudo snap install --classic --dangerous kata-containers_[VERSION]_[ARCH].snap
+$ sudo snap install --classic --dangerous "kata-containers_${version}_${arch}.snap"
 ```
 
-Replace `VERSION` with the current version of Kata Containers and `ARCH` with
+Replace `${version}` with the current version of Kata Containers and `${arch}` with
 the system architecture.
 
 ## Configure Kata Containers
@@ -76,12 +90,12 @@ then a new configuration file can be [created](#configure-kata-containers)
 and [configured][7].
 
 [1]: https://docs.snapcraft.io/snaps/intro
-[2]: ../docs/design/architecture/README.md#root-filesystem-image
+[2]: ../../docs/design/architecture/README.md#root-filesystem-image
 [3]: https://docs.snapcraft.io/reference/confinement#classic
-[4]: https://github.com/kata-containers/runtime#configuration
+[4]: https://github.com/kata-containers/kata-containers/tree/main/src/runtime#configuration
 [5]: https://docs.docker.com/engine/reference/commandline/dockerd
-[6]: ../docs/install/docker/ubuntu-docker-install.md
-[7]: ../docs/Developer-Guide.md#configure-to-use-initrd-or-rootfs-image
+[6]: ../../docs/install/docker/ubuntu-docker-install.md
+[7]: ../../docs/Developer-Guide.md#configure-to-use-initrd-or-rootfs-image
 [8]: https://snapcraft.io/kata-containers
-[9]: ../docs/Developer-Guide.md#run-kata-containers-with-docker
-[10]: ../docs/Developer-Guide.md#run-kata-containers-with-kubernetes
+[9]: ../../docs/Developer-Guide.md#run-kata-containers-with-docker
+[10]: ../../docs/Developer-Guide.md#run-kata-containers-with-kubernetes

snap/local/snap-common.sh

@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2022 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Description: Idempotent script to be sourced by all parts in a
+#   snapcraft config file.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# XXX: Bash-specific code. zsh doesn't support this option and that *does*
+# matter if this script is run sourced... since it'll be using zsh! ;)
+[ -n "$BASH_VERSION" ] && set -o errtrace
+
+[ -n "${DEBUG:-}" ] && set -o xtrace
+
+die()
+{
+	echo >&2 "ERROR: $0: $*"
+}
+
+[ -n "${SNAPCRAFT_STAGE:-}" ] ||\
+	die "must be sourced from a snapcraft config file"
+
+snap_yq_version=3.4.1
+
+snap_common_install_yq()
+{
+      export yq="${SNAPCRAFT_STAGE}/bin/yq"
+
+      local yq_pkg
+      yq_pkg="github.com/mikefarah/yq"
+
+      local yq_url
+      yq_url="https://${yq_pkg}/releases/download/${snap_yq_version}/yq_${goos}_${goarch}"
+      curl -o "${yq}" -L "${yq_url}"
+      chmod +x "${yq}"
+}
+
+# Function that should be called for each snap "part" in
+# snapcraft.yaml.
+snap_common_main()
+{
+	# Architecture
+	arch="$(uname -m)"
+
+	case "${arch}" in
+		aarch64)
+			goarch="arm64"
+			qemu_arch="${arch}"
+			;;
+
+		ppc64le)
+			goarch="ppc64le"
+			qemu_arch="ppc64"
+			;;
+
+		s390x)
+			goarch="${arch}"
+			qemu_arch="${arch}"
+			;;
+
+		x86_64)
+			goarch="amd64"
+			qemu_arch="${arch}"
+			;;
+
+		*) die "unsupported architecture: ${arch}" ;;
+	esac
+
+	dpkg_arch=$(dpkg --print-architecture)
+
+	# golang
+	#
+	# We need the O/S name in golang format, but since we don't
+	# know if the godeps part has run, we don't know if golang is
+	# available yet, hence fall back to a standard system command.
+	goos="$(go env GOOS &>/dev/null || true)"
+	[ -z "$goos" ] && goos=$(uname -s|tr '[A-Z]' '[a-z]')
+
+	export GOROOT="${SNAPCRAFT_STAGE}"
+	export GOPATH="${GOROOT}/gopath"
+	export GO111MODULE="auto"
+
+	mkdir -p "${GOPATH}/bin"
+	export PATH="${GOPATH}/bin:${PATH}"
+
+	# Proxy
+	export http_proxy="${http_proxy:-}"
+	export https_proxy="${https_proxy:-}"
+
+	# Binaries
+	mkdir -p "${SNAPCRAFT_STAGE}/bin"
+
+	export PATH="$PATH:${SNAPCRAFT_STAGE}/bin"
+
+	# YAML query tool
+	export yq="${SNAPCRAFT_STAGE}/bin/yq"
+
+	# Kata paths
+	export kata_dir=$(printf "%s/src/github.com/%s/%s" \
+		"${GOPATH}" \
+		"${SNAPCRAFT_PROJECT_NAME}" \
+		"${SNAPCRAFT_PROJECT_NAME}")
+
+	export versions_file="${kata_dir}/versions.yaml"
+
+	[ -n "${yq:-}" ] && [ -x "${yq:-}" ] || snap_common_install_yq
+}
+
+snap_common_main

snap/snapcraft.yaml

@@ -1,4 +1,5 @@
 name: kata-containers
+website: https://github.com/kata-containers/kata-containers
 summary: Build lightweight VMs that seamlessly plug into the containers ecosystem
 description: |
   Kata Containers is an open source project and community working to build a
@@ -18,20 +19,18 @@ parts:
       - git
       - git-extras
     override-pull: |
-      version="9999"
-      kata_url="https://github.com/kata-containers/kata-containers"
+      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
 
-      if echo "${GITHUB_REF}" | grep -q -E "^refs/tags"; then
-        version=$(echo ${GITHUB_REF} | cut -d/ -f3)
+      version="9999"
+
+      if echo "${GITHUB_REF:-}" | grep -q -E "^refs/tags"; then
+        version=$(echo ${GITHUB_REF:-} | cut -d/ -f3)
         git checkout ${version}
       fi
 
       snapcraftctl set-grade "stable"
       snapcraftctl set-version "${version}"
 
-      # setup GOPATH - this repo dir should be there
-      export GOPATH=${SNAPCRAFT_STAGE}/gopath
-      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
       mkdir -p $(dirname ${kata_dir})
       ln -sf $(realpath "${SNAPCRAFT_STAGE}/..") ${kata_dir}
 
@@ -43,31 +42,46 @@ parts:
     build-packages:
       - curl
     override-build: |
+      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
+
       # put everything in stage
-      cd ${SNAPCRAFT_STAGE}
+      cd "${SNAPCRAFT_STAGE}"
 
-      yq_path="./yq"
-      yq_pkg="github.com/mikefarah/yq"
-      goos="linux"
-      case "$(uname -m)" in
-        aarch64) goarch="arm64";;
-        ppc64le) goarch="ppc64le";;
-        x86_64) goarch="amd64";;
-        s390x) goarch="s390x";;
-        *) echo "unsupported architecture: $(uname -m)"; exit 1;;
-      esac
-
-      yq_version=3.4.1
-      yq_url="https://${yq_pkg}/releases/download/${yq_version}/yq_${goos}_${goarch}"
-      curl -o "${yq_path}" -L "${yq_url}"
-      chmod +x "${yq_path}"
-
-      kata_dir=gopath/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
-      version="$(${yq_path} r ${kata_dir}/versions.yaml languages.golang.meta.newest-version)"
+      version="$(${yq} r ${kata_dir}/versions.yaml languages.golang.meta.newest-version)"
       tarfile="go${version}.${goos}-${goarch}.tar.gz"
       curl -LO https://golang.org/dl/${tarfile}
       tar -xf ${tarfile} --strip-components=1
 
+  rustdeps:
+    after: [metadata]
+    plugin: nil
+    prime:
+      - -*
+    build-packages:
+      - curl
+    override-build: |
+      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
+
+      # put everything in stage
+      cd "${SNAPCRAFT_STAGE}"
+
+      version="$(${yq} r ${kata_dir}/versions.yaml languages.rust.meta.newest-version)"
+      if ! command -v rustup > /dev/null; then
+        curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain ${version}
+      fi
+
+      export PATH=${PATH}:${HOME}/.cargo/bin
+      rustup toolchain install ${version}
+      rustup default ${version}
+      if [ "${arch}" == "ppc64le" ] || [ "${arch}" == "s390x" ] ; then
+        [ "${arch}" == "ppc64le" ] && arch="powerpc64le"
+        rustup target add ${arch}-unknown-linux-gnu
+      else
+        rustup target add ${arch}-unknown-linux-musl
+        $([ "$(whoami)" != "root" ] && echo sudo) ln -sf /usr/bin/g++ /bin/musl-g++
+      fi
+      rustup component add rustfmt
+
   image:
     after: [godeps, qemu, kernel]
     plugin: nil
@@ -80,28 +94,17 @@ parts:
       - uidmap
       - gnupg2
     override-build: |
-      [ "$(uname -m)" = "ppc64le" ] || [ "$(uname -m)" = "s390x" ] && sudo apt-get --no-install-recommends install -y protobuf-compiler
+      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
 
-      yq=${SNAPCRAFT_STAGE}/yq
+      [ "${arch}" = "ppc64le" ] || [ "${arch}" = "s390x" ] && sudo apt-get --no-install-recommends install -y protobuf-compiler
 
-      # set GOPATH
-      export GOPATH=${SNAPCRAFT_STAGE}/gopath
-      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
-
-      export GOROOT=${SNAPCRAFT_STAGE}
-      export PATH="${GOROOT}/bin:${PATH}"
-      export GO111MODULE="auto"
-
-      http_proxy=${http_proxy:-""}
-      https_proxy=${https_proxy:-""}
       if [ -n "$http_proxy" ]; then
         echo "Setting proxy $http_proxy"
-        sudo -E systemctl set-environment http_proxy=$http_proxy || true
-        sudo -E systemctl set-environment https_proxy=$https_proxy || true
+        sudo -E systemctl set-environment http_proxy="$http_proxy" || true
+        sudo -E systemctl set-environment https_proxy="$https_proxy" || true
       fi
 
       # Copy yq binary. It's used in the container
-      mkdir -p "${GOPATH}/bin/"
       cp -a "${yq}" "${GOPATH}/bin/"
 
       echo "Unmasking docker service"
@@ -112,63 +115,54 @@ parts:
       echo "Starting docker"
       sudo -E systemctl start docker || true
 
-      cd ${kata_dir}/tools/osbuilder
+      cd "${kata_dir}/tools/osbuilder"
 
       # build image
       export AGENT_INIT=yes
       export USE_DOCKER=1
       export DEBUG=1
-      arch="$(uname -m)"
       initrd_distro=$(${yq} r -X ${kata_dir}/versions.yaml assets.initrd.architecture.${arch}.name)
       image_distro=$(${yq} r -X ${kata_dir}/versions.yaml assets.image.architecture.${arch}.name)
       case "$arch" in
         x86_64)
           # In some build systems it's impossible to build a rootfs image, try with the initrd image
-          sudo -E PATH=$PATH make image DISTRO=${image_distro} || sudo -E PATH=$PATH make initrd DISTRO=${initrd_distro}
+          sudo -E PATH=$PATH make image DISTRO="${image_distro}" || sudo -E PATH="$PATH" make initrd DISTRO="${initrd_distro}"
         ;;
 
         aarch64|ppc64le|s390x)
-          sudo -E PATH=$PATH make initrd DISTRO=${initrd_distro}
+          sudo -E PATH="$PATH" make initrd DISTRO="${initrd_distro}"
         ;;
 
-        *) echo "unsupported architecture: $(uname -m)"; exit 1;;
+        *) die "unsupported architecture: ${arch}" ;;
       esac
 
       # Install image
-      kata_image_dir=${SNAPCRAFT_PART_INSTALL}/usr/share/kata-containers
-      mkdir -p ${kata_image_dir}
-      cp kata-containers*.img ${kata_image_dir}
+      kata_image_dir="${SNAPCRAFT_PART_INSTALL}/usr/share/kata-containers"
+      mkdir -p "${kata_image_dir}"
+      cp kata-containers*.img "${kata_image_dir}"
 
   runtime:
     after: [godeps, image, cloud-hypervisor]
     plugin: nil
     build-attributes: [no-patchelf]
     override-build: |
-      # set GOPATH
-      export GOPATH=${SNAPCRAFT_STAGE}/gopath
-      export GOROOT=${SNAPCRAFT_STAGE}
-      export PATH="${GOROOT}/bin:${PATH}"
-      export GO111MODULE="auto"
-      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
+      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
 
-      cd ${kata_dir}/src/runtime
+      cd "${kata_dir}/src/runtime"
 
-      # setup arch
-      arch=$(uname -m)
-      if [ ${arch} = "ppc64le" ]; then
-        arch="ppc64"
-      fi
+      qemu_cmd="qemu-system-${qemu_arch}"
 
       # build and install runtime
       make \
-        PREFIX=/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr \
+        PREFIX="/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr" \
         SKIP_GO_VERSION_CHECK=1 \
-        QEMUCMD=qemu-system-$arch
+        QEMUCMD="${qemu_cmd}"
+
       make install \
         PREFIX=/usr \
-        DESTDIR=${SNAPCRAFT_PART_INSTALL} \
+        DESTDIR="${SNAPCRAFT_PART_INSTALL}" \
         SKIP_GO_VERSION_CHECK=1 \
-        QEMUCMD=qemu-system-$arch
+        QEMUCMD="${qemu_cmd}"
 
       if [ ! -f ${SNAPCRAFT_PART_INSTALL}/../../image/install/usr/share/kata-containers/kata-containers.img ]; then
         sed -i -e "s|^image =.*|initrd = \"/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr/share/kata-containers/kata-containers-initrd.img\"|" \
@@ -185,44 +179,37 @@ parts:
       - bison
       - flex
     override-build: |
-      yq=${SNAPCRAFT_STAGE}/yq
-      export PATH="${PATH}:${SNAPCRAFT_STAGE}"
-      export GOPATH=${SNAPCRAFT_STAGE}/gopath
-      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
-      versions_file="${kata_dir}/versions.yaml"
+      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
+
       kernel_version="$(${yq} r $versions_file assets.kernel.version)"
       #Remove extra 'v'
-      kernel_version=${kernel_version#v}
+      kernel_version="${kernel_version#v}"
 
-      [ "$(uname -m)" = "s390x" ] && sudo apt-get --no-install-recommends install -y libssl-dev
+      [ "${arch}" = "s390x" ] && sudo apt-get --no-install-recommends install -y libssl-dev
 
-      export GOPATH=${SNAPCRAFT_STAGE}/gopath
-      export GO111MODULE="auto"
-      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
-
-      cd ${kata_dir}/tools/packaging/kernel
+      cd "${kata_dir}/tools/packaging/kernel"
       kernel_dir_prefix="kata-linux-"
 
       # Setup and build kernel
-      ./build-kernel.sh -v ${kernel_version} -d setup
+      ./build-kernel.sh -v "${kernel_version}" -d setup
       cd ${kernel_dir_prefix}*
       make -j $(($(nproc)-1)) EXTRAVERSION=".container"
 
-      kernel_suffix=${kernel_version}.container
-      kata_kernel_dir=${SNAPCRAFT_PART_INSTALL}/usr/share/kata-containers
-      mkdir -p ${kata_kernel_dir}
+      kernel_suffix="${kernel_version}.container"
+      kata_kernel_dir="${SNAPCRAFT_PART_INSTALL}/usr/share/kata-containers"
+      mkdir -p "${kata_kernel_dir}"
 
       # Install bz kernel
-      make install INSTALL_PATH=${kata_kernel_dir} EXTRAVERSION=".container" || true
-      vmlinuz_name=vmlinuz-${kernel_suffix}
-      ln -sf ${vmlinuz_name} ${kata_kernel_dir}/vmlinuz.container
+      make install INSTALL_PATH="${kata_kernel_dir}" EXTRAVERSION=".container" || true
+      vmlinuz_name="vmlinuz-${kernel_suffix}"
+      ln -sf "${vmlinuz_name}" "${kata_kernel_dir}/vmlinuz.container"
 
       # Install raw kernel
-      vmlinux_path=vmlinux
-      [ "$(uname -m)" = "s390x" ] && vmlinux_path=arch/s390/boot/compressed/vmlinux
-      vmlinux_name=vmlinux-${kernel_suffix}
-      cp ${vmlinux_path} ${kata_kernel_dir}/${vmlinux_name}
-      ln -sf ${vmlinux_name} ${kata_kernel_dir}/vmlinux.container
+      vmlinux_path="vmlinux"
+      [ "${arch}" = "s390x" ] && vmlinux_path="arch/s390/boot/compressed/vmlinux"
+      vmlinux_name="vmlinux-${kernel_suffix}"
+      cp "${vmlinux_path}" "${kata_kernel_dir}/${vmlinux_name}"
+      ln -sf "${vmlinux_name}" "${kata_kernel_dir}/vmlinux.container"
 
   qemu:
     plugin: make
@@ -249,12 +236,8 @@ parts:
       - libselinux1-dev
       - ninja-build
     override-build: |
-      yq=${SNAPCRAFT_STAGE}/yq
-      export GOPATH=${SNAPCRAFT_STAGE}/gopath
-      export GO111MODULE="auto"
-      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
+      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
 
-      versions_file="${kata_dir}/versions.yaml"
       branch="$(${yq} r ${versions_file} assets.hypervisor.qemu.version)"
       url="$(${yq} r ${versions_file} assets.hypervisor.qemu.url)"
       commit=""
@@ -262,11 +245,11 @@ parts:
       patches_version_dir="${kata_dir}/tools/packaging/qemu/patches/tag_patches/${branch}"
 
       # download source
-      qemu_dir=${SNAPCRAFT_STAGE}/qemu
+      qemu_dir="${SNAPCRAFT_STAGE}/qemu"
       rm -rf "${qemu_dir}"
       git clone --depth 1 --branch ${branch} --single-branch ${url} "${qemu_dir}"
-      cd ${qemu_dir}
-      [ -z "${commit}" ] || git checkout ${commit}
+      cd "${qemu_dir}"
+      [ -z "${commit}" ] || git checkout "${commit}"
 
       [ -n "$(ls -A ui/keycodemapdb)" ] || git clone --depth 1 https://github.com/qemu/keycodemapdb ui/keycodemapdb/
       [ -n "$(ls -A capstone)" ] || git clone --depth 1 https://github.com/qemu/capstone capstone
@@ -277,10 +260,10 @@ parts:
       ${kata_dir}/tools/packaging/scripts/apply_patches.sh "${patches_version_dir}"
 
       # Only x86_64 supports libpmem
-      [ "$(uname -m)" = "x86_64" ] && sudo apt-get --no-install-recommends install -y apt-utils ca-certificates libpmem-dev
+      [ "${arch}" = "x86_64" ] && sudo apt-get --no-install-recommends install -y apt-utils ca-certificates libpmem-dev
 
-      configure_hypervisor=${kata_dir}/tools/packaging/scripts/configure-hypervisor.sh
-      chmod +x ${configure_hypervisor}
+      configure_hypervisor="${kata_dir}/tools/packaging/scripts/configure-hypervisor.sh"
+      chmod +x "${configure_hypervisor}"
       # static build. The --prefix, --libdir, --libexecdir, --datadir arguments are
       # based on PREFIX and set by configure-hypervisor.sh
       echo "$(PREFIX=/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr ${configure_hypervisor} -s kata-qemu) \
@@ -290,17 +273,17 @@ parts:
       # Copy QEMU configurations (Kconfigs)
       case "${branch}" in
       "v5.1.0")
-        cp -a ${kata_dir}/tools/packaging/qemu/default-configs/* default-configs
+        cp -a "${kata_dir}"/tools/packaging/qemu/default-configs/* default-configs
         ;;
 
       *)
-        cp -a ${kata_dir}/tools/packaging/qemu/default-configs/* configs/devices/
+        cp -a "${kata_dir}"/tools/packaging/qemu/default-configs/* configs/devices/
         ;;
       esac
 
       # build and install
       make -j $(($(nproc)-1))
-      make install DESTDIR=${SNAPCRAFT_PART_INSTALL}
+      make install DESTDIR="${SNAPCRAFT_PART_INSTALL}"
     prime:
       - -snap/
       - -usr/bin/qemu-ga
@@ -316,26 +299,67 @@ parts:
       # Hack: move qemu to /
       "snap/kata-containers/current/": "./"
 
+  virtiofsd:
+    plugin: nil
+    after: [godeps, rustdeps]
+    override-build: |
+      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
+
+      # Currently, powerpc makes use of the QEMU's C implementation.
+      # The other platforms make use of the new rust virtiofsd.
+      #
+      # See "tools/packaging/scripts/configure-hypervisor.sh".
+      if [ "${arch}" == "ppc64le" ]
+      then
+          echo "INFO: Building QEMU's C version of virtiofsd"
+          # Handled by the 'qemu' part, so nothing more to do here.
+          exit 0
+      else
+          echo "INFO: Building rust version of virtiofsd"
+      fi
+
+      cd "${kata_dir}"
+
+      export PATH=${PATH}:${HOME}/.cargo/bin
+      # Download the rust implementation of virtiofsd
+      tools/packaging/static-build/virtiofsd/build-static-virtiofsd.sh
+      sudo install \
+        --owner='root' \
+        --group='root' \
+        --mode=0755 \
+        -D \
+        --target-directory="${SNAPCRAFT_PART_INSTALL}/usr/libexec/" \
+        virtiofsd/virtiofsd
+
   cloud-hypervisor:
     plugin: nil
     after: [godeps]
     override-build: |
-      arch=$(uname -m)
-      if [ "{$arch}" == "aarch64" ] || [ "${arch}" == "x64_64" ]; then 
+      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
+
+      if [ "${arch}" == "aarch64" ] || [ "${arch}" == "x86_64" ]; then
           sudo apt-get -y update
           sudo apt-get -y install ca-certificates curl gnupg lsb-release
-          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
-          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          curl -fsSL https://download.docker.com/linux/ubuntu/gpg |\
+              sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+          distro_codename=$(lsb_release -cs)
+          echo "deb [arch=${dpkg_arch} signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu ${distro_codename} stable" |\
+              sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
           sudo apt-get -y update
           sudo apt-get -y install docker-ce docker-ce-cli containerd.io
           sudo systemctl start docker.socket
 
-          export GOPATH=${SNAPCRAFT_STAGE}/gopath
-          kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
-          cd ${kata_dir}
+          cd "${SNAPCRAFT_PROJECT_DIR}"
           sudo -E NO_TTY=true make cloud-hypervisor-tarball
-          tar xvJpf build/kata-static-cloud-hypervisor.tar.xz -C /tmp/
-          install -D /tmp/opt/kata/bin/cloud-hypervisor ${SNAPCRAFT_PART_INSTALL}/usr/bin/cloud-hypervisor
+
+          tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-cloud-hypervisor.tar.xz"
+          tmpdir=$(mktemp -d)
+
+          tar -xvJpf "${tarfile}" -C "${tmpdir}"
+
+          install -D "${tmpdir}/opt/kata/bin/cloud-hypervisor" "${SNAPCRAFT_PART_INSTALL}/usr/bin/cloud-hypervisor"
+
+          rm -rf "${tmpdir}"
       fi
 
 apps:

src/agent/Cargo.toml

@@ -18,7 +18,7 @@ serde_json = "1.0.39"
 scan_fmt = "0.2.3"
 scopeguard = "1.0.0"
 thiserror = "1.0.26"
-regex = "1.5.4"
+regex = "1.5.5"
 serial_test = "0.5.1"
 sysinfo = "0.23.0"
 
@@ -81,8 +81,3 @@ standard-oci-runtime = ["rustjail/standard-oci-runtime"]
 [[bin]]
 name = "kata-agent"
 path = "src/main.rs"
-
-[[bin]]
-name = "oci-kata-agent"
-path = "src/main.rs"
-required-features = ["standard-oci-runtime"]

src/agent/Makefile

@@ -113,14 +113,14 @@ logging-crate-tests:
 	make -C $(CWD)/../libs/logging
 
 $(TARGET_PATH): show-summary
-	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
+	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) $(if $(findstring release,$(BUILD_TYPE)),--release) $(EXTRA_RUSTFEATURES)
 
 $(GENERATED_FILES): %: %.in
 	@sed $(foreach r,$(GENERATED_REPLACEMENTS),-e 's|@$r@|$($r)|g') "$<" > "$@"
 
 ##TARGET optimize: optimized  build
 optimize: show-summary show-header
-	@RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
+	@RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) $(if $(findstring release,$(BUILD_TYPE)),--release) $(EXTRA_RUSTFEATURES)
 
 ##TARGET install: install agent
 install: install-services

src/agent/rustjail/Cargo.toml

@@ -20,7 +20,7 @@ protobuf = "=2.14.0"
 slog = "2.5.2"
 slog-scope = "4.1.2"
 scan_fmt = "0.2.6"
-regex = "1.5.4"
+regex = "1.5.5"
 path-absolutize = "1.2.0"
 anyhow = "1.0.32"
 cgroups = { package = "cgroups-rs", version = "0.2.8" }

src/agent/rustjail/src/console.rs

@@ -58,10 +58,7 @@ pub fn setup_master_console(socket_fd: RawFd) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::skip_if_not_root;
-    use std::fs::File;
     use std::os::unix::net::UnixListener;
-    use std::path::PathBuf;
     use tempfile::{self, tempdir};
 
     const CONSOLE_SOCKET: &str = "console-socket";

src/agent/rustjail/src/container.rs

@@ -42,7 +42,7 @@ use nix::pty;
 use nix::sched::{self, CloneFlags};
 use nix::sys::signal::{self, Signal};
 use nix::sys::stat::{self, Mode};
-use nix::unistd::{self, fork, ForkResult, Gid, Pid, Uid};
+use nix::unistd::{self, fork, ForkResult, Gid, Pid, Uid, User};
 use std::os::unix::fs::MetadataExt;
 use std::os::unix::io::AsRawFd;
 
@@ -64,8 +64,6 @@ use rlimit::{setrlimit, Resource, Rlim};
 use tokio::io::AsyncBufReadExt;
 use tokio::sync::Mutex;
 
-use crate::utils;
-
 pub const EXEC_FIFO_FILENAME: &str = "exec.fifo";
 
 const INIT: &str = "INIT";
@@ -78,9 +76,6 @@ const HOME_ENV_KEY: &str = "HOME";
 const PIDNS_FD: &str = "PIDNS_FD";
 const CONSOLE_SOCKET_FD: &str = "CONSOLE_SOCKET_FD";
 
-#[cfg(feature = "standard-oci-runtime")]
-const OCI_AGENT_BINARY: &str = "oci-kata-agent";
-
 #[derive(Debug)]
 pub struct ContainerStatus {
     pre_status: ContainerState,
@@ -227,7 +222,7 @@ pub trait BaseContainer {
     async fn start(&mut self, p: Process) -> Result<()>;
     async fn run(&mut self, p: Process) -> Result<()>;
     async fn destroy(&mut self) -> Result<()>;
-    fn exec(&mut self) -> Result<()>;
+    async fn exec(&mut self) -> Result<()>;
 }
 
 // LinuxContainer protected by Mutex
@@ -592,14 +587,20 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
 
     // only change stdio devices owner when user
     // isn't root.
-    if guser.uid != 0 {
-        set_stdio_permissions(guser.uid)?;
+    if !uid.is_root() {
+        set_stdio_permissions(uid)?;
     }
 
     setid(uid, gid)?;
 
     if !guser.additional_gids.is_empty() {
-        setgroups(guser.additional_gids.as_slice()).map_err(|e| {
+        let gids: Vec<Gid> = guser
+            .additional_gids
+            .iter()
+            .map(|gid| Gid::from_raw(*gid))
+            .collect();
+
+        unistd::setgroups(&gids).map_err(|e| {
             let _ = write_sync(
                 cwfd,
                 SYNC_FAILED,
@@ -639,11 +640,6 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         capabilities::drop_privileges(cfd_log, c)?;
     }
 
-    if init {
-        // notify parent to run poststart hooks
-        write_sync(cwfd, SYNC_SUCCESS, "")?;
-    }
-
     let args = oci_process.args.to_vec();
     let env = oci_process.env.to_vec();
 
@@ -665,12 +661,17 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         }
     }
 
-    // set the "HOME" env getting from "/etc/passwd", if
-    // there's no uid entry in /etc/passwd, set "/" as the
-    // home env.
     if env::var_os(HOME_ENV_KEY).is_none() {
-        let home_dir = utils::home_dir(guser.uid).unwrap_or_else(|_| String::from("/"));
-        env::set_var(HOME_ENV_KEY, home_dir);
+        // try to set "HOME" env by uid
+        if let Ok(Some(user)) = User::from_uid(Uid::from_raw(guser.uid)) {
+            if let Ok(user_home_dir) = user.dir.into_os_string().into_string() {
+                env::set_var(HOME_ENV_KEY, user_home_dir);
+            }
+        }
+        // set default home dir as "/" if "HOME" env is still empty
+        if env::var_os(HOME_ENV_KEY).is_none() {
+            env::set_var(HOME_ENV_KEY, String::from("/"));
+        }
     }
 
     let exec_file = Path::new(&args[0]);
@@ -730,7 +731,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
 // within the container to the specified user.
 // The ownership needs to match because it is created outside of
 // the container and needs to be localized.
-fn set_stdio_permissions(uid: libc::uid_t) -> Result<()> {
+fn set_stdio_permissions(uid: Uid) -> Result<()> {
     let meta = fs::metadata("/dev/null")?;
     let fds = [
         std::io::stdin().as_raw_fd(),
@@ -745,19 +746,13 @@ fn set_stdio_permissions(uid: libc::uid_t) -> Result<()> {
             continue;
         }
 
-        // According to the POSIX specification, -1 is used to indicate that owner and group
-        // are not to be changed.  Since uid_t and gid_t are unsigned types, we have to wrap
-        // around to get -1.
-        let gid = 0u32.wrapping_sub(1);
-
         // We only change the uid owner (as it is possible for the mount to
         // prefer a different gid, and there's no reason for us to change it).
         // The reason why we don't just leave the default uid=X mount setup is
         // that users expect to be able to actually use their console. Without
         // this code, you couldn't effectively run as a non-root user inside a
         // container and also have a console set up.
-        let res = unsafe { libc::fchown(*fd, uid, gid) };
-        Errno::result(res).map_err(|e| anyhow!(e).context("set stdio permissions failed"))?;
+        unistd::fchown(*fd, Some(uid), None).with_context(|| "set stdio permissions failed")?;
     }
 
     Ok(())
@@ -951,15 +946,7 @@ impl BaseContainer for LinuxContainer {
             let _ = unistd::close(pid);
         });
 
-        cfg_if::cfg_if! {
-            if #[cfg(feature = "standard-oci-runtime")] {
-                let exec_path = PathBuf::from(OCI_AGENT_BINARY);
-            }
-            else {
-                let exec_path = std::env::current_exe()?;
-            }
-        }
-
+        let exec_path = std::env::current_exe()?;
         let mut child = std::process::Command::new(exec_path);
 
         #[allow(unused_mut)]
@@ -1062,7 +1049,7 @@ impl BaseContainer for LinuxContainer {
         self.start(p).await?;
 
         if init {
-            self.exec()?;
+            self.exec().await?;
             self.status.transition(ContainerState::Running);
         }
 
@@ -1074,7 +1061,19 @@ impl BaseContainer for LinuxContainer {
         let st = self.oci_state()?;
 
         for pid in self.processes.keys() {
-            signal::kill(Pid::from_raw(*pid), Some(Signal::SIGKILL))?;
+            match signal::kill(Pid::from_raw(*pid), Some(Signal::SIGKILL)) {
+                Err(Errno::ESRCH) => {
+                    info!(
+                        self.logger,
+                        "kill encounters ESRCH, pid: {}, container: {}",
+                        pid,
+                        self.id.clone()
+                    );
+                    continue;
+                }
+                Err(err) => return Err(anyhow!(err)),
+                Ok(_) => continue,
+            }
         }
 
         if spec.hooks.is_some() {
@@ -1098,7 +1097,7 @@ impl BaseContainer for LinuxContainer {
         Ok(())
     }
 
-    fn exec(&mut self) -> Result<()> {
+    async fn exec(&mut self) -> Result<()> {
         let fifo = format!("{}/{}", &self.root, EXEC_FIFO_FILENAME);
         let fd = fcntl::open(fifo.as_str(), OFlag::O_WRONLY, Mode::from_bits_truncate(0))?;
         let data: &[u8] = &[0];
@@ -1110,6 +1109,26 @@ impl BaseContainer for LinuxContainer {
             .as_secs();
 
         self.status.transition(ContainerState::Running);
+
+        let spec = self
+            .config
+            .spec
+            .as_ref()
+            .ok_or_else(|| anyhow!("OCI spec was not found"))?;
+        let st = self.oci_state()?;
+
+        // run poststart hook
+        if spec.hooks.is_some() {
+            info!(self.logger, "poststart hook");
+            let hooks = spec
+                .hooks
+                .as_ref()
+                .ok_or_else(|| anyhow!("OCI hooks were not found"))?;
+            for h in hooks.poststart.iter() {
+                execute_hook(&self.logger, h, &st).await?;
+            }
+        }
+
         unistd::close(fd)?;
 
         Ok(())
@@ -1331,20 +1350,6 @@ async fn join_namespaces(
         // notify child run prestart hooks completed
         info!(logger, "notify child run prestart hook completed!");
         write_async(pipe_w, SYNC_SUCCESS, "").await?;
-
-        info!(logger, "notify child parent ready to run poststart hook!");
-        // wait to run poststart hook
-        read_async(pipe_r).await?;
-        info!(logger, "get ready to run poststart hook!");
-
-        // run poststart hook
-        if spec.hooks.is_some() {
-            info!(logger, "poststart hook");
-            let hooks = spec.hooks.as_ref().unwrap();
-            for h in hooks.poststart.iter() {
-                execute_hook(&logger, h, st).await?;
-            }
-        }
     }
 
     info!(logger, "wait for child process ready to run exec");
@@ -1473,12 +1478,6 @@ impl LinuxContainer {
     }
 }
 
-fn setgroups(grps: &[libc::gid_t]) -> Result<()> {
-    let ret = unsafe { libc::setgroups(grps.len(), grps.as_ptr() as *const libc::gid_t) };
-    Errno::result(ret).map(drop)?;
-    Ok(())
-}
-
 use std::fs::OpenOptions;
 use std::io::Write;
 
@@ -1648,6 +1647,7 @@ mod tests {
     use super::*;
     use crate::process::Process;
     use crate::skip_if_not_root;
+    use nix::unistd::Uid;
     use std::fs;
     use std::os::unix::fs::MetadataExt;
     use std::os::unix::io::AsRawFd;
@@ -1793,7 +1793,7 @@ mod tests {
         let old_uid = meta.uid();
 
         let uid = 1000;
-        set_stdio_permissions(uid).unwrap();
+        set_stdio_permissions(Uid::from_raw(uid)).unwrap();
 
         let meta = fs::metadata("/dev/stdin").unwrap();
         assert_eq!(meta.uid(), uid);
@@ -1805,7 +1805,7 @@ mod tests {
         assert_eq!(meta.uid(), uid);
 
         // restore the uid
-        set_stdio_permissions(old_uid).unwrap();
+        set_stdio_permissions(Uid::from_raw(old_uid)).unwrap();
     }
 
     #[test]
@@ -2086,9 +2086,10 @@ mod tests {
         assert!(ret.is_ok(), "Expecting Ok, Got {:?}", ret);
     }
 
-    #[test]
-    fn test_linuxcontainer_exec() {
-        let ret = new_linux_container_and_then(|mut c: LinuxContainer| c.exec());
+    #[tokio::test]
+    async fn test_linuxcontainer_exec() {
+        let (c, _dir) = new_linux_container();
+        let ret = c.unwrap().exec().await;
         assert!(ret.is_err(), "Expecting Err, Got {:?}", ret);
     }
 

src/agent/rustjail/src/lib.rs

@@ -41,7 +41,6 @@ pub mod seccomp;
 pub mod specconv;
 pub mod sync;
 pub mod sync_with_async;
-pub mod utils;
 pub mod validator;
 
 use std::collections::HashMap;

src/agent/rustjail/src/process.rs

@@ -5,7 +5,7 @@
 
 use libc::pid_t;
 use std::fs::File;
-use std::os::unix::io::RawFd;
+use std::os::unix::io::{AsRawFd, RawFd};
 use tokio::sync::mpsc::Sender;
 
 use nix::errno::Errno;
@@ -137,19 +137,25 @@ impl Process {
         info!(logger, "before create console socket!");
 
         if !p.tty {
-            info!(logger, "created console socket!");
+            if cfg!(feature = "standard-oci-runtime") {
+                p.stdin = Some(std::io::stdin().as_raw_fd());
+                p.stdout = Some(std::io::stdout().as_raw_fd());
+                p.stderr = Some(std::io::stderr().as_raw_fd());
+            } else {
+                info!(logger, "created console socket!");
 
-            let (stdin, pstdin) = unistd::pipe2(OFlag::O_CLOEXEC)?;
-            p.parent_stdin = Some(pstdin);
-            p.stdin = Some(stdin);
+                let (stdin, pstdin) = unistd::pipe2(OFlag::O_CLOEXEC)?;
+                p.parent_stdin = Some(pstdin);
+                p.stdin = Some(stdin);
 
-            let (pstdout, stdout) = create_extended_pipe(OFlag::O_CLOEXEC, pipe_size)?;
-            p.parent_stdout = Some(pstdout);
-            p.stdout = Some(stdout);
+                let (pstdout, stdout) = create_extended_pipe(OFlag::O_CLOEXEC, pipe_size)?;
+                p.parent_stdout = Some(pstdout);
+                p.stdout = Some(stdout);
 
-            let (pstderr, stderr) = create_extended_pipe(OFlag::O_CLOEXEC, pipe_size)?;
-            p.parent_stderr = Some(pstderr);
-            p.stderr = Some(stderr);
+                let (pstderr, stderr) = create_extended_pipe(OFlag::O_CLOEXEC, pipe_size)?;
+                p.parent_stderr = Some(pstderr);
+                p.stderr = Some(stderr);
+            }
         }
         Ok(p)
     }
@@ -284,5 +290,11 @@ mod tests {
         // group of the calling process.
         process.pid = 0;
         assert!(process.signal(libc::SIGCONT).is_ok());
+
+        if cfg!(feature = "standard-oci-runtime") {
+            assert_eq!(process.stdin.unwrap(), std::io::stdin().as_raw_fd());
+            assert_eq!(process.stdout.unwrap(), std::io::stdout().as_raw_fd());
+            assert_eq!(process.stderr.unwrap(), std::io::stderr().as_raw_fd());
+        }
     }
 }

src/agent/rustjail/src/utils.rs

@@ -1,120 +0,0 @@
-// Copyright (c) 2021 Ant Group
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-use anyhow::{anyhow, Context, Result};
-use libc::gid_t;
-use libc::uid_t;
-use std::fs::File;
-use std::io::{BufRead, BufReader};
-
-const PASSWD_FILE: &str = "/etc/passwd";
-
-// An entry from /etc/passwd
-#[derive(Debug, PartialEq, PartialOrd)]
-pub struct PasswdEntry {
-    // username
-    pub name: String,
-    // user password
-    pub passwd: String,
-    // user id
-    pub uid: uid_t,
-    // group id
-    pub gid: gid_t,
-    // user Information
-    pub gecos: String,
-    // home directory
-    pub dir: String,
-    // User's Shell
-    pub shell: String,
-}
-
-// get an entry for a given `uid` from `/etc/passwd`
-fn get_entry_by_uid(uid: uid_t, path: &str) -> Result<PasswdEntry> {
-    let file = File::open(path).with_context(|| format!("open file {}", path))?;
-    let mut reader = BufReader::new(file);
-
-    let mut line = String::new();
-    loop {
-        line.clear();
-        match reader.read_line(&mut line) {
-            Ok(0) => return Err(anyhow!(format!("file {} is empty", path))),
-            Ok(_) => (),
-            Err(e) => {
-                return Err(anyhow!(format!(
-                    "failed to read file {} with {:?}",
-                    path, e
-                )))
-            }
-        }
-
-        if line.starts_with('#') {
-            continue;
-        }
-
-        let parts: Vec<&str> = line.split(':').map(|part| part.trim()).collect();
-        if parts.len() != 7 {
-            continue;
-        }
-
-        match parts[2].parse() {
-            Err(_e) => continue,
-            Ok(new_uid) => {
-                if uid != new_uid {
-                    continue;
-                }
-
-                let entry = PasswdEntry {
-                    name: parts[0].to_string(),
-                    passwd: parts[1].to_string(),
-                    uid: new_uid,
-                    gid: parts[3].parse().unwrap_or(0),
-                    gecos: parts[4].to_string(),
-                    dir: parts[5].to_string(),
-                    shell: parts[6].to_string(),
-                };
-
-                return Ok(entry);
-            }
-        }
-    }
-}
-
-pub fn home_dir(uid: uid_t) -> Result<String> {
-    get_entry_by_uid(uid, PASSWD_FILE).map(|entry| entry.dir)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::io::Write;
-    use tempfile::Builder;
-
-    #[test]
-    fn test_get_entry_by_uid() {
-        let tmpdir = Builder::new().tempdir().unwrap();
-        let tmpdir_path = tmpdir.path().to_str().unwrap();
-        let temp_passwd = format!("{}/passwd", tmpdir_path);
-
-        let mut tempf = File::create(temp_passwd.as_str()).unwrap();
-        let passwd_entries = "root:x:0:0:root:/root0:/bin/bash
-root:x:1:0:root:/root1:/bin/bash
-#root:x:1:0:root:/rootx:/bin/bash
-root:x:2:0:root:/root2:/bin/bash
-root:x:3:0:root:/root3
-root:x:3:0:root:/root3:/bin/bash";
-        writeln!(tempf, "{}", passwd_entries).unwrap();
-
-        let entry = get_entry_by_uid(0, temp_passwd.as_str()).unwrap();
-        assert_eq!(entry.dir.as_str(), "/root0");
-
-        let entry = get_entry_by_uid(1, temp_passwd.as_str()).unwrap();
-        assert_eq!(entry.dir.as_str(), "/root1");
-
-        let entry = get_entry_by_uid(2, temp_passwd.as_str()).unwrap();
-        assert_eq!(entry.dir.as_str(), "/root2");
-
-        let entry = get_entry_by_uid(3, temp_passwd.as_str()).unwrap();
-        assert_eq!(entry.dir.as_str(), "/root3");
-    }
-}

src/agent/src/main.rs

@@ -27,7 +27,6 @@ use nix::unistd::{self, dup, Pid};
 use std::env;
 use std::ffi::OsStr;
 use std::fs::{self, File};
-use std::os::unix::ffi::OsStrExt;
 use std::os::unix::fs as unixfs;
 use std::os::unix::io::AsRawFd;
 use std::path::Path;
@@ -382,27 +381,13 @@ fn init_agent_as_init(logger: &Logger, unified_cgroup_hierarchy: bool) -> Result
     let contents_array: Vec<&str> = contents.split(' ').collect();
     let hostname = contents_array[0].trim();
 
-    if sethostname(OsStr::new(hostname)).is_err() {
+    if unistd::sethostname(OsStr::new(hostname)).is_err() {
         warn!(logger, "failed to set hostname");
     }
 
     Ok(())
 }
 
-#[instrument]
-fn sethostname(hostname: &OsStr) -> Result<()> {
-    let size = hostname.len() as usize;
-
-    let result =
-        unsafe { libc::sethostname(hostname.as_bytes().as_ptr() as *const libc::c_char, size) };
-
-    if result != 0 {
-        Err(anyhow!("failed to set hostname"))
-    } else {
-        Ok(())
-    }
-}
-
 // The Rust standard library had suppressed the default SIGPIPE behavior,
 // see https://github.com/rust-lang/rust/pull/13158.
 // Since the parent's signal handler would be inherited by it's child process,

src/agent/src/mount.rs

@@ -840,15 +840,13 @@ pub fn get_mount_fs_type_from_file(mount_file: &str, mount_point: &str) -> Resul
         return Err(anyhow!("Invalid mount point {}", mount_point));
     }
 
-    let file = File::open(mount_file)?;
-    let reader = BufReader::new(file);
+    let content = fs::read_to_string(mount_file)?;
 
     let re = Regex::new(format!("device .+ mounted on {} with fstype (.+)", mount_point).as_str())?;
 
     // Read the file line by line using the lines() iterator from std::io::BufRead.
-    for (_index, line) in reader.lines().enumerate() {
-        let line = line?;
-        let capes = match re.captures(line.as_str()) {
+    for (_index, line) in content.lines().enumerate() {
+        let capes = match re.captures(line) {
             Some(c) => c,
             None => continue,
         };
@@ -859,8 +857,9 @@ pub fn get_mount_fs_type_from_file(mount_file: &str, mount_point: &str) -> Resul
     }
 
     Err(anyhow!(
-        "failed to find FS type for mount point {}",
-        mount_point
+        "failed to find FS type for mount point {}, mount file content: {:?}",
+        mount_point,
+        content
     ))
 }
 
@@ -1018,7 +1017,7 @@ fn parse_options(option_list: Vec<String>) -> HashMap<String, String> {
 mod tests {
     use super::*;
     use crate::test_utils::test_utils::TestUserType;
-    use crate::{skip_if_not_root, skip_loop_if_not_root, skip_loop_if_root};
+    use crate::{skip_if_not_root, skip_loop_by_user, skip_loop_if_not_root, skip_loop_if_root};
     use protobuf::RepeatedField;
     use protocols::agent::FSGroup;
     use std::fs::File;
@@ -1112,11 +1111,7 @@ mod tests {
         for (i, d) in tests.iter().enumerate() {
             let msg = format!("test[{}]: {:?}", i, d);
 
-            if d.test_user == TestUserType::RootOnly {
-                skip_loop_if_not_root!(msg);
-            } else if d.test_user == TestUserType::NonRootOnly {
-                skip_loop_if_root!(msg);
-            }
+            skip_loop_by_user!(msg, d.test_user);
 
             let src: PathBuf;
             let dest: PathBuf;
@@ -1649,11 +1644,8 @@ mod tests {
 
         for (i, d) in tests.iter().enumerate() {
             let msg = format!("test[{}]: {:?}", i, d);
-            if d.test_user == TestUserType::RootOnly {
-                skip_loop_if_not_root!(msg);
-            } else if d.test_user == TestUserType::NonRootOnly {
-                skip_loop_if_root!(msg);
-            }
+
+            skip_loop_by_user!(msg, d.test_user);
 
             let drain = slog::Discard;
             let logger = slog::Logger::root(drain, o!());
@@ -1762,11 +1754,7 @@ mod tests {
 
         for (i, d) in tests.iter().enumerate() {
             let msg = format!("test[{}]: {:?}", i, d);
-            if d.test_user == TestUserType::RootOnly {
-                skip_loop_if_not_root!(msg);
-            } else if d.test_user == TestUserType::NonRootOnly {
-                skip_loop_if_root!(msg);
-            }
+            skip_loop_by_user!(msg, d.test_user);
 
             let drain = slog::Discard;
             let logger = slog::Logger::root(drain, o!());

src/agent/src/netlink.rs

@@ -523,7 +523,7 @@ impl Handle {
             .as_ref()
             .map(|to| to.address.as_str()) // Extract address field
             .and_then(|addr| if addr.is_empty() { None } else { Some(addr) }) // Make sure it's not empty
-            .ok_or(anyhow!(nix::Error::EINVAL))?;
+            .ok_or_else(|| anyhow!(nix::Error::EINVAL))?;
 
         let ip = IpAddr::from_str(ip_address)
             .map_err(|e| anyhow!("Failed to parse IP {}: {:?}", ip_address, e))?;
@@ -612,7 +612,7 @@ fn parse_mac_address(addr: &str) -> Result<[u8; 6]> {
 
     // Parse single Mac address block
     let mut parse_next = || -> Result<u8> {
-        let v = u8::from_str_radix(split.next().ok_or(anyhow!(nix::Error::EINVAL))?, 16)?;
+        let v = u8::from_str_radix(split.next().ok_or_else(|| anyhow!(nix::Error::EINVAL))?, 16)?;
         Ok(v)
     };
 

src/agent/src/random.rs

@@ -82,7 +82,7 @@ mod tests {
         if nix::unistd::Uid::effective().is_root() {
             assert!(ret.is_ok());
         } else {
-            assert!(!ret.is_ok());
+            assert!(ret.is_err());
         }
     }
 
@@ -90,6 +90,6 @@ mod tests {
     fn test_reseed_rng_zero_data() {
         let seed = [];
         let ret = reseed_rng(&seed);
-        assert!(!ret.is_ok());
+        assert!(ret.is_err());
     }
 }

src/agent/src/rpc.rs

@@ -23,8 +23,9 @@ use cgroups::freezer::FreezerState;
 use oci::{LinuxNamespace, Root, Spec};
 use protobuf::{Message, RepeatedField, SingularPtrField};
 use protocols::agent::{
-    AddSwapRequest, AgentDetails, CopyFileRequest, GuestDetailsResponse, Interfaces, Metrics,
-    OOMEvent, ReadStreamResponse, Routes, StatsContainerResponse, VolumeStatsRequest,
+    AddSwapRequest, AgentDetails, CopyFileRequest, GetIPTablesRequest, GetIPTablesResponse,
+    GuestDetailsResponse, Interfaces, Metrics, OOMEvent, ReadStreamResponse, Routes,
+    SetIPTablesRequest, SetIPTablesResponse, StatsContainerResponse, VolumeStatsRequest,
     WaitProcessResponse, WriteStreamResponse,
 };
 use protocols::csi::{VolumeCondition, VolumeStatsResponse, VolumeUsage, VolumeUsage_Unit};
@@ -40,13 +41,11 @@ use rustjail::specconv::CreateOpts;
 
 use nix::errno::Errno;
 use nix::mount::MsFlags;
-use nix::sys::stat;
+use nix::sys::{stat, statfs};
 use nix::unistd::{self, Pid};
 use rustjail::cgroups::Manager;
 use rustjail::process::ProcessOperations;
 
-use sysinfo::{DiskExt, System, SystemExt};
-
 use crate::device::{
     add_devices, get_virtio_blk_pci_device_name, update_device_cgroup, update_env_pci,
 };
@@ -71,25 +70,35 @@ use tracing::instrument;
 
 use libc::{self, c_char, c_ushort, pid_t, winsize, TIOCSWINSZ};
 use std::fs;
-use std::os::unix::fs::MetadataExt;
 use std::os::unix::prelude::PermissionsExt;
 use std::process::{Command, Stdio};
 use std::time::Duration;
 
 use nix::unistd::{Gid, Uid};
 use std::fs::{File, OpenOptions};
-use std::io::{BufRead, BufReader};
+use std::io::{BufRead, BufReader, Write};
 use std::os::unix::fs::FileExt;
 use std::path::PathBuf;
 
 const CONTAINER_BASE: &str = "/run/kata-containers";
 const MODPROBE_PATH: &str = "/sbin/modprobe";
 
+const IPTABLES_SAVE: &str = "/sbin/iptables-save";
+const IPTABLES_RESTORE: &str = "/sbin/iptables-restore";
+const IP6TABLES_SAVE: &str = "/sbin/ip6tables-save";
+const IP6TABLES_RESTORE: &str = "/sbin/ip6tables-restore";
+
 const ERR_CANNOT_GET_WRITER: &str = "Cannot get writer";
 const ERR_INVALID_BLOCK_SIZE: &str = "Invalid block size";
 const ERR_NO_LINUX_FIELD: &str = "Spec does not contain linux field";
 const ERR_NO_SANDBOX_PIDNS: &str = "Sandbox does not have sandbox_pidns";
 
+// IPTABLES_RESTORE_WAIT_SEC is the timeout value provided to iptables-restore --wait. Since we
+// don't expect other writers to iptables, we don't expect contention for grabbing the iptables
+// filesystem lock. Based on this, 5 seconds seems a resonable timeout period in case the lock is
+// not available.
+const IPTABLES_RESTORE_WAIT_SEC: u64 = 5;
+
 // Convenience macro to obtain the scope logger
 macro_rules! sl {
     () => {
@@ -260,7 +269,7 @@ impl AgentService {
             .get_container(&cid)
             .ok_or_else(|| anyhow!("Invalid container id"))?;
 
-        ctr.exec()?;
+        ctr.exec().await?;
 
         if sid == cid {
             return Ok(());
@@ -997,6 +1006,140 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         })
     }
 
+    async fn get_ip_tables(
+        &self,
+        ctx: &TtrpcContext,
+        req: GetIPTablesRequest,
+    ) -> ttrpc::Result<GetIPTablesResponse> {
+        trace_rpc_call!(ctx, "get_iptables", req);
+        is_allowed!(req);
+
+        info!(sl!(), "get_ip_tables: request received");
+
+        let cmd = if req.is_ipv6 {
+            IP6TABLES_SAVE
+        } else {
+            IPTABLES_SAVE
+        }
+        .to_string();
+
+        match Command::new(cmd.clone()).output() {
+            Ok(output) => Ok(GetIPTablesResponse {
+                data: output.stdout,
+                ..Default::default()
+            }),
+            Err(e) => {
+                warn!(sl!(), "failed to run {}: {:?}", cmd, e.kind());
+                return Err(ttrpc_error!(ttrpc::Code::INTERNAL, e));
+            }
+        }
+    }
+
+    async fn set_ip_tables(
+        &self,
+        ctx: &TtrpcContext,
+        req: SetIPTablesRequest,
+    ) -> ttrpc::Result<SetIPTablesResponse> {
+        trace_rpc_call!(ctx, "set_iptables", req);
+        is_allowed!(req);
+
+        info!(sl!(), "set_ip_tables request received");
+
+        let cmd = if req.is_ipv6 {
+            IP6TABLES_RESTORE
+        } else {
+            IPTABLES_RESTORE
+        }
+        .to_string();
+
+        let mut child = match Command::new(cmd.clone())
+            .arg("--wait")
+            .arg(IPTABLES_RESTORE_WAIT_SEC.to_string())
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .spawn()
+        {
+            Ok(child) => child,
+            Err(e) => {
+                warn!(sl!(), "failure to spawn {}: {:?}", cmd, e.kind());
+                return Err(ttrpc_error!(ttrpc::Code::INTERNAL, e));
+            }
+        };
+
+        let mut stdin = match child.stdin.take() {
+            Some(si) => si,
+            None => {
+                println!("failed to get stdin from child");
+                return Err(ttrpc_error!(
+                    ttrpc::Code::INTERNAL,
+                    "failed to take stdin from child".to_string()
+                ));
+            }
+        };
+
+        let (tx, rx) = tokio::sync::oneshot::channel::<i32>();
+        let handle = tokio::spawn(async move {
+            let _ = match stdin.write_all(&req.data) {
+                Ok(o) => o,
+                Err(e) => {
+                    warn!(sl!(), "error writing stdin: {:?}", e.kind());
+                    return;
+                }
+            };
+            if tx.send(1).is_err() {
+                warn!(sl!(), "stdin writer thread receiver dropped");
+            };
+        });
+
+        if tokio::time::timeout(Duration::from_secs(IPTABLES_RESTORE_WAIT_SEC), rx)
+            .await
+            .is_err()
+        {
+            return Err(ttrpc_error!(
+                ttrpc::Code::INTERNAL,
+                "timeout waiting for stdin writer to complete".to_string()
+            ));
+        }
+
+        if handle.await.is_err() {
+            return Err(ttrpc_error!(
+                ttrpc::Code::INTERNAL,
+                "stdin writer thread failure".to_string()
+            ));
+        }
+
+        let output = match child.wait_with_output() {
+            Ok(o) => o,
+            Err(e) => {
+                warn!(
+                    sl!(),
+                    "failure waiting for spawned {} to complete: {:?}",
+                    cmd,
+                    e.kind()
+                );
+                return Err(ttrpc_error!(ttrpc::Code::INTERNAL, e));
+            }
+        };
+
+        if !output.status.success() {
+            warn!(sl!(), "{} failed: {:?}", cmd, output.stderr);
+            return Err(ttrpc_error!(
+                ttrpc::Code::INTERNAL,
+                format!(
+                    "{} failed: {:?}",
+                    cmd,
+                    String::from_utf8_lossy(&output.stderr)
+                )
+            ));
+        }
+
+        Ok(SetIPTablesResponse {
+            data: output.stdout,
+            ..Default::default()
+        })
+    }
+
     async fn list_interfaces(
         &self,
         ctx: &TtrpcContext,
@@ -1468,20 +1611,12 @@ fn get_memory_info(
 fn get_volume_capacity_stats(path: &str) -> Result<VolumeUsage> {
     let mut usage = VolumeUsage::new();
 
-    let s = System::new();
-    for disk in s.disks() {
-        if let Some(v) = disk.name().to_str() {
-            if v.to_string().eq(path) {
-                usage.available = disk.available_space();
-                usage.total = disk.total_space();
-                usage.used = usage.total - usage.available;
-                usage.unit = VolumeUsage_Unit::BYTES; // bytes
-                break;
-            }
-        } else {
-            return Err(anyhow!(nix::Error::EINVAL));
-        }
-    }
+    let stat = statfs::statfs(path)?;
+    let block_size = stat.block_size() as u64;
+    usage.total = stat.blocks() * block_size;
+    usage.available = stat.blocks_free() * block_size;
+    usage.used = usage.total - usage.available;
+    usage.unit = VolumeUsage_Unit::BYTES;
 
     Ok(usage)
 }
@@ -1489,20 +1624,11 @@ fn get_volume_capacity_stats(path: &str) -> Result<VolumeUsage> {
 fn get_volume_inode_stats(path: &str) -> Result<VolumeUsage> {
     let mut usage = VolumeUsage::new();
 
-    let s = System::new();
-    for disk in s.disks() {
-        if let Some(v) = disk.name().to_str() {
-            if v.to_string().eq(path) {
-                let meta = fs::metadata(disk.mount_point())?;
-                let inode = meta.ino();
-                usage.used = inode;
-                usage.unit = VolumeUsage_Unit::INODES;
-                break;
-            }
-        } else {
-            return Err(anyhow!(nix::Error::EINVAL));
-        }
-    }
+    let stat = statfs::statfs(path)?;
+    usage.total = stat.files();
+    usage.available = stat.files_free();
+    usage.used = usage.total - usage.available;
+    usage.unit = VolumeUsage_Unit::INODES;
 
     Ok(usage)
 }
@@ -1667,34 +1793,25 @@ fn is_signal_handled(proc_status_file: &str, signum: u32) -> bool {
     let sig_mask: u64 = 1 << shift_count;
     let reader = BufReader::new(file);
 
-    // Read the file line by line using the lines() iterator from std::io::BufRead.
-    for (_index, line) in reader.lines().enumerate() {
-        let line = match line {
-            Ok(l) => l,
-            Err(_) => {
-                warn!(sl!(), "failed to read file {}", proc_status_file);
-                return false;
-            }
-        };
-        if line.starts_with("SigCgt:") {
+    // read lines start with SigBlk/SigIgn/SigCgt and check any match the signal mask
+    reader
+        .lines()
+        .flatten()
+        .filter(|line| {
+            line.starts_with("SigBlk:")
+                || line.starts_with("SigIgn:")
+                || line.starts_with("SigCgt:")
+        })
+        .any(|line| {
             let mask_vec: Vec<&str> = line.split(':').collect();
-            if mask_vec.len() != 2 {
-                warn!(sl!(), "parse the SigCgt field failed");
-                return false;
-            }
-            let sig_cgt_str = mask_vec[1];
-            let sig_cgt_mask = match u64::from_str_radix(sig_cgt_str, 16) {
-                Ok(h) => h,
-                Err(_) => {
-                    warn!(sl!(), "failed to parse the str {} to hex", sig_cgt_str);
-                    return false;
+            if mask_vec.len() == 2 {
+                let sig_str = mask_vec[1].trim();
+                if let Ok(sig) = u64::from_str_radix(sig_str, 16) {
+                    return sig & sig_mask == sig_mask;
                 }
-            };
-
-            return (sig_cgt_mask & sig_mask) == sig_mask;
-        }
-    }
-    false
+            }
+            false
+        })
 }
 
 fn do_mem_hotplug_by_probe(addrs: &[u64]) -> Result<()> {
@@ -1894,7 +2011,12 @@ fn load_kernel_module(module: &protocols::agent::KernelModule) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::{assert_result, namespace::Namespace, protocols::agent_ttrpc::AgentService as _};
+    use crate::{
+        assert_result, namespace::Namespace, protocols::agent_ttrpc::AgentService as _,
+        skip_if_not_root,
+    };
+    use nix::mount;
+    use nix::sched::{unshare, CloneFlags};
     use oci::{Hook, Hooks, Linux, LinuxNamespace};
     use tempfile::{tempdir, TempDir};
     use ttrpc::{r#async::TtrpcContext, MessageHeader};
@@ -2453,6 +2575,26 @@ OtherField:other
                 signum: 4,
                 result: true,
             },
+            TestData {
+                status_file_data: Some("SigCgt:\t000000004b813efb"),
+                signum: 4,
+                result: true,
+            },
+            TestData {
+                status_file_data: Some("SigCgt: 000000004b813efb"),
+                signum: 4,
+                result: true,
+            },
+            TestData {
+                status_file_data: Some("SigCgt:000000004b813efb "),
+                signum: 4,
+                result: true,
+            },
+            TestData {
+                status_file_data: Some("SigCgt:\t000000004b813efb "),
+                signum: 4,
+                result: true,
+            },
             TestData {
                 status_file_data: Some("SigCgt:000000004b813efb"),
                 signum: 3,
@@ -2491,7 +2633,12 @@ OtherField:other
             TestData {
                 status_file_data: Some("SigBlk:0000000000000001"),
                 signum: 1,
-                result: false,
+                result: true,
+            },
+            TestData {
+                status_file_data: Some("SigIgn:0000000000000001"),
+                signum: 1,
+                result: true,
             },
             TestData {
                 status_file_data: None,
@@ -2749,4 +2896,224 @@ OtherField:other
             }
         }
     }
+
+    #[tokio::test]
+    async fn test_volume_capacity_stats() {
+        skip_if_not_root!();
+
+        // Verify error if path does not exist
+        assert!(get_volume_capacity_stats("/does-not-exist").is_err());
+
+        // Create a new tmpfs mount, and verify the initial values
+        let mount_dir = tempfile::tempdir().unwrap();
+        mount::mount(
+            Some("tmpfs"),
+            mount_dir.path().to_str().unwrap(),
+            Some("tmpfs"),
+            mount::MsFlags::empty(),
+            None::<&str>,
+        )
+        .unwrap();
+        let mut stats = get_volume_capacity_stats(mount_dir.path().to_str().unwrap()).unwrap();
+        assert_eq!(stats.used, 0);
+        assert_ne!(stats.available, 0);
+        let available = stats.available;
+
+        // Verify that writing a file will result in increased utilization
+        fs::write(mount_dir.path().join("file.dat"), "foobar").unwrap();
+        stats = get_volume_capacity_stats(mount_dir.path().to_str().unwrap()).unwrap();
+
+        assert_eq!(stats.used, 4 * 1024);
+        assert_eq!(stats.available, available - 4 * 1024);
+    }
+
+    #[tokio::test]
+    async fn test_get_volume_inode_stats() {
+        skip_if_not_root!();
+
+        // Verify error if path does not exist
+        assert!(get_volume_inode_stats("/does-not-exist").is_err());
+
+        // Create a new tmpfs mount, and verify the initial values
+        let mount_dir = tempfile::tempdir().unwrap();
+        mount::mount(
+            Some("tmpfs"),
+            mount_dir.path().to_str().unwrap(),
+            Some("tmpfs"),
+            mount::MsFlags::empty(),
+            None::<&str>,
+        )
+        .unwrap();
+        let mut stats = get_volume_inode_stats(mount_dir.path().to_str().unwrap()).unwrap();
+        assert_eq!(stats.used, 1);
+        assert_ne!(stats.available, 0);
+        let available = stats.available;
+
+        // Verify that creating a directory and writing a file will result in increased utilization
+        let dir = mount_dir.path().join("foobar");
+        fs::create_dir_all(&dir).unwrap();
+        fs::write(dir.as_path().join("file.dat"), "foobar").unwrap();
+        stats = get_volume_inode_stats(mount_dir.path().to_str().unwrap()).unwrap();
+
+        assert_eq!(stats.used, 3);
+        assert_eq!(stats.available, available - 2);
+    }
+
+    #[tokio::test]
+    async fn test_ip_tables() {
+        skip_if_not_root!();
+
+        let logger = slog::Logger::root(slog::Discard, o!());
+        let sandbox = Sandbox::new(&logger).unwrap();
+        let agent_service = Box::new(AgentService {
+            sandbox: Arc::new(Mutex::new(sandbox)),
+        });
+
+        let ctx = mk_ttrpc_context();
+
+        // Move to a new netns in order to ensure we don't trash the hosts' iptables
+        unshare(CloneFlags::CLONE_NEWNET).unwrap();
+
+        // Get initial iptables, we expect to be empty:
+        let result = agent_service
+            .get_ip_tables(
+                &ctx,
+                GetIPTablesRequest {
+                    is_ipv6: false,
+                    ..Default::default()
+                },
+            )
+            .await;
+        assert!(result.is_ok(), "get ip tables should succeed");
+        assert_eq!(
+            result.unwrap().data.len(),
+            0,
+            "ip tables should be empty initially"
+        );
+
+        // Initial ip6 ip tables should also be empty:
+        let result = agent_service
+            .get_ip_tables(
+                &ctx,
+                GetIPTablesRequest {
+                    is_ipv6: true,
+                    ..Default::default()
+                },
+            )
+            .await;
+        assert!(result.is_ok(), "get ip6 tables should succeed");
+        assert_eq!(
+            result.unwrap().data.len(),
+            0,
+            "ip tables should be empty initially"
+        );
+
+        // Verify that attempting to write 'empty' iptables results in no error:
+        let empty_rules = "";
+        let result = agent_service
+            .set_ip_tables(
+                &ctx,
+                SetIPTablesRequest {
+                    is_ipv6: false,
+                    data: empty_rules.as_bytes().to_vec(),
+                    ..Default::default()
+                },
+            )
+            .await;
+        assert!(result.is_ok(), "set ip tables with no data should succeed");
+
+        // Verify that attempting to write "garbage" iptables results in an error:
+        let garbage_rules = r#"
+this
+is
+just garbage
+"#;
+        let result = agent_service
+            .set_ip_tables(
+                &ctx,
+                SetIPTablesRequest {
+                    is_ipv6: false,
+                    data: garbage_rules.as_bytes().to_vec(),
+                    ..Default::default()
+                },
+            )
+            .await;
+        assert!(result.is_err(), "set iptables with garbage should fail");
+
+        // Verify setup of valid iptables:Setup  valid set of iptables:
+        let valid_rules = r#"
+*nat
+-A PREROUTING -d 192.168.103.153/32 -j DNAT --to-destination 192.168.188.153
+
+COMMIT
+
+"#;
+        let result = agent_service
+            .set_ip_tables(
+                &ctx,
+                SetIPTablesRequest {
+                    is_ipv6: false,
+                    data: valid_rules.as_bytes().to_vec(),
+                    ..Default::default()
+                },
+            )
+            .await;
+        assert!(result.is_ok(), "set ip tables should succeed");
+
+        let result = agent_service
+            .get_ip_tables(
+                &ctx,
+                GetIPTablesRequest {
+                    is_ipv6: false,
+                    ..Default::default()
+                },
+            )
+            .await
+            .unwrap();
+        assert!(!result.data.is_empty(), "we should have non-zero output:");
+        assert!(
+            std::str::from_utf8(&*result.data).unwrap().contains(
+                "PREROUTING -d 192.168.103.153/32 -j DNAT --to-destination 192.168.188.153"
+            ),
+            "We should see the resulting rule"
+        );
+
+        // Verify setup of valid ip6tables:
+        let valid_ipv6_rules = r#"
+*filter
+-A INPUT -s 2001:db8:100::1/128 -i sit+ -p tcp -m tcp --sport 512:65535
+
+COMMIT
+
+"#;
+        let result = agent_service
+            .set_ip_tables(
+                &ctx,
+                SetIPTablesRequest {
+                    is_ipv6: true,
+                    data: valid_ipv6_rules.as_bytes().to_vec(),
+                    ..Default::default()
+                },
+            )
+            .await;
+        assert!(result.is_ok(), "set ip6 tables should succeed");
+
+        let result = agent_service
+            .get_ip_tables(
+                &ctx,
+                GetIPTablesRequest {
+                    is_ipv6: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .unwrap();
+        assert!(!result.data.is_empty(), "we should have non-zero output:");
+        assert!(
+            std::str::from_utf8(&*result.data)
+                .unwrap()
+                .contains("INPUT -s 2001:db8:100::1/128 -i sit+ -p tcp -m tcp --sport 512:65535"),
+            "We should see the resulting rule"
+        );
+    }
 }

src/agent/src/sandbox.rs

@@ -470,7 +470,7 @@ fn online_memory(logger: &Logger) -> Result<()> {
 
 #[cfg(test)]
 mod tests {
-    use super::Sandbox;
+    use super::*;
     use crate::{mount::baremount, skip_if_not_root};
     use anyhow::{anyhow, Error};
     use nix::mount::MsFlags;
@@ -480,6 +480,7 @@ mod tests {
     use rustjail::specconv::CreateOpts;
     use slog::Logger;
     use std::fs::{self, File};
+    use std::io::prelude::*;
     use std::os::unix::fs::PermissionsExt;
     use std::path::Path;
     use tempfile::{tempdir, Builder, TempDir};
@@ -847,4 +848,259 @@ mod tests {
         let p = s.find_container_process("not-exist-cid", "");
         assert!(p.is_err(), "Expecting Error, Got {:?}", p);
     }
+
+    #[tokio::test]
+    async fn test_find_process() {
+        let logger = slog::Logger::root(slog::Discard, o!());
+
+        let test_pids = [std::i32::MIN, -1, 0, 1, std::i32::MAX];
+
+        for test_pid in test_pids {
+            let mut s = Sandbox::new(&logger).unwrap();
+            let (mut linux_container, _root) = create_linuxcontainer();
+
+            let mut test_process = Process::new(
+                &logger,
+                &oci::Process::default(),
+                "this_is_a_test_process",
+                true,
+                1,
+            )
+            .unwrap();
+            // processes interally only have pids when manually set
+            test_process.pid = test_pid;
+
+            linux_container.processes.insert(test_pid, test_process);
+
+            s.add_container(linux_container);
+
+            let find_result = s.find_process(test_pid);
+
+            // test first if it finds anything
+            assert!(find_result.is_some(), "Should be able to find a process");
+
+            let found_process = find_result.unwrap();
+
+            // then test if it founds the correct process
+            assert_eq!(
+                found_process.pid, test_pid,
+                "Should be able to find correct process"
+            );
+        }
+
+        // to test for nonexistent pids, any pid that isn't the one set
+        // above should work, as linuxcontainer starts with no processes
+        let mut s = Sandbox::new(&logger).unwrap();
+
+        let nonexistent_test_pid = 1234;
+
+        let find_result = s.find_process(nonexistent_test_pid);
+
+        assert!(
+            find_result.is_none(),
+            "Shouldn't find a process for non existent pid"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_online_resources() {
+        #[derive(Debug, Default)]
+        struct TestFile {
+            name: String,
+            content: String,
+        }
+
+        #[derive(Debug, Default)]
+        struct TestDirectory<'a> {
+            name: String,
+            files: &'a [TestFile],
+        }
+
+        #[derive(Debug)]
+        struct TestData<'a> {
+            directory_autogen_name: String,
+            number_autogen_directories: u32,
+
+            extra_directories: &'a [TestDirectory<'a>],
+            pattern: String,
+            to_enable: i32,
+
+            result: Result<i32>,
+        }
+
+        impl Default for TestData<'_> {
+            fn default() -> Self {
+                TestData {
+                    directory_autogen_name: Default::default(),
+                    number_autogen_directories: Default::default(),
+                    extra_directories: Default::default(),
+                    pattern: Default::default(),
+                    to_enable: Default::default(),
+                    result: Ok(Default::default()),
+                }
+            }
+        }
+
+        let tests = &[
+            // 4 well formed directories, request enabled 4,
+            // correct result 4 enabled, should pass
+            TestData {
+                directory_autogen_name: String::from("cpu"),
+                number_autogen_directories: 4,
+                pattern: String::from(r"cpu[0-9]+"),
+                to_enable: 4,
+                result: Ok(4),
+                ..Default::default()
+            },
+            // 0 well formed directories, request enabled 4,
+            // correct result 0 enabled, should pass
+            TestData {
+                number_autogen_directories: 0,
+                to_enable: 4,
+                result: Ok(0),
+                ..Default::default()
+            },
+            // 10 well formed directories, request enabled 4,
+            // correct result 4 enabled, should pass
+            TestData {
+                directory_autogen_name: String::from("cpu"),
+                number_autogen_directories: 10,
+                pattern: String::from(r"cpu[0-9]+"),
+                to_enable: 4,
+                result: Ok(4),
+                ..Default::default()
+            },
+            // 0 well formed directories, request enabled 0,
+            // correct result 0 enabled, should pass
+            TestData {
+                number_autogen_directories: 0,
+                pattern: String::from(r"cpu[0-9]+"),
+                to_enable: 0,
+                result: Ok(0),
+                ..Default::default()
+            },
+            // 4 well formed directories, 1 malformed (no online file),
+            // request enable 5, correct result 4
+            TestData {
+                directory_autogen_name: String::from("cpu"),
+                number_autogen_directories: 4,
+                pattern: String::from(r"cpu[0-9]+"),
+                extra_directories: &[TestDirectory {
+                    name: String::from("cpu4"),
+                    files: &[],
+                }],
+                to_enable: 5,
+                result: Ok(4),
+            },
+            // 3 malformed directories (no online files),
+            // request enable 3, correct result 0
+            TestData {
+                pattern: String::from(r"cpu[0-9]+"),
+                extra_directories: &[
+                    TestDirectory {
+                        name: String::from("cpu0"),
+                        files: &[],
+                    },
+                    TestDirectory {
+                        name: String::from("cpu1"),
+                        files: &[],
+                    },
+                    TestDirectory {
+                        name: String::from("cpu2"),
+                        files: &[],
+                    },
+                ],
+                to_enable: 3,
+                result: Ok(0),
+                ..Default::default()
+            },
+            // 1 malformed directories (online file with content "1"),
+            // request enable 1, correct result 0
+            TestData {
+                pattern: String::from(r"cpu[0-9]+"),
+                extra_directories: &[TestDirectory {
+                    name: String::from("cpu0"),
+                    files: &[TestFile {
+                        name: SYSFS_ONLINE_FILE.to_string(),
+                        content: String::from("1"),
+                    }],
+                }],
+                to_enable: 1,
+                result: Ok(0),
+                ..Default::default()
+            },
+            // 2 well formed directories, 1 malformed (online file with content "1"),
+            // request enable 3, correct result 2
+            TestData {
+                directory_autogen_name: String::from("cpu"),
+                number_autogen_directories: 2,
+                pattern: String::from(r"cpu[0-9]+"),
+                extra_directories: &[TestDirectory {
+                    name: String::from("cpu2"),
+                    files: &[TestFile {
+                        name: SYSFS_ONLINE_FILE.to_string(),
+                        content: String::from("1"),
+                    }],
+                }],
+                to_enable: 3,
+                result: Ok(2),
+            },
+        ];
+
+        let logger = slog::Logger::root(slog::Discard, o!());
+        let tmpdir = Builder::new().tempdir().unwrap();
+        let tmpdir_path = tmpdir.path().to_str().unwrap();
+
+        for (i, d) in tests.iter().enumerate() {
+            let current_test_dir_path = format!("{}/test_{}", tmpdir_path, i);
+            fs::create_dir(&current_test_dir_path).unwrap();
+
+            // create numbered directories and fill using root name
+            for j in 0..d.number_autogen_directories {
+                let subdir_path = format!(
+                    "{}/{}{}",
+                    current_test_dir_path, d.directory_autogen_name, j
+                );
+                let subfile_path = format!("{}/{}", subdir_path, SYSFS_ONLINE_FILE);
+                fs::create_dir(&subdir_path).unwrap();
+                let mut subfile = File::create(subfile_path).unwrap();
+                subfile.write_all(b"0").unwrap();
+            }
+            // create extra directories and fill to specification
+            for j in d.extra_directories {
+                let subdir_path = format!("{}/{}", current_test_dir_path, j.name);
+                fs::create_dir(&subdir_path).unwrap();
+                for file in j.files {
+                    let subfile_path = format!("{}/{}", subdir_path, file.name);
+                    let mut subfile = File::create(&subfile_path).unwrap();
+                    subfile.write_all(file.content.as_bytes()).unwrap();
+                }
+            }
+
+            // run created directory structure against online_resources
+            let result = online_resources(&logger, &current_test_dir_path, &d.pattern, d.to_enable);
+
+            let mut msg = format!(
+                "test[{}]: {:?}, expected {}, actual {}",
+                i,
+                d,
+                d.result.is_ok(),
+                result.is_ok()
+            );
+
+            assert_eq!(result.is_ok(), d.result.is_ok(), "{}", msg);
+
+            if d.result.is_ok() {
+                let test_result_val = *d.result.as_ref().ok().unwrap();
+                let result_val = result.ok().unwrap();
+
+                msg = format!(
+                    "test[{}]: {:?}, expected {}, actual {}",
+                    i, d, test_result_val, result_val
+                );
+
+                assert_eq!(test_result_val, result_val, "{}", msg);
+            }
+        }
+    }
 }

src/agent/src/test_utils.rs

@@ -85,4 +85,15 @@ pub mod test_utils {
             }
         };
     }
+
+    #[macro_export]
+    macro_rules! skip_loop_by_user {
+        ($msg:expr, $user:expr) => {
+            if $user == TestUserType::RootOnly {
+                skip_loop_if_not_root!($msg);
+            } else if $user == TestUserType::NonRootOnly {
+                skip_loop_if_root!($msg);
+            }
+        };
+    }
 }

src/libs/Cargo.lock

@@ -1,11 +1,30 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "anyhow"
+version = "1.0.57"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc"
+
 [[package]]
 name = "arc-swap"
 version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c5d78ce20460b82d3fa150275ed9d55e21064fc7951177baacf86a145c4a4b1f"
 
+[[package]]
+name = "async-trait"
+version = "0.1.53"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "autocfg"
 version = "1.0.1"
@@ -14,9 +33,37 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
 
 [[package]]
 name = "bitflags"
-version = "1.3.2"
+version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
+
+[[package]]
+name = "byteorder"
+version = "1.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
+
+[[package]]
+name = "bytes"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c"
+dependencies = [
+ "byteorder",
+ "iovec",
+]
+
+[[package]]
+name = "bytes"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"
+
+[[package]]
+name = "cc"
+version = "1.0.73"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
 
 [[package]]
 name = "cfg-if"
@@ -49,14 +96,31 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.6"
+version = "0.8.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cfcae03edb34f947e64acdb1c33ec169824e20657e9ecb61cef6c8c74dcb8120"
+checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38"
 dependencies = [
  "cfg-if",
  "lazy_static",
 ]
 
+[[package]]
+name = "derive-new"
+version = "0.5.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3418329ca0ad70234b9735dc4ceed10af4df60eff9c8e7b06cb5e520d92c3535"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "either"
+version = "1.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
+
 [[package]]
 name = "fastrand"
 version = "1.6.0"
@@ -66,6 +130,126 @@ dependencies = [
  "instant",
 ]
 
+[[package]]
+name = "fixedbitset"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d"
+
+[[package]]
+name = "futures"
+version = "0.3.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f73fe65f54d1e12b726f517d3e2135ca3125a437b6d998caf1962961f7172d9e"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-executor",
+ "futures-io",
+ "futures-sink",
+ "futures-task",
+ "futures-util",
+]
+
+[[package]]
+name = "futures-channel"
+version = "0.3.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+]
+
+[[package]]
+name = "futures-core"
+version = "0.3.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3"
+
+[[package]]
+name = "futures-executor"
+version = "0.3.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "futures-util",
+]
+
+[[package]]
+name = "futures-io"
+version = "0.3.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b"
+
+[[package]]
+name = "futures-macro"
+version = "0.3.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "futures-sink"
+version = "0.3.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868"
+
+[[package]]
+name = "futures-task"
+version = "0.3.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a"
+
+[[package]]
+name = "futures-util"
+version = "0.3.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-io",
+ "futures-macro",
+ "futures-sink",
+ "futures-task",
+ "memchr",
+ "pin-project-lite",
+ "pin-utils",
+ "slab",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
+
+[[package]]
+name = "heck"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
+dependencies = [
+ "unicode-segmentation",
+]
+
+[[package]]
+name = "indexmap"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee"
+dependencies = [
+ "autocfg",
+ "hashbrown",
+]
+
 [[package]]
 name = "instant"
 version = "0.1.12"
@@ -75,6 +259,24 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "iovec"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "itertools"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.1"
@@ -89,9 +291,18 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.113"
+version = "0.2.124"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eef78b64d87775463c549fbd80e19249ef436ea3bf1de2a1eb7e717ec7fab1e9"
+checksum = "21a41fed9d98f27ab1c6d161da622a4fa35e8a54a8adc24bbf3ddd0ef70b0e50"
+
+[[package]]
+name = "log"
+version = "0.4.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8"
+dependencies = [
+ "cfg-if",
+]
 
 [[package]]
 name = "logging"
@@ -105,6 +316,85 @@ dependencies = [
  "tempfile",
 ]
 
+[[package]]
+name = "memchr"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
+
+[[package]]
+name = "memoffset"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "mio"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9"
+dependencies = [
+ "libc",
+ "log",
+ "miow",
+ "ntapi",
+ "wasi 0.11.0+wasi-snapshot-preview1",
+ "winapi",
+]
+
+[[package]]
+name = "miow"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "multimap"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
+
+[[package]]
+name = "nix"
+version = "0.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f5e06129fb611568ef4e868c14b326274959aa70ff7776e9d55323531c374945"
+dependencies = [
+ "bitflags",
+ "cc",
+ "cfg-if",
+ "libc",
+ "memoffset",
+]
+
+[[package]]
+name = "nix"
+version = "0.23.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f866317acbd3a240710c63f065ffb1e4fd466259045ccb504130b7f668f35c6"
+dependencies = [
+ "bitflags",
+ "cc",
+ "cfg-if",
+ "libc",
+ "memoffset",
+]
+
+[[package]]
+name = "ntapi"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "num-integer"
 version = "0.1.44"
@@ -130,6 +420,138 @@ version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5"
 
+[[package]]
+name = "petgraph"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7"
+dependencies = [
+ "fixedbitset",
+ "indexmap",
+]
+
+[[package]]
+name = "pin-project-lite"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c"
+
+[[package]]
+name = "pin-utils"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.37"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1"
+dependencies = [
+ "unicode-xid",
+]
+
+[[package]]
+name = "prost"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de5e2533f59d08fcf364fd374ebda0692a70bd6d7e66ef97f306f45c6c5d8020"
+dependencies = [
+ "bytes 1.1.0",
+ "prost-derive",
+]
+
+[[package]]
+name = "prost-build"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "355f634b43cdd80724ee7848f95770e7e70eefa6dcf14fea676216573b8fd603"
+dependencies = [
+ "bytes 1.1.0",
+ "heck",
+ "itertools",
+ "log",
+ "multimap",
+ "petgraph",
+ "prost",
+ "prost-types",
+ "tempfile",
+ "which",
+]
+
+[[package]]
+name = "prost-derive"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "600d2f334aa05acb02a755e217ef1ab6dea4d51b58b7846588b747edec04efba"
+dependencies = [
+ "anyhow",
+ "itertools",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "prost-types"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "603bbd6394701d13f3f25aada59c7de9d35a6a5887cfc156181234a44002771b"
+dependencies = [
+ "bytes 1.1.0",
+ "prost",
+]
+
+[[package]]
+name = "protobuf"
+version = "2.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e86d370532557ae7573551a1ec8235a0f8d6cb276c7c9e6aa490b511c447485"
+dependencies = [
+ "serde",
+ "serde_derive",
+]
+
+[[package]]
+name = "protobuf-codegen"
+version = "2.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de113bba758ccf2c1ef816b127c958001b7831136c9bc3f8e9ec695ac4e82b0c"
+dependencies = [
+ "protobuf",
+]
+
+[[package]]
+name = "protobuf-codegen-pure"
+version = "2.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d1a4febc73bf0cada1d77c459a0c8e5973179f1cfd5b0f1ab789d45b17b6440"
+dependencies = [
+ "protobuf",
+ "protobuf-codegen",
+]
+
+[[package]]
+name = "protocols"
+version = "0.1.0"
+dependencies = [
+ "async-trait",
+ "protobuf",
+ "serde",
+ "serde_json",
+ "ttrpc",
+ "ttrpc-codegen",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1"
+dependencies = [
+ "proc-macro2",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.2.10"
@@ -167,6 +589,20 @@ name = "serde"
 version = "1.0.133"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "97565067517b60e2d1ea8b268e59ce036de907ac523ad83a0475da04e818989a"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.133"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed201699328568d8d08208fdd080e3ff594e6c422e438b6705905da01005d537"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
 
 [[package]]
 name = "serde_json"
@@ -179,6 +615,12 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "slab"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32"
+
 [[package]]
 name = "slog"
 version = "2.7.0"
@@ -220,6 +662,27 @@ dependencies = [
  "slog",
 ]
 
+[[package]]
+name = "socket2"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "syn"
+version = "1.0.91"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b683b2b825c8eef438b77c36a06dc262294da3d5a5813fac20da149241dcd44d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-xid",
+]
+
 [[package]]
 name = "take_mut"
 version = "0.2.2"
@@ -240,6 +703,26 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "thiserror"
+version = "1.0.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "thread_local"
 version = "1.1.3"
@@ -256,16 +739,141 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255"
 dependencies = [
  "libc",
- "wasi",
+ "wasi 0.10.0+wasi-snapshot-preview1",
  "winapi",
 ]
 
+[[package]]
+name = "tokio"
+version = "1.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee"
+dependencies = [
+ "bytes 1.1.0",
+ "libc",
+ "memchr",
+ "mio",
+ "pin-project-lite",
+ "socket2",
+ "tokio-macros",
+ "winapi",
+]
+
+[[package]]
+name = "tokio-macros"
+version = "1.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "tokio-vsock"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e0723fc001950a3b018947b05eeb45014fd2b7c6e8f292502193ab74486bdb6"
+dependencies = [
+ "bytes 0.4.12",
+ "futures",
+ "libc",
+ "tokio",
+ "vsock",
+]
+
+[[package]]
+name = "ttrpc"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "66a973ce6d5eaa20c173635b29ffb660dafbc7ef109172c0015ba44e47a23711"
+dependencies = [
+ "async-trait",
+ "byteorder",
+ "futures",
+ "libc",
+ "log",
+ "nix 0.20.2",
+ "protobuf",
+ "protobuf-codegen-pure",
+ "thiserror",
+ "tokio",
+ "tokio-vsock",
+]
+
+[[package]]
+name = "ttrpc-codegen"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "809eda4e459820237104e4b61d6b41bbe6c9e1ce6adf4057955e6e6722a90408"
+dependencies = [
+ "protobuf",
+ "protobuf-codegen",
+ "protobuf-codegen-pure",
+ "ttrpc-compiler",
+]
+
+[[package]]
+name = "ttrpc-compiler"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2978ed3fa047d8fd55cbeb4d4a61d461fb3021a90c9618519c73ce7e5bb66c15"
+dependencies = [
+ "derive-new",
+ "prost",
+ "prost-build",
+ "prost-types",
+ "protobuf",
+ "protobuf-codegen",
+ "tempfile",
+]
+
+[[package]]
+name = "unicode-segmentation"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99"
+
+[[package]]
+name = "unicode-xid"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
+
+[[package]]
+name = "vsock"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e32675ee2b3ce5df274c0ab52d19b28789632406277ca26bffee79a8e27dc133"
+dependencies = [
+ "libc",
+ "nix 0.23.1",
+]
+
 [[package]]
 name = "wasi"
 version = "0.10.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
 
+[[package]]
+name = "wasi"
+version = "0.11.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+
+[[package]]
+name = "which"
+version = "4.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae"
+dependencies = [
+ "either",
+ "lazy_static",
+ "libc",
+]
+
 [[package]]
 name = "winapi"
 version = "0.3.9"

src/libs/Cargo.toml

@@ -2,5 +2,6 @@
 members = [
     "logging",
     "safe-path",
+    "protocols",
 ]
 resolver = "2"

src/libs/oci/src/lib.rs

@@ -43,7 +43,7 @@ pub struct Spec {
     pub process: Option<Process>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub root: Option<Root>,
-    #[serde(default, skip_serializing_if = "String:: is_empty")]
+    #[serde(default, skip_serializing_if = "String::is_empty")]
     pub hostname: String,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub mounts: Vec<Mount>,

src/libs/protocols/protos/agent.proto

@@ -51,6 +51,8 @@ service AgentService {
 	rpc ListInterfaces(ListInterfacesRequest) returns(Interfaces);
 	rpc ListRoutes(ListRoutesRequest) returns (Routes);
 	rpc AddARPNeighbors(AddARPNeighborsRequest) returns (google.protobuf.Empty);
+	rpc GetIPTables(GetIPTablesRequest) returns (GetIPTablesResponse);
+	rpc SetIPTables(SetIPTablesRequest) returns (SetIPTablesResponse);
 
 	// observability
 	rpc GetMetrics(GetMetricsRequest) returns (Metrics);
@@ -328,6 +330,28 @@ message AddARPNeighborsRequest {
        ARPNeighbors neighbors = 1;
 }
 
+message GetIPTablesRequest {
+       bool is_ipv6 = 1;
+}
+
+message GetIPTablesResponse{
+        // raw stdout from iptables-save or ip6tables-save
+        bytes data = 1;
+}
+
+message SetIPTablesRequest {
+       bool is_ipv6 = 1;
+
+       // iptables, in raw format expected to be passed to stdin
+       // of iptables-save or ip6tables-save
+       bytes data = 2;
+}
+
+message SetIPTablesResponse{
+        // raw stdout from iptables-restore or ip6tables-restore
+        bytes data = 1;
+}
+
 message OnlineCPUMemRequest {
 	// Wait specifies if the caller waits for the agent to online all resources.
 	// If true the agent returns once all resources have been connected, otherwise all

src/libs/safe-path/src/pinned_path_buf.rs

@@ -253,7 +253,7 @@ mod tests {
         fs::write(rootfs_path.join("endpoint"), "test").unwrap();
 
         // Pin the target and validate the path/content.
-        let path = PinnedPathBuf::new(rootfs_path.to_path_buf(), "symlink_dir/endpoint").unwrap();
+        let path = PinnedPathBuf::new(rootfs_path, "symlink_dir/endpoint").unwrap();
         assert!(!path.is_dir());
         let path_ref = path.deref();
         let target = fs::read_link(path_ref).unwrap();
@@ -344,6 +344,7 @@ mod tests {
         PinnedPathBuf::new(rootfs_path, "does_not_exist").unwrap_err();
     }
 
+    #[allow(clippy::zero_prefixed_literal)]
     #[test]
     fn test_new_pinned_path_buf_without_read_perm() {
         let rootfs_dir = tempfile::tempdir().expect("failed to create tmpdir");

src/libs/safe-path/src/scoped_dir_builder.rs

@@ -87,7 +87,7 @@ impl ScopedDirBuilder {
             )
         })?;
 
-        self.do_mkdir(&stripped_path)
+        self.do_mkdir(stripped_path)
     }
 
     /// Creates sub-directory with the options configured in this builder.
@@ -134,7 +134,7 @@ impl ScopedDirBuilder {
                     if !self.recursive && idx != levels {
                         return Err(Error::new(
                             ErrorKind::NotFound,
-                            format!("parent directory does not exist"),
+                            "parent directory does not exist".to_string(),
                         ));
                     }
                     dir = dir.mkdir(comp, self.mode)?;
@@ -146,6 +146,7 @@ impl ScopedDirBuilder {
     }
 }
 
+#[allow(clippy::zero_prefixed_literal)]
 #[cfg(test)]
 mod tests {
     use super::*;

src/runtime/.gitignore

@@ -2,6 +2,7 @@
 *.patch
 *.swp
 coverage.txt
+coverage.txt.tmp
 coverage.html
 .git-commit
 .git-commit.tmp

src/runtime/Makefile

@@ -158,15 +158,22 @@ DEFMEMSZ := 2048
 # - vm template memory
 # - hugepage memory
 DEFMEMSLOTS := 10
+# Default maximum memory in MiB
+DEFMAXMEMSZ := 0
 #Default number of bridges
 DEFBRIDGES := 1
-DEFENABLEANNOTATIONS := []
+DEFENABLEANNOTATIONS := [\"enable_iommu\"]
 DEFDISABLEGUESTSECCOMP := true
 DEFDISABLEGUESTEMPTYDIR := false
 #Default experimental features enabled
 DEFAULTEXPFEATURES := []
 
 DEFDISABLESELINUX := false
+#Default SeccomSandbox param
+#The same default policy is used by libvirt
+#More explanation on https://lists.gnu.org/archive/html/qemu-devel/2017-02/msg03348.html
+# Note: "elevateprivileges=deny" doesn't work with daemonize option, so it's removed from the seccomp sandbox
+DEFSECCOMPSANDBOXPARAM := on,obsolete=deny,spawn=deny,resourcecontrol=deny
 
 #Default entropy source
 DEFENTROPYSOURCE := /dev/urandom
@@ -175,7 +182,10 @@ DEFVALIDENTROPYSOURCES := [\"/dev/urandom\",\"/dev/random\",\"\"]
 DEFDISABLEBLOCK := false
 DEFSHAREDFS_CLH_VIRTIOFS := virtio-fs
 DEFSHAREDFS_QEMU_VIRTIOFS := virtio-fs
+DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/virtiofsd
+ifeq ($(ARCH),ppc64le)
 DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/kata-qemu/virtiofsd
+endif
 DEFVALIDVIRTIOFSDAEMONPATHS := [\"$(DEFVIRTIOFSDAEMON)\"]
 # Default DAX mapping cache size in MiB
 #if value is 0, DAX is not enabled
@@ -434,6 +444,7 @@ USER_VARS += DEFMAXVCPUS
 USER_VARS += DEFMAXVCPUS_ACRN
 USER_VARS += DEFMEMSZ
 USER_VARS += DEFMEMSLOTS
+USER_VARS += DEFMAXMEMSZ
 USER_VARS += DEFBRIDGES
 USER_VARS += DEFNETWORKMODEL_ACRN
 USER_VARS += DEFNETWORKMODEL_CLH
@@ -456,6 +467,7 @@ USER_VARS += DEFVIRTIOFSCACHE
 USER_VARS += DEFVIRTIOFSEXTRAARGS
 USER_VARS += DEFENABLEANNOTATIONS
 USER_VARS += DEFENABLEIOTHREADS
+USER_VARS += DEFSECCOMPSANDBOXPARAM
 USER_VARS += DEFENABLEVHOSTUSERSTORE
 USER_VARS += DEFVHOSTUSERSTOREPATH
 USER_VARS += DEFVALIDVHOSTUSERSTOREPATHS
@@ -596,7 +608,7 @@ hook:
 
 go-test: $(GENERATED_FILES)
 	go clean -testcache
-	$(QUIET_TEST)../../ci/go-test.sh
+	$(QUIET_TEST)./go-test.sh
 
 fast-test: $(GENERATED_FILES)
 	go clean -testcache

src/runtime/README.md

@@ -87,6 +87,27 @@ following locations (in order):
 
 > **Note:** For both binaries, the first path that exists will be used.
 
+#### Drop-in configuration file fragments
+
+To enable changing configuration without changing the configuration file
+itself, drop-in configuration file fragments are supported.  Once a
+configuration file is parsed, if there is a subdirectory called `config.d` in
+the same directory as the configuration file its contents will be loaded
+in alphabetical order and each item will be parsed as a config file.  Settings
+loaded from these configuration file fragments override settings loaded from
+the main configuration file and earlier fragments.  Users are encouraged to use
+familiar naming conventions to order the fragments (e.g. `config.d/10-this`,
+`config.d/20-that` etc.).
+
+Non-existent or empty `config.d` directory is not an error (in other words, not
+using configuration file fragments is fine).  On the other hand, if fragments
+are used, they must be valid - any errors while parsing fragments (unreadable
+fragment files, contents not valid TOML) are treated the same as errors
+while parsing the main configuration file.  A `config.d` subdirectory affects
+only the `configuration.toml` _in the same directory_.  For fragments in
+`config.d` to be parsed, there has to be a valid main configuration file _in
+that location_ (it can be empty though).
+
 ### Hypervisor specific configuration
 
 Kata Containers supports multiple hypervisors so your `configuration.toml`
@@ -125,7 +146,7 @@ $ kata-runtime env
 
 For detailed information and analysis on obtaining logs for other system
 components, see the documentation for the
-[`kata-log-parser`](https://github.com/kata-containers/tests/tree/main/cmd/log-parser)
+[`kata-log-parser`](../tools/log-parser)
 tool.
 
 ### Kata containerd shimv2

src/runtime/cmd/kata-monitor/README.md

@@ -15,6 +15,14 @@ Available metrics include:
 All the provided metrics are in Prometheus format. While `kata-monitor` can be used as a standalone daemon on any host running Kata Containers workloads and can be used for retrieving profiling data from the running Kata runtimes, its main expected usage is to be deployed as a DaemonSet on a Kubernetes cluster: there Prometheus should scrape the metrics from the kata-monitor endpoints.
 For more information on the Kata Containers metrics architecture and a detailed list of the available metrics provided by Kata monitor check the [Kata 2.0 Metrics Design](../../../../docs/design/kata-2-0-metrics.md) document.
 
+## Local network considerations
+
+The `kata-monitor` daemon is not run unless explicitly
+[started](#kata-monitor-arguments). However, when it is running it
+will accept connections on the `localhost` network interface (by
+default) and provide metrics to any client process that connects to
+it, whether they are privileged or not.
+
 ## Usage
 Each `kata-monitor` instance detects and monitors the Kata Container workloads running on the same node.
 

src/runtime/cmd/kata-runtime/factory_test.go

@@ -44,7 +44,7 @@ func TestFactoryCLIFunctionInit(t *testing.T) {
 
 	tmpdir := t.TempDir()
 
-	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, true)
 	assert.NoError(err)
 
 	set := flag.NewFlagSet("", 0)
@@ -91,7 +91,7 @@ func TestFactoryCLIFunctionDestroy(t *testing.T) {
 
 	tmpdir := t.TempDir()
 
-	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, true)
 	assert.NoError(err)
 
 	set := flag.NewFlagSet("", 0)
@@ -123,7 +123,7 @@ func TestFactoryCLIFunctionStatus(t *testing.T) {
 
 	tmpdir := t.TempDir()
 
-	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, true)
 	assert.NoError(err)
 
 	set := flag.NewFlagSet("", 0)

src/runtime/cmd/kata-runtime/kata-iptables.go

@@ -0,0 +1,122 @@
+// Copyright (c) 2022 Apple Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+
+	containerdshim "github.com/kata-containers/kata-containers/src/runtime/pkg/containerd-shim-v2"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils/shimclient"
+	"github.com/urfave/cli"
+)
+
+var (
+	sandboxID string
+	isIPv6    bool
+)
+var iptablesSubCmds = []cli.Command{
+	getIPTablesCommand,
+	setIPTablesCommand,
+}
+
+var kataIPTablesCommand = cli.Command{
+	Name:        "iptables",
+	Usage:       "get or set iptables within the Kata Containers guest",
+	Subcommands: iptablesSubCmds,
+	Action: func(context *cli.Context) {
+		cli.ShowSubcommandHelp(context)
+	},
+}
+
+var getIPTablesCommand = cli.Command{
+	Name:  "get",
+	Usage: "get iptables from the Kata Containers guest",
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:        "sandbox-id",
+			Usage:       "the target sandbox for getting the iptables",
+			Required:    true,
+			Destination: &sandboxID,
+		},
+		cli.BoolFlag{
+			Name:        "v6",
+			Usage:       "indicate we're requesting ipv6 iptables",
+			Destination: &isIPv6,
+		},
+	},
+	Action: func(c *cli.Context) error {
+		// verify sandbox exists:
+		if err := katautils.VerifyContainerID(sandboxID); err != nil {
+			return err
+		}
+
+		url := containerdshim.IPTablesUrl
+		if isIPv6 {
+			url = containerdshim.IP6TablesUrl
+		}
+		body, err := shimclient.DoGet(sandboxID, defaultTimeout, url)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(string(body))
+		return nil
+	},
+}
+
+var setIPTablesCommand = cli.Command{
+	Name:  "set",
+	Usage: "set iptables in a specifc Kata Containers guest based on file",
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:        "sandbox-id",
+			Usage:       "the target sandbox for setting the iptables",
+			Required:    true,
+			Destination: &sandboxID,
+		},
+		cli.BoolFlag{
+			Name:        "v6",
+			Usage:       "indicate we're requesting ipv6 iptables",
+			Destination: &isIPv6,
+		},
+	},
+	Action: func(c *cli.Context) error {
+		iptablesFile := c.Args().Get(0)
+
+		// verify sandbox exists:
+		if err := katautils.VerifyContainerID(sandboxID); err != nil {
+			return err
+		}
+
+		// verify iptables were provided:
+		if iptablesFile == "" {
+			return fmt.Errorf("iptables file not provided")
+		}
+
+		if !katautils.FileExists(iptablesFile) {
+			return fmt.Errorf("iptables file does not exist: %s", iptablesFile)
+		}
+
+		// Read file into buffer, and make request to the appropriate shim
+		buf, err := ioutil.ReadFile(iptablesFile)
+		if err != nil {
+			return err
+		}
+
+		url := containerdshim.IPTablesUrl
+		if isIPv6 {
+			url = containerdshim.IP6TablesUrl
+		}
+
+		if err = shimclient.DoPut(sandboxID, defaultTimeout, url, "application/octet-stream", buf); err != nil {
+			return fmt.Errorf("Error observed when making iptables-set request(%s): %s", iptablesFile, err)
+		}
+
+		return nil
+	},
+}

src/runtime/cmd/kata-runtime/kata-volume.go

@@ -7,10 +7,11 @@ package main
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/url"
 
 	containerdshim "github.com/kata-containers/kata-containers/src/runtime/pkg/containerd-shim-v2"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume"
+	volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils/shimclient"
 
 	"github.com/urfave/cli"
@@ -89,12 +90,14 @@ var statsCommand = cli.Command{
 			Destination: &volumePath,
 		},
 	},
-	Action: func(c *cli.Context) (string, error) {
+	Action: func(c *cli.Context) error {
 		stats, err := Stats(volumePath)
 		if err != nil {
-			return "", cli.NewExitError(err.Error(), 1)
+			return cli.NewExitError(err.Error(), 1)
 		}
-		return string(stats), nil
+
+		fmt.Println(string(stats))
+		return nil
 	},
 }
 
@@ -127,8 +130,14 @@ func Stats(volumePath string) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	urlSafeDevicePath := url.PathEscape(volumePath)
-	body, err := shimclient.DoGet(sandboxId, defaultTimeout, containerdshim.DirectVolumeStatUrl+"/"+urlSafeDevicePath)
+	volumeMountInfo, err := volume.VolumeMountInfo(volumePath)
+	if err != nil {
+		return nil, err
+	}
+
+	urlSafeDevicePath := url.PathEscape(volumeMountInfo.Device)
+	body, err := shimclient.DoGet(sandboxId, defaultTimeout,
+		fmt.Sprintf("%s?%s=%s", containerdshim.DirectVolumeStatUrl, containerdshim.DirectVolumePathKey, urlSafeDevicePath))
 	if err != nil {
 		return nil, err
 	}
@@ -141,13 +150,18 @@ func Resize(volumePath string, size uint64) error {
 	if err != nil {
 		return err
 	}
+	volumeMountInfo, err := volume.VolumeMountInfo(volumePath)
+	if err != nil {
+		return err
+	}
+
 	resizeReq := containerdshim.ResizeRequest{
-		VolumePath: volumePath,
+		VolumePath: volumeMountInfo.Device,
 		Size:       size,
 	}
 	encoded, err := json.Marshal(resizeReq)
 	if err != nil {
 		return err
 	}
-	return shimclient.DoPost(sandboxId, defaultTimeout, containerdshim.DirectVolumeResizeUrl, encoded)
+	return shimclient.DoPost(sandboxId, defaultTimeout, containerdshim.DirectVolumeResizeUrl, "application/json", encoded)
 }

src/runtime/cmd/kata-runtime/main.go

@@ -125,6 +125,7 @@ var runtimeCommands = []cli.Command{
 	kataMetricsCLICommand,
 	factoryCLICommand,
 	kataVolumeCommand,
+	kataIPTablesCommand,
 }
 
 // runtimeBeforeSubcommands is the function to run before command-line

src/runtime/cmd/kata-runtime/main_test.go

@@ -33,8 +33,6 @@ const (
 	testDirMode     = os.FileMode(0750)
 	testFileMode    = os.FileMode(0640)
 	testExeFileMode = os.FileMode(0750)
-
-	testConsole = "/dev/pts/999"
 )
 
 var (
@@ -151,7 +149,7 @@ func newTestHypervisorConfig(dir string, create bool) (vc.HypervisorConfig, erro
 }
 
 // newTestRuntimeConfig creates a new RuntimeConfig
-func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConfig, error) {
+func newTestRuntimeConfig(dir string, create bool) (oci.RuntimeConfig, error) {
 	if dir == "" {
 		return oci.RuntimeConfig{}, errors.New("BUG: need directory")
 	}
@@ -164,7 +162,6 @@ func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConf
 	return oci.RuntimeConfig{
 		HypervisorType:   vc.QemuHypervisor,
 		HypervisorConfig: hypervisorConfig,
-		Console:          consolePath,
 	}, nil
 }
 

src/runtime/config/configuration-clh.toml.in

@@ -105,6 +105,12 @@ default_memory = @DEFMEMSZ@
 # This is will determine the times that memory will be hotadded to sandbox/VM.
 #memory_slots = @DEFMEMSLOTS@
 
+# Default maximum memory in MiB per SB / VM
+# unspecified or == 0           --> will be set to the actual amount of physical RAM
+# > 0 <= amount of physical RAM --> will be set to the specified number
+# > amount of physical RAM      --> will be set to the actual amount of physical RAM
+default_maxmemory = @DEFMAXMEMSZ@
+
 # Shared file system type:
 #   - virtio-fs (default)
 #   - virtio-fs-nydus

src/runtime/config/configuration-fc.toml.in

@@ -91,6 +91,7 @@ default_bridges = @DEFBRIDGES@
 # Default memory size in MiB for SB/VM.
 # If unspecified then it will be set @DEFMEMSZ@ MiB.
 default_memory = @DEFMEMSZ@
+
 #
 # Default memory slots per SB/VM.
 # If unspecified then it will be set @DEFMEMSLOTS@.
@@ -104,6 +105,12 @@ default_memory = @DEFMEMSZ@
 # Default 0
 #memory_offset = 0
 
+# Default maximum memory in MiB per SB / VM
+# unspecified or == 0           --> will be set to the actual amount of physical RAM
+# > 0 <= amount of physical RAM --> will be set to the specified number
+# > amount of physical RAM      --> will be set to the actual amount of physical RAM
+default_maxmemory = @DEFMAXMEMSZ@
+
 # Block storage driver to be used for the hypervisor in case the container
 # rootfs is backed by a block device. This is virtio-scsi, virtio-blk
 # or nvdimm.

src/runtime/config/configuration-qemu.toml.in

@@ -76,6 +76,14 @@ firmware_volume = "@FIRMWAREVOLUMEPATH@"
 # For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"`
 machine_accelerators="@MACHINEACCELERATORS@"
 
+# Qemu seccomp sandbox feature
+# comma-separated list of seccomp sandbox features to control the syscall access.
+# For example, `seccompsandbox= "on,obsolete=deny,spawn=deny,resourcecontrol=deny"`
+# Note: "elevateprivileges=deny" doesn't work with daemonize option, so it's removed from the seccomp sandbox
+# Another note: enabling this feature may reduce performance, you may enable
+# /proc/sys/net/core/bpf_jit_enable to reduce the impact. see https://man7.org/linux/man-pages/man8/bpfc.8.html
+#seccompsandbox="@DEFSECCOMPSANDBOXPARAM@"
+
 # CPU features
 # comma-separated list of cpu features to pass to the cpu
 # For example, `cpu_features = "pmu=off,vmx=off"
@@ -126,6 +134,12 @@ default_memory = @DEFMEMSZ@
 # This is will determine the times that memory will be hotadded to sandbox/VM.
 #memory_slots = @DEFMEMSLOTS@
 
+# Default maximum memory in MiB per SB / VM
+# unspecified or == 0           --> will be set to the actual amount of physical RAM
+# > 0 <= amount of physical RAM --> will be set to the specified number
+# > amount of physical RAM      --> will be set to the actual amount of physical RAM
+default_maxmemory = @DEFMAXMEMSZ@
+
 # The size in MiB will be plused to max memory of hypervisor.
 # It is the memory address space for the NVDIMM devie.
 # If set block storage driver (block_device_driver) to "nvdimm",
@@ -389,6 +403,9 @@ valid_entropy_sources = @DEFVALIDENTROPYSOURCES@
 # be default_memory.
 #enable_guest_swap = true
 
+# use legacy serial for guest console if available and implemented for architecture. Default false
+#use_legacy_serial = true
+
 [factory]
 # VM templating support. Once enabled, new VMs are created from template
 # using vm cloning. They will share the same initial kernel, initramfs and

src/runtime/go-test.sh

@@ -0,0 +1,167 @@
+#!/bin/bash
+#
+# Copyright (c) 2017-2018 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -e
+
+script_name=${0##*/}
+typeset -A long_options
+
+long_options=(
+	[help]="Show usage"
+	[package:]="Specify test package to run"
+)
+
+# Set up go test flags
+go_test_flags="${KATA_GO_TEST_FLAGS}"
+if [ -z "$go_test_flags" ]; then
+    # KATA_GO_TEST_TIMEOUT can be set to any value accepted by
+    # "go test -timeout X"
+    go_test_flags="-timeout ${KATA_GO_TEST_TIMEOUT:-30s}"
+
+    # -race flag is not supported on s390x
+    [ "$(go env GOARCH)" != "s390x" ] && go_test_flags+=" -race"
+
+    # s390x requires special linker flags
+    [ "$(go env GOARCH)" = s390x ] && go_test_flags+=" -ldflags '-extldflags -Wl,--s390-pgste'"
+fi
+
+# The "master" coverage file that contains the coverage results for
+# all packages run under all scenarios.
+test_coverage_file="coverage.txt"
+
+# Temporary coverage file created for a "go test" run. The results in
+# this file will be added to the master coverage file.
+tmp_coverage_file="${test_coverage_file}.tmp"
+
+warn()
+{
+	local msg="$*"
+	echo >&2 "WARNING: $msg"
+}
+
+usage()
+{
+	cat <<EOF
+
+Usage: $script_name [options]
+
+Options:
+
+EOF
+
+	local option
+	local description
+
+	local long_option_names="${!long_options[@]}"
+
+	# Sort space-separated list by converting to newline separated list
+	# and back again.
+	long_option_names=$(echo "$long_option_names"|tr ' ' '\n'|sort|tr '\n' ' ')
+
+	# Display long options
+	for option in ${long_option_names}
+	do
+		description=${long_options[$option]}
+
+		# Remove any trailing colon which is for getopt(1) alone.
+		option=$(echo "$option"|sed 's/:$//g')
+
+		printf "    --%-10.10s # %s\n" "$option" "$description"
+	done
+}
+
+# Run a command as either root or the current user (which might still be root).
+#
+# If the first argument is "root", run using sudo, else run as the current
+# user. All arguments after the first will be treated as the command to run.
+run_as_user()
+{
+	local user="$1"
+
+	shift
+
+	local cmd=$*
+
+	if [ "$user" = root ]; then
+		# use a shell to ensure PATH is correct.
+		sudo -E PATH="$PATH" sh -c "$cmd"
+	else
+		eval "$cmd"
+	fi
+}
+
+# Test a single golang package
+test_go_package()
+{
+	local -r pkg="$1"
+	local -r user="$2"
+
+	printf "INFO: Running 'go test' as %s user on package '%s' with flags '%s'\n" \
+		"$user" "$pkg" "$go_test_flags"
+
+	run_as_user "$user" go test "$go_test_flags" -covermode=atomic -coverprofile=$tmp_coverage_file "$pkg"
+
+	# Merge test results into the master coverage file.
+	run_as_user "$user" tail -n +2 "$tmp_coverage_file" >> "$test_coverage_file"
+	rm -f "$tmp_coverage_file"
+}
+
+# Run all tests and generate a test coverage file.
+test_coverage()
+{
+	echo "mode: atomic" > "$test_coverage_file"
+
+	users="current"
+
+	if [ "$(id -u)" -eq 0 ]; then
+		warn "Already running as root so will not re-run tests as non-root user."
+		warn "As a result, only a subset of tests will be run"
+		warn "(run this script as a non-privileged to ensure all tests are run)."
+	else
+		# Run the unit-tests *twice* (since some must run as
+		# root and others must run as non-root), combining the
+		# resulting test coverage files.
+		users+=" root"
+	fi
+
+	echo "INFO: Currently running as user '$(id -un)'"
+	for user in $users; do
+	    test_go_package "$package" "$user"
+	done
+}
+
+main()
+{
+	local long_option_names="${!long_options[@]}"
+
+	local args=$(getopt \
+		-n "$script_name" \
+		-a \
+		--options="h" \
+		--longoptions="$long_option_names" \
+		-- "$@")
+
+	package="./..."
+
+	eval set -- "$args"
+	[ $? -ne 0 ] && { usage >&2; exit 1; }
+
+	while [ $# -gt 1 ]
+	do
+		case "$1" in
+			-h|--help) usage; exit 0 ;;
+			--package) package="$2"; shift 2;;
+			--) shift; break ;;
+		esac
+
+		shift
+	done
+
+	test_coverage
+}
+
+main "$@"

src/runtime/go.mod

@@ -9,12 +9,12 @@ require (
 	github.com/blang/semver/v4 v4.0.0
 	github.com/containerd/cgroups v1.0.3
 	github.com/containerd/console v1.0.3
-	github.com/containerd/containerd v1.6.1
+	github.com/containerd/containerd v1.6.6
 	github.com/containerd/cri-containerd v1.11.1-0.20190125013620-4dd6735020f5
 	github.com/containerd/fifo v1.0.0
 	github.com/containerd/ttrpc v1.1.0
 	github.com/containerd/typeurl v1.0.2
-	github.com/containernetworking/plugins v1.0.1
+	github.com/containernetworking/plugins v1.1.1
 	github.com/coreos/go-systemd/v22 v22.3.2
 	github.com/cri-o/cri-o v1.0.0-rc2.0.20170928185954-3394b3b2d6af
 	github.com/docker/go-units v0.4.0
@@ -25,17 +25,17 @@ require (
 	github.com/go-openapi/strfmt v0.18.0
 	github.com/go-openapi/swag v0.19.14
 	github.com/go-openapi/validate v0.18.0
-	github.com/godbus/dbus/v5 v5.0.4
+	github.com/godbus/dbus/v5 v5.0.6
 	github.com/gogo/protobuf v1.3.2
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/intel-go/cpuid v0.0.0-20210602155658-5747e5cec0d9
 	github.com/mdlayher/vsock v1.1.0
-	github.com/opencontainers/image-spec v1.0.2 // indirect
-	github.com/opencontainers/runc v1.1.0
+	github.com/opencontainers/runc v1.1.2
 	github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417
-	github.com/opencontainers/selinux v1.10.0
+	github.com/opencontainers/selinux v1.10.1
+	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.11.0
+	github.com/prometheus/client_golang v1.11.1
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/common v0.30.0
 	github.com/prometheus/procfs v0.7.3
@@ -45,13 +45,14 @@ require (
 	github.com/urfave/cli v1.22.2
 	github.com/vishvananda/netlink v1.1.1-0.20210924202909-187053b97868
 	github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f
+	gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20220601114329-47893b162965
 	go.opentelemetry.io/otel v1.3.0
 	go.opentelemetry.io/otel/exporters/jaeger v1.0.0
 	go.opentelemetry.io/otel/sdk v1.3.0
 	go.opentelemetry.io/otel/trace v1.3.0
 	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd
 	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f
-	golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a
+	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad
 	google.golang.org/grpc v1.43.0
 	k8s.io/apimachinery v0.22.5
 	k8s.io/cri-api v0.23.1

src/runtime/go.sum

@@ -82,8 +82,9 @@ github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+V
 github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4=
 github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4=
 github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg=
-github.com/Microsoft/hcsshim v0.9.2 h1:wB06W5aYFfUB3IvootYAY2WnOmIdgPGfqSI6tufQNnY=
 github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc=
+github.com/Microsoft/hcsshim v0.9.3 h1:k371PzBuRrz2b+ebGuI2nVgVhgsVX60jMfSw80NECxo=
+github.com/Microsoft/hcsshim v0.9.3/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc=
 github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU=
 github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
@@ -154,8 +155,9 @@ github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmE
 github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc=
 github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs=
 github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
-github.com/cilium/ebpf v0.6.2 h1:iHsfF/t4aW4heW2YKfeHrVPGdtYTL4C4KocpM8KTSnI=
 github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
+github.com/cilium/ebpf v0.7.0 h1:1k/q3ATgxSXRdrmPfH8d7YK0GfqVsEKZAX9dQZvs56k=
+github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
@@ -208,8 +210,9 @@ github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoT
 github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g=
 github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c=
 github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s=
-github.com/containerd/containerd v1.6.1 h1:oa2uY0/0G+JX4X7hpGCYvkp9FjUancz56kSNnb1sG3o=
 github.com/containerd/containerd v1.6.1/go.mod h1:1nJz5xCZPusx6jJU8Frfct988y0NpumIq9ODB0kLtoE=
+github.com/containerd/containerd v1.6.6 h1:xJNPhbrmz8xAMDNoVjHy9YHtWwEQNS+CDkcIRh7t8Y0=
+github.com/containerd/containerd v1.6.6/go.mod h1:ZoP1geJldzCVY3Tonoz7b1IXk8rIX0Nltt5QE4OMNk0=
 github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
@@ -232,6 +235,7 @@ github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZH
 github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk=
 github.com/containerd/go-cni v1.1.0/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA=
 github.com/containerd/go-cni v1.1.3/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA=
+github.com/containerd/go-cni v1.1.6/go.mod h1:BWtoWl5ghVymxu6MBjg79W9NZrCRyHIdUtk4cauMe34=
 github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
 github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
 github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g=
@@ -243,6 +247,7 @@ github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6T
 github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow=
 github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms=
 github.com/containerd/imgcrypt v1.1.3/go.mod h1:/TPA1GIDXMzbj01yd8pIbQiLdQxed5ue1wb8bP7PQu4=
+github.com/containerd/imgcrypt v1.1.4/go.mod h1:LorQnPtzL/T0IyCeftcsMEO7AqxUDbdO8j/tSUpgxvo=
 github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c=
 github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
 github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
@@ -267,16 +272,19 @@ github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNR
 github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
 github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
 github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
-github.com/containernetworking/cni v1.0.1 h1:9OIL/sZmMYDBe+G8svzILAlulUpaDTUjeAbtH/JNLBo=
 github.com/containernetworking/cni v1.0.1/go.mod h1:AKuhXbN5EzmD4yTNtfSsX3tPcmtrBI6QcRV0NiNt15Y=
+github.com/containernetworking/cni v1.1.1 h1:ky20T7c0MvKvbMOwS/FrlbNwjEoqJEUUYfsL4b0mc4k=
+github.com/containernetworking/cni v1.1.1/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw=
 github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM=
 github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8=
-github.com/containernetworking/plugins v1.0.1 h1:wwCfYbTCj5FC0EJgyzyjTXmqysOiJE9r712Z+2KVZAk=
 github.com/containernetworking/plugins v1.0.1/go.mod h1:QHCfGpaTwYTbbH+nZXKVTxNBDZcxSOplJT5ico8/FLE=
+github.com/containernetworking/plugins v1.1.1 h1:+AGfFigZ5TiQH00vhR8qPeSatj53eNGz0C1d3wVYlHE=
+github.com/containernetworking/plugins v1.1.1/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19sZPp3ry5uHSkI4LPxV8=
 github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc=
 github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4=
 github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY=
 github.com/containers/ocicrypt v1.1.2/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY=
+github.com/containers/ocicrypt v1.1.3/go.mod h1:xpdkbVAuaH3WzbEabUd5yDsl9SwJA5pABH85425Es2g=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -445,8 +453,9 @@ github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblf
 github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e h1:BWhy2j3IXJhjCbC68FptL43tDKIq8FladmaTs3Xs7Z8=
 github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/godbus/dbus/v5 v5.0.6 h1:mkgN1ofwASrYnJ5W6U/BxG15eXXXjirgZc7CLqkcaro=
+github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU=
 github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -527,6 +536,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -661,6 +671,7 @@ github.com/mdlayher/vsock v1.1.0 h1:2k9udP/hUkLUOboGxXMHOk4f0GWWZwS3IuE3Ee/YYfk=
 github.com/mdlayher/vsock v1.1.0/go.mod h1:nsVhPsVuBBwAKh6i6PzdNoke6/TNYTjkxoRKAp/+pXs=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
+github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -696,6 +707,7 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
+github.com/networkplumbing/go-nft v0.2.0/go.mod h1:HnnM+tYvlGAsMU7yoYwXEVLLiDW9gdMmb5HoGcwpuQs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
@@ -715,6 +727,8 @@ github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1ls
 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
 github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
+github.com/onsi/ginkgo/v2 v2.1.3 h1:e/3Cwtogj0HA+25nMP1jCMDIf8RtRYbGwGGuBIFztkc=
+github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
 github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
@@ -724,8 +738,9 @@ github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoT
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
 github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
-github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c=
 github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
+github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
+github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
@@ -747,10 +762,13 @@ github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mo
 github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE=
 github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo=
 github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=
-github.com/opencontainers/selinux v1.10.0 h1:rAiKF8hTcgLI3w0DHm6i0ylVVcOrlgR1kK99DRLDhyU=
 github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
+github.com/opencontainers/selinux v1.10.1 h1:09LIPVRP3uuZGQvgR+SgMSNBd1Eb3vlRbGqQpoHsF8w=
+github.com/opencontainers/selinux v1.10.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
+github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
 github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
@@ -772,8 +790,9 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
-github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s=
+github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -908,6 +927,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
 github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
 github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA=
 github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
+gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20220601114329-47893b162965 h1:EXE1ZsUqiUWGV5Dw2oTYpXx24ffxj0//yhTB0Ppv+4s=
+gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20220601114329-47893b162965/go.mod h1:TBB3sR7/jg4RCThC/cgT4fB8mAbbMO307TycfgeR59w=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
@@ -1192,11 +1213,13 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a h1:ppl5mZgokTT8uPkmYOyEUmPTr3ypaKkg5eFOGrAmxxE=
 golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0=
+golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1281,6 +1304,7 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

src/runtime/pkg/containerd-shim-v2/create.go

@@ -97,7 +97,7 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con
 		}
 
 		// create root span
-		rootSpan, newCtx := katatrace.Trace(s.ctx, shimLog, "root span", shimTracingTags)
+		rootSpan, newCtx := katatrace.Trace(s.ctx, shimLog, "rootSpan", shimTracingTags)
 		s.rootCtx = newCtx
 		defer rootSpan.End()
 
@@ -144,7 +144,7 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con
 		// ctx will be canceled after this rpc service call, but the sandbox will live
 		// across multiple rpc service calls.
 		//
-		sandbox, _, err := katautils.CreateSandbox(s.ctx, vci, *ociSpec, *s.config, rootFs, r.ID, bundlePath, "", disableOutput, false)
+		sandbox, _, err := katautils.CreateSandbox(s.ctx, vci, *ociSpec, *s.config, rootFs, r.ID, bundlePath, disableOutput, false)
 		if err != nil {
 			return nil, err
 		}
@@ -179,7 +179,7 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con
 			}
 		}()
 
-		_, err = katautils.CreateContainer(ctx, s.sandbox, *ociSpec, rootFs, r.ID, bundlePath, "", disableOutput, runtimeConfig.DisableGuestEmptyDir)
+		_, err = katautils.CreateContainer(ctx, s.sandbox, *ociSpec, rootFs, r.ID, bundlePath, disableOutput, runtimeConfig.DisableGuestEmptyDir)
 		if err != nil {
 			return nil, err
 		}

src/runtime/pkg/containerd-shim-v2/create_test.go

@@ -51,7 +51,7 @@ func TestCreateSandboxSuccess(t *testing.T) {
 
 	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 
-	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, true)
 	assert.NoError(err)
 
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
@@ -99,7 +99,7 @@ func TestCreateSandboxFail(t *testing.T) {
 
 	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 
-	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, true)
 	assert.NoError(err)
 
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
@@ -136,7 +136,7 @@ func TestCreateSandboxConfigFail(t *testing.T) {
 
 	tmpdir, bundlePath, _ := ktu.SetupOCIConfigFile(t)
 
-	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, true)
 	assert.NoError(err)
 
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
@@ -185,7 +185,7 @@ func TestCreateContainerSuccess(t *testing.T) {
 
 	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 
-	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, true)
 	assert.NoError(err)
 
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
@@ -224,7 +224,7 @@ func TestCreateContainerFail(t *testing.T) {
 
 	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 
-	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, true)
 	assert.NoError(err)
 
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
@@ -274,7 +274,7 @@ func TestCreateContainerConfigFail(t *testing.T) {
 
 	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 
-	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, true)
 	assert.NoError(err)
 
 	spec, err := compatoci.ParseConfigJSON(bundlePath)

src/runtime/pkg/containerd-shim-v2/service.go

@@ -7,9 +7,11 @@ package containerdshim
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"os"
 	sysexec "os/exec"
+	goruntime "runtime"
 	"sync"
 	"syscall"
 	"time"
@@ -31,6 +33,7 @@ import (
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
@@ -83,6 +86,11 @@ func New(ctx context.Context, id string, publisher cdshim.Publisher, shutdown fu
 	vci.SetLogger(ctx, shimLog)
 	katautils.SetLogger(ctx, shimLog, shimLog.Logger.Level)
 
+	ns, found := namespaces.Namespace(ctx)
+	if !found {
+		return nil, fmt.Errorf("shim namespace cannot be empty")
+	}
+
 	s := &service{
 		id:         id,
 		pid:        uint32(os.Getpid()),
@@ -91,6 +99,7 @@ func New(ctx context.Context, id string, publisher cdshim.Publisher, shutdown fu
 		events:     make(chan interface{}, chSize),
 		ec:         make(chan exit, bufferSize),
 		cancel:     shutdown,
+		namespace:  ns,
 	}
 
 	go s.processExits()
@@ -129,6 +138,9 @@ type service struct {
 
 	id string
 
+	// Namespace from upper container engine
+	namespace string
+
 	mu          sync.Mutex
 	eventSendMu sync.Mutex
 
@@ -234,9 +246,19 @@ func (s *service) StartShim(ctx context.Context, opts cdshim.StartOpts) (_ strin
 
 	cmd.ExtraFiles = append(cmd.ExtraFiles, f)
 
+	goruntime.LockOSThread()
+	if os.Getenv("SCHED_CORE") != "" {
+		if err := utils.Create(utils.ProcessGroup); err != nil {
+			return "", errors.Wrap(err, "enable sched core support")
+		}
+	}
+
 	if err := cmd.Start(); err != nil {
 		return "", err
 	}
+
+	goruntime.UnlockOSThread()
+
 	defer func() {
 		if retErr != nil {
 			cmd.Process.Kill()

src/runtime/pkg/containerd-shim-v2/shim_io_binary.go

@@ -0,0 +1,216 @@
+// Copyright (c) 2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"syscall"
+	"time"
+
+	"golang.org/x/sys/execabs"
+
+	"github.com/hashicorp/go-multierror"
+)
+
+const (
+	binaryIOProcTermTimeout = 12 * time.Second // Give logger process solid 10 seconds for cleanup
+)
+
+var (
+	_ IO = &binaryIO{}
+)
+
+// binaryIO related code is from https://github.com/containerd/containerd/blob/v1.6.6/pkg/process/io.go#L311
+type binaryIO struct {
+	cmd      *execabs.Cmd
+	out, err *pipe
+}
+
+// https://github.com/containerd/containerd/blob/v1.6.6/pkg/process/io.go#L248
+func newBinaryIO(ctx context.Context, ns, id string, uri *url.URL) (bio *binaryIO, err error) {
+	var closers []func() error
+	defer func() {
+		if err == nil {
+			return
+		}
+		result := multierror.Append(err)
+		for _, fn := range closers {
+			result = multierror.Append(result, fn())
+		}
+		err = multierror.Flatten(result)
+	}()
+
+	out, err := newPipe()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create stdout pipes: %w", err)
+	}
+	closers = append(closers, out.Close)
+
+	serr, err := newPipe()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create stderr pipes: %w", err)
+	}
+	closers = append(closers, serr.Close)
+
+	r, w, err := os.Pipe()
+	if err != nil {
+		return nil, err
+	}
+	closers = append(closers, r.Close, w.Close)
+
+	cmd := newBinaryCmd(uri, id, ns)
+	cmd.ExtraFiles = append(cmd.ExtraFiles, out.r, serr.r, w)
+	// don't need to register this with the reaper or wait when
+	// running inside a shim
+	if err := cmd.Start(); err != nil {
+		return nil, fmt.Errorf("failed to start binary process: %w", err)
+	}
+	closers = append(closers, func() error { return cmd.Process.Kill() })
+
+	// close our side of the pipe after start
+	if err := w.Close(); err != nil {
+		return nil, fmt.Errorf("failed to close write pipe after start: %w", err)
+	}
+
+	// wait for the logging binary to be ready
+	b := make([]byte, 1)
+	if _, err := r.Read(b); err != nil && err != io.EOF {
+		return nil, fmt.Errorf("failed to read from logging binary: %w", err)
+	}
+
+	return &binaryIO{
+		cmd: cmd,
+		out: out,
+		err: serr,
+	}, nil
+}
+
+// newBinaryCmd returns a Cmd to be used to start a logging binary.
+// The Cmd is generated from the provided uri, and the container ID and
+// namespace are appended to the Cmd environment.
+func newBinaryCmd(binaryURI *url.URL, id, ns string) *execabs.Cmd {
+	var args []string
+	for k, vs := range binaryURI.Query() {
+		args = append(args, k)
+		if len(vs) > 0 {
+			args = append(args, vs[0])
+		}
+	}
+
+	cmd := execabs.Command(binaryURI.Path, args...)
+
+	cmd.Env = append(cmd.Env,
+		"CONTAINER_ID="+id,
+		"CONTAINER_NAMESPACE="+ns,
+	)
+
+	return cmd
+}
+
+func (bi *binaryIO) Stdin() io.ReadCloser {
+	return nil
+}
+
+func (bi *binaryIO) Stdout() io.Writer {
+	return bi.out.w
+}
+
+func (bi *binaryIO) Stderr() io.Writer {
+	return bi.err.w
+}
+
+func (bi *binaryIO) Close() error {
+	var (
+		result *multierror.Error
+	)
+
+	for _, v := range []*pipe{bi.out, bi.err} {
+		if v != nil {
+			if err := v.Close(); err != nil {
+				result = multierror.Append(result, err)
+			}
+		}
+	}
+
+	if err := bi.cancel(); err != nil {
+		result = multierror.Append(result, err)
+	}
+
+	return result.ErrorOrNil()
+}
+
+func (bi *binaryIO) cancel() error {
+	if bi.cmd == nil || bi.cmd.Process == nil {
+		return nil
+	}
+
+	// Send SIGTERM first, so logger process has a chance to flush and exit properly
+	if err := bi.cmd.Process.Signal(syscall.SIGTERM); err != nil {
+		result := multierror.Append(fmt.Errorf("failed to send SIGTERM: %w", err))
+
+		shimLog.WithError(err).Warn("failed to send SIGTERM signal, killing logging shim")
+
+		if err := bi.cmd.Process.Kill(); err != nil {
+			result = multierror.Append(result, fmt.Errorf("failed to kill process after faulty SIGTERM: %w", err))
+		}
+
+		return result.ErrorOrNil()
+	}
+
+	done := make(chan error, 1)
+	go func() {
+		done <- bi.cmd.Wait()
+	}()
+
+	select {
+	case err := <-done:
+		return err
+	case <-time.After(binaryIOProcTermTimeout):
+		shimLog.Warn("failed to wait for shim logger process to exit, killing")
+
+		err := bi.cmd.Process.Kill()
+		if err != nil {
+			return fmt.Errorf("failed to kill shim logger process: %w", err)
+		}
+
+		return nil
+	}
+}
+
+func newPipe() (*pipe, error) {
+	r, w, err := os.Pipe()
+	if err != nil {
+		return nil, err
+	}
+	return &pipe{
+		r: r,
+		w: w,
+	}, nil
+}
+
+type pipe struct {
+	r *os.File
+	w *os.File
+}
+
+// https://github.com/containerd/containerd/blob/v1.6.6/vendor/github.com/containerd/go-runc/io.go#L71
+func (p *pipe) Close() error {
+	var result *multierror.Error
+
+	if err := p.w.Close(); err != nil {
+		result = multierror.Append(result, fmt.Errorf("failed to close write pipe: %w", err))
+	}
+
+	if err := p.r.Close(); err != nil {
+		result = multierror.Append(result, fmt.Errorf("failed to close read pipe: %w", err))
+	}
+
+	return multierror.Prefix(result.ErrorOrNil(), "pipe:")
+}

src/runtime/pkg/containerd-shim-v2/shim_io_file.go

@@ -0,0 +1,80 @@
+// Copyright (c) 2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"context"
+	"io"
+	"net/url"
+	"os"
+	"path/filepath"
+
+	cioutil "github.com/containerd/containerd/pkg/ioutil"
+)
+
+var (
+	_ IO = &fileIO{}
+)
+
+// fileIO only support write both stdout/stderr to the same file
+type fileIO struct {
+	outw io.WriteCloser
+	errw io.WriteCloser
+	path string
+}
+
+// openLogFile opens/creates a container log file with its directory.
+func openLogFile(path string) (*os.File, error) {
+	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+		return nil, err
+	}
+	return os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0640)
+}
+
+func newFileIO(ctx context.Context, stdio *stdio, uri *url.URL) (*fileIO, error) {
+	var outw, errw, f io.WriteCloser
+	var err error
+
+	logFile := uri.Path
+	if f, err = openLogFile(logFile); err != nil {
+		return nil, err
+	}
+
+	if stdio.Stdout != "" {
+		outw = cioutil.NewSerialWriteCloser(f)
+	}
+
+	if !stdio.Console && stdio.Stderr != "" {
+		errw = cioutil.NewSerialWriteCloser(f)
+	}
+
+	return &fileIO{
+		path: logFile,
+		outw: outw,
+		errw: errw,
+	}, nil
+}
+
+func (fi *fileIO) Close() error {
+	if fi.outw != nil {
+		return wc(fi.outw)
+	} else if fi.errw != nil {
+		return wc(fi.errw)
+	}
+	return nil
+}
+
+func (fi *fileIO) Stdin() io.ReadCloser {
+	return nil
+}
+
+func (fi *fileIO) Stdout() io.Writer {
+	return fi.outw
+}
+
+func (fi *fileIO) Stderr() io.Writer {
+	return fi.errw
+}

src/runtime/pkg/containerd-shim-v2/shim_io_pipe.go

@@ -0,0 +1,95 @@
+// Copyright (c) 2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"syscall"
+
+	"github.com/containerd/fifo"
+	"github.com/hashicorp/go-multierror"
+)
+
+var (
+	_ IO = &pipeIO{}
+)
+
+type pipeIO struct {
+	in   io.ReadCloser
+	outw io.WriteCloser
+	errw io.WriteCloser
+}
+
+func newPipeIO(ctx context.Context, stdio *stdio) (*pipeIO, error) {
+	var in io.ReadCloser
+	var outw io.WriteCloser
+	var errw io.WriteCloser
+	var err error
+
+	if stdio.Stdin != "" {
+		in, err = fifo.OpenFifo(ctx, stdio.Stdin, syscall.O_RDONLY|syscall.O_NONBLOCK, 0)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if stdio.Stdout != "" {
+		outw, err = fifo.OpenFifo(ctx, stdio.Stdout, syscall.O_RDWR, 0)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if !stdio.Console && stdio.Stderr != "" {
+		errw, err = fifo.OpenFifo(ctx, stdio.Stderr, syscall.O_RDWR, 0)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	pipeIO := &pipeIO{
+		in:   in,
+		outw: outw,
+		errw: errw,
+	}
+
+	return pipeIO, nil
+}
+
+func (pi *pipeIO) Stdin() io.ReadCloser {
+	return pi.in
+}
+
+func (pi *pipeIO) Stdout() io.Writer {
+	return pi.outw
+}
+
+func (pi *pipeIO) Stderr() io.Writer {
+	return pi.errw
+}
+
+func (pi *pipeIO) Close() error {
+	var result *multierror.Error
+
+	if pi.in != nil {
+		if err := pi.in.Close(); err != nil {
+			result = multierror.Append(result, fmt.Errorf("failed to close stdin: %w", err))
+		}
+		pi.in = nil
+	}
+
+	if err := wc(pi.outw); err != nil {
+		result = multierror.Append(result, fmt.Errorf("failed to close stdout: %w", err))
+	}
+
+	if err := wc(pi.errw); err != nil {
+		result = multierror.Append(result, fmt.Errorf("failed to close stderr: %w", err))
+	}
+
+	return result.ErrorOrNil()
+}

src/runtime/pkg/containerd-shim-v2/shim_management.go

@@ -29,11 +29,17 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	dto "github.com/prometheus/client_model/go"
 	"github.com/prometheus/common/expfmt"
+	"github.com/sirupsen/logrus"
 )
 
 const (
+	DirectVolumePathKey   = "path"
+	AgentUrl              = "/agent-url"
 	DirectVolumeStatUrl   = "/direct-volume/stats"
 	DirectVolumeResizeUrl = "/direct-volume/resize"
+	IPTablesUrl           = "/iptables"
+	IP6TablesUrl          = "/ip6tables"
+	MetricsUrl            = "/metrics"
 )
 
 var (
@@ -139,7 +145,16 @@ func decodeAgentMetrics(body string) []*dto.MetricFamily {
 }
 
 func (s *service) serveVolumeStats(w http.ResponseWriter, r *http.Request) {
-	volumePath, err := url.PathUnescape(strings.TrimPrefix(r.URL.Path, DirectVolumeStatUrl))
+	val := r.URL.Query().Get(DirectVolumePathKey)
+	if val == "" {
+		msg := fmt.Sprintf("Required parameter %s not found", DirectVolumePathKey)
+		shimMgtLog.Info(msg)
+		w.WriteHeader(http.StatusBadRequest)
+		w.Write([]byte(msg))
+		return
+	}
+
+	volumePath, err := url.PathUnescape(val)
 	if err != nil {
 		shimMgtLog.WithError(err).Error("failed to unescape the volume stat url path")
 		w.WriteHeader(http.StatusInternalServerError)
@@ -184,6 +199,48 @@ func (s *service) serveVolumeResize(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte(""))
 }
 
+func (s *service) ip6TablesHandler(w http.ResponseWriter, r *http.Request) {
+	s.genericIPTablesHandler(w, r, true)
+}
+
+func (s *service) ipTablesHandler(w http.ResponseWriter, r *http.Request) {
+	s.genericIPTablesHandler(w, r, false)
+}
+
+func (s *service) genericIPTablesHandler(w http.ResponseWriter, r *http.Request, isIPv6 bool) {
+	logger := shimMgtLog.WithFields(logrus.Fields{"handler": "iptables", "ipv6": isIPv6})
+
+	switch r.Method {
+	case http.MethodPut:
+		body, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			logger.WithError(err).Error("failed to read request body")
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte(err.Error()))
+			return
+		}
+
+		if err = s.sandbox.SetIPTables(context.Background(), isIPv6, body); err != nil {
+			logger.WithError(err).Error("failed to set IPTables")
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte(err.Error()))
+		}
+		w.Write([]byte(""))
+
+	case http.MethodGet:
+		buf, err := s.sandbox.GetIPTables(context.Background(), isIPv6)
+		if err != nil {
+			logger.WithError(err).Error("failed to get IPTables")
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte(err.Error()))
+		}
+		w.Write(buf)
+	default:
+		w.WriteHeader(http.StatusNotImplemented)
+		return
+	}
+}
+
 func (s *service) startManagementServer(ctx context.Context, ociSpec *specs.Spec) {
 	// metrics socket will under sandbox's bundle path
 	metricsAddress := SocketAddress(s.id)
@@ -204,10 +261,12 @@ func (s *service) startManagementServer(ctx context.Context, ociSpec *specs.Spec
 
 	// bind handler
 	m := http.NewServeMux()
-	m.Handle("/metrics", http.HandlerFunc(s.serveMetrics))
-	m.Handle("/agent-url", http.HandlerFunc(s.agentURL))
+	m.Handle(MetricsUrl, http.HandlerFunc(s.serveMetrics))
+	m.Handle(AgentUrl, http.HandlerFunc(s.agentURL))
 	m.Handle(DirectVolumeStatUrl, http.HandlerFunc(s.serveVolumeStats))
 	m.Handle(DirectVolumeResizeUrl, http.HandlerFunc(s.serveVolumeResize))
+	m.Handle(IPTablesUrl, http.HandlerFunc(s.ipTablesHandler))
+	m.Handle(IP6TablesUrl, http.HandlerFunc(s.ip6TablesHandler))
 	s.mountPprofHandle(m, ociSpec)
 
 	// register shim metrics

src/runtime/pkg/containerd-shim-v2/start.go

@@ -8,6 +8,7 @@ package containerdshim
 import (
 	"context"
 	"fmt"
+
 	"github.com/sirupsen/logrus"
 
 	"github.com/containerd/containerd/api/types/task"
@@ -75,7 +76,7 @@ func startContainer(ctx context.Context, s *service, c *container) (retErr error
 	c.stdinPipe = stdin
 
 	if c.stdin != "" || c.stdout != "" || c.stderr != "" {
-		tty, err := newTtyIO(ctx, c.stdin, c.stdout, c.stderr, c.terminal)
+		tty, err := newTtyIO(ctx, s.namespace, c.id, c.stdin, c.stdout, c.stderr, c.terminal)
 		if err != nil {
 			return err
 		}
@@ -141,7 +142,7 @@ func startExec(ctx context.Context, s *service, containerID, execID string) (e *
 
 	execs.stdinPipe = stdin
 
-	tty, err := newTtyIO(ctx, execs.tty.stdin, execs.tty.stdout, execs.tty.stderr, execs.tty.terminal)
+	tty, err := newTtyIO(ctx, s.namespace, execs.id, execs.tty.stdin, execs.tty.stdout, execs.tty.stderr, execs.tty.terminal)
 	if err != nil {
 		return nil, err
 	}

src/runtime/pkg/containerd-shim-v2/stream.go

@@ -7,16 +7,22 @@ package containerdshim
 
 import (
 	"context"
+	"fmt"
 	"io"
+	"net/url"
 	"sync"
-	"syscall"
 
-	"github.com/containerd/fifo"
 	"github.com/sirupsen/logrus"
 )
 
-// The buffer size used to specify the buffer for IO streams copy
-const bufSize = 32 << 10
+const (
+	// The buffer size used to specify the buffer for IO streams copy
+	bufSize = 32 << 10
+
+	shimLogPluginBinary = "binary"
+	shimLogPluginFifo   = "fifo"
+	shimLogPluginFile   = "file"
+)
 
 var (
 	bufPool = sync.Pool{
@@ -27,76 +33,84 @@ var (
 	}
 )
 
+type stdio struct {
+	Stdin   string
+	Stdout  string
+	Stderr  string
+	Console bool
+}
+type IO interface {
+	io.Closer
+	Stdin() io.ReadCloser
+	Stdout() io.Writer
+	Stderr() io.Writer
+}
+
 type ttyIO struct {
-	Stdin  io.ReadCloser
-	Stdout io.Writer
-	Stderr io.Writer
+	io  IO
+	raw *stdio
 }
 
 func (tty *ttyIO) close() {
-
-	if tty.Stdin != nil {
-		tty.Stdin.Close()
-		tty.Stdin = nil
-	}
-	cf := func(w io.Writer) {
-		if w == nil {
-			return
-		}
-		if c, ok := w.(io.WriteCloser); ok {
-			c.Close()
-		}
-	}
-	cf(tty.Stdout)
-	cf(tty.Stderr)
+	tty.io.Close()
 }
 
-func newTtyIO(ctx context.Context, stdin, stdout, stderr string, console bool) (*ttyIO, error) {
-	var in io.ReadCloser
-	var outw io.Writer
-	var errw io.Writer
+// newTtyIO creates a new ttyIO struct.
+// ns(namespace)/id(container ID) are used for containerd binary IO.
+// containerd will pass the ns/id as ENV to the binary log driver,
+// and the binary log driver will use ns/id to get the log options config file.
+// for example nerdctl: https://github.com/containerd/nerdctl/blob/v0.21.0/pkg/logging/logging.go#L102
+func newTtyIO(ctx context.Context, ns, id, stdin, stdout, stderr string, console bool) (*ttyIO, error) {
 	var err error
+	var io IO
 
-	if stdin != "" {
-		in, err = fifo.OpenFifo(ctx, stdin, syscall.O_RDONLY|syscall.O_NONBLOCK, 0)
-		if err != nil {
-			return nil, err
-		}
+	raw := &stdio{
+		Stdin:   stdin,
+		Stdout:  stdout,
+		Stderr:  stderr,
+		Console: console,
 	}
 
-	if stdout != "" {
-		outw, err = fifo.OpenFifo(ctx, stdout, syscall.O_RDWR, 0)
-		if err != nil {
-			return nil, err
-		}
+	uri, err := url.Parse(stdout)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse stdout uri: %w", err)
 	}
 
-	if !console && stderr != "" {
-		errw, err = fifo.OpenFifo(ctx, stderr, syscall.O_RDWR, 0)
-		if err != nil {
-			return nil, err
-		}
+	if uri.Scheme == "" {
+		uri.Scheme = "fifo"
 	}
 
-	ttyIO := &ttyIO{
-		Stdin:  in,
-		Stdout: outw,
-		Stderr: errw,
+	switch uri.Scheme {
+	case shimLogPluginFifo:
+		io, err = newPipeIO(ctx, raw)
+	case shimLogPluginBinary:
+		io, err = newBinaryIO(ctx, ns, id, uri)
+	case shimLogPluginFile:
+		io, err = newFileIO(ctx, raw, uri)
+	default:
+		return nil, fmt.Errorf("unknown STDIO scheme %s", uri.Scheme)
 	}
 
-	return ttyIO, nil
+	if err != nil {
+		return nil, fmt.Errorf("failed to creat io stream: %w", err)
+	}
+
+	return &ttyIO{
+		io:  io,
+		raw: raw,
+	}, nil
 }
 
 func ioCopy(shimLog *logrus.Entry, exitch, stdinCloser chan struct{}, tty *ttyIO, stdinPipe io.WriteCloser, stdoutPipe, stderrPipe io.Reader) {
 	var wg sync.WaitGroup
 
-	if tty.Stdin != nil {
+	if tty.io.Stdin() != nil {
 		wg.Add(1)
 		go func() {
 			shimLog.Debug("stdin io stream copy started")
 			p := bufPool.Get().(*[]byte)
 			defer bufPool.Put(p)
-			io.CopyBuffer(stdinPipe, tty.Stdin, *p)
+			io.CopyBuffer(stdinPipe, tty.io.Stdin(), *p)
 			// notify that we can close process's io safely.
 			close(stdinCloser)
 			wg.Done()
@@ -104,30 +118,30 @@ func ioCopy(shimLog *logrus.Entry, exitch, stdinCloser chan struct{}, tty *ttyIO
 		}()
 	}
 
-	if tty.Stdout != nil {
+	if tty.io.Stdout() != nil {
 		wg.Add(1)
 
 		go func() {
 			shimLog.Debug("stdout io stream copy started")
 			p := bufPool.Get().(*[]byte)
 			defer bufPool.Put(p)
-			io.CopyBuffer(tty.Stdout, stdoutPipe, *p)
+			io.CopyBuffer(tty.io.Stdout(), stdoutPipe, *p)
 			wg.Done()
-			if tty.Stdin != nil {
+			if tty.io.Stdin() != nil {
 				// close stdin to make the other routine stop
-				tty.Stdin.Close()
+				tty.io.Stdin().Close()
 			}
 			shimLog.Debug("stdout io stream copy exited")
 		}()
 	}
 
-	if tty.Stderr != nil && stderrPipe != nil {
+	if tty.io.Stderr() != nil && stderrPipe != nil {
 		wg.Add(1)
 		go func() {
 			shimLog.Debug("stderr io stream copy started")
 			p := bufPool.Get().(*[]byte)
 			defer bufPool.Put(p)
-			io.CopyBuffer(tty.Stderr, stderrPipe, *p)
+			io.CopyBuffer(tty.io.Stderr(), stderrPipe, *p)
 			wg.Done()
 			shimLog.Debug("stderr io stream copy exited")
 		}()
@@ -138,3 +152,10 @@ func ioCopy(shimLog *logrus.Entry, exitch, stdinCloser chan struct{}, tty *ttyIO
 	close(exitch)
 	shimLog.Debug("all io stream copy goroutines exited")
 }
+
+func wc(w io.WriteCloser) error {
+	if w == nil {
+		return nil
+	}
+	return w.Close()
+}

src/runtime/pkg/containerd-shim-v2/stream_test.go

@@ -7,7 +7,6 @@ package containerdshim
 
 import (
 	"context"
-	"github.com/sirupsen/logrus"
 	"io"
 	"os"
 	"path/filepath"
@@ -15,6 +14,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/sirupsen/logrus"
+
 	"github.com/containerd/fifo"
 	"github.com/stretchr/testify/assert"
 )
@@ -45,7 +46,7 @@ func TestNewTtyIOFifoReopen(t *testing.T) {
 	defer outr.Close()
 	errr = createReadFifo(stderr)
 	defer errr.Close()
-	tty, err = newTtyIO(ctx, "", stdout, stderr, false)
+	tty, err = newTtyIO(ctx, "", "", "", stdout, stderr, false)
 	assert.NoError(err)
 	defer tty.close()
 
@@ -72,9 +73,9 @@ func TestNewTtyIOFifoReopen(t *testing.T) {
 		}
 	}
 
-	checkFifoWrite(tty.Stdout)
+	checkFifoWrite(tty.io.Stdout())
 	checkFifoRead(outr)
-	checkFifoWrite(tty.Stderr)
+	checkFifoWrite(tty.io.Stderr())
 	checkFifoRead(errr)
 
 	err = outr.Close()
@@ -84,8 +85,8 @@ func TestNewTtyIOFifoReopen(t *testing.T) {
 
 	// Make sure that writing to tty fifo will not get `EPIPE`
 	// when the read side is closed
-	checkFifoWrite(tty.Stdout)
-	checkFifoWrite(tty.Stderr)
+	checkFifoWrite(tty.io.Stdout())
+	checkFifoWrite(tty.io.Stderr())
 
 	// Reopen the fifo
 	outr = createReadFifo(stdout)
@@ -171,7 +172,7 @@ func TestIoCopy(t *testing.T) {
 			defer srcInW.Close()
 		}
 
-		tty, err := newTtyIO(ctx, srcStdinPath, dstStdoutPath, dstStderrPath, false)
+		tty, err := newTtyIO(ctx, "", "", srcStdinPath, dstStdoutPath, dstStderrPath, false)
 		assert.NoError(err)
 		defer tty.close()
 

src/runtime/pkg/containerd-shim-v2/utils_test.go

@@ -28,7 +28,6 @@ const (
 
 	testSandboxID   = "777-77-77777777"
 	testContainerID = "42"
-	testConsole     = "/dev/pts/888"
 
 	testContainerTypeAnnotation = "io.kubernetes.cri.container-type"
 	testSandboxIDAnnotation     = "io.kubernetes.cri.sandbox-id"
@@ -91,7 +90,7 @@ func newTestHypervisorConfig(dir string, create bool) (vc.HypervisorConfig, erro
 }
 
 // newTestRuntimeConfig creates a new RuntimeConfig
-func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConfig, error) {
+func newTestRuntimeConfig(dir string, create bool) (oci.RuntimeConfig, error) {
 	if dir == "" {
 		return oci.RuntimeConfig{}, errors.New("BUG: need directory")
 	}
@@ -104,7 +103,6 @@ func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConf
 	return oci.RuntimeConfig{
 		HypervisorType:   vc.QemuHypervisor,
 		HypervisorConfig: hypervisorConfig,
-		Console:          consolePath,
 	}, nil
 }
 

src/runtime/pkg/containerd-shim-v2/wait.go

@@ -53,6 +53,11 @@ func wait(ctx context.Context, s *service, c *container, execID string) (int32,
 			"container": c.id,
 			"pid":       processID,
 		}).Error("Wait for process failed")
+
+		// set return code if wait failed
+		if ret == 0 {
+			ret = exitCode255
+		}
 	}
 
 	timeStamp := time.Now()
@@ -78,7 +83,7 @@ func wait(ctx context.Context, s *service, c *container, execID string) (int32,
 				shimLog.WithField("sandbox", s.sandbox.ID()).Error("failed to delete sandbox")
 			}
 		} else {
-			if _, err = s.sandbox.StopContainer(ctx, c.id, false); err != nil {
+			if _, err = s.sandbox.StopContainer(ctx, c.id, true); err != nil {
 				shimLog.WithError(err).WithField("container", c.id).Warn("stop container failed")
 			}
 		}
@@ -158,10 +163,10 @@ func watchOOMEvents(ctx context.Context, s *service) {
 			containerID, err := s.sandbox.GetOOMEvent(ctx)
 			if err != nil {
 				if err.Error() == "ttrpc: closed" || err.Error() == "Dead agent" {
-					shimLog.WithError(err).Warn("agent has shutdown, return from watching of OOM events")
+					shimLog.WithError(err).Info("agent has shutdown, return from watching of OOM events")
 					return
 				}
-				shimLog.WithError(err).Warn("failed to get OOM event from sandbox")
+				shimLog.WithError(err).Info("failed to get OOM event from sandbox")
 				time.Sleep(defaultCheckInterval)
 				continue
 			}

src/runtime/pkg/device/api/interface.go

@@ -9,8 +9,7 @@ package api
 import (
 	"context"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/sirupsen/logrus"
 )
 
@@ -78,10 +77,10 @@ type Device interface {
 	Dereference() uint
 
 	// Save converts Device to DeviceState
-	Save() persistapi.DeviceState
+	Save() config.DeviceState
 
 	// Load loads DeviceState and converts it to specific device
-	Load(persistapi.DeviceState)
+	Load(config.DeviceState)
 }
 
 // DeviceManager can be used to create a new device, this can be used as single
@@ -94,5 +93,5 @@ type DeviceManager interface {
 	IsDeviceAttached(string) bool
 	GetDeviceByID(string) Device
 	GetAllDevices() []Device
-	LoadDevices([]persistapi.DeviceState)
+	LoadDevices([]config.DeviceState)
 }

src/runtime/pkg/device/api/mockDeviceReceiver.go

@@ -8,7 +8,7 @@ package api
 import (
 	"context"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 )
 
 // MockDeviceReceiver is a fake DeviceReceiver API implementation only used for test

src/runtime/pkg/device/config/config.go

@@ -441,3 +441,45 @@ func getVhostUserDevName(dirname string, majorNum, minorNum uint32) (string, err
 	return "", fmt.Errorf("Required device node (%d:%d) doesn't exist under directory %s",
 		majorNum, minorNum, dirname)
 }
+
+// DeviceState is a structure which represents host devices
+// plugged to a hypervisor, one Device can be shared among containers in POD
+// Refs: pkg/device/drivers/generic.go:GenericDevice
+type DeviceState struct {
+	// DriverOptions is specific options for each device driver
+	// for example, for BlockDevice, we can set DriverOptions["block-driver"]="virtio-blk"
+	DriverOptions map[string]string
+
+	// VhostUserDeviceAttrs is specific for vhost-user device driver
+	VhostUserDev *VhostUserDeviceAttrs `json:",omitempty"`
+
+	// BlockDrive is specific for block device driver
+	BlockDrive *BlockDrive `json:",omitempty"`
+
+	ID string
+
+	// Type is used to specify driver type
+	// Refs: pkg/device/config/config.go:DeviceType
+	Type string
+
+	// Type of device: c, b, u or p
+	// c , u - character(unbuffered)
+	// p - FIFO
+	// b - block(buffered) special file
+	// More info in mknod(1).
+	DevType string
+
+	// VFIODev is specific VFIO device driver
+	VFIODevs []*VFIODev `json:",omitempty"`
+
+	RefCount    uint
+	AttachCount uint
+
+	// Major, minor numbers for device.
+	Major int64
+	Minor int64
+
+	// ColdPlug specifies whether the device must be cold plugged (true)
+	// or hot plugged (false).
+	ColdPlug bool
+}

src/runtime/pkg/device/config/pmem.go

@@ -26,7 +26,7 @@ const (
 )
 
 var (
-	pmemLog = logrus.WithField("source", "virtcontainers/device/config")
+	pmemLog = logrus.WithField("source", "pkg/device/config")
 )
 
 // SetLogger sets up a logger for this pkg

src/runtime/pkg/device/drivers/block.go

@@ -10,9 +10,8 @@ import (
 	"context"
 	"path/filepath"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
 
@@ -159,51 +158,21 @@ func (device *BlockDevice) GetDeviceInfo() interface{} {
 }
 
 // Save converts Device to DeviceState
-func (device *BlockDevice) Save() persistapi.DeviceState {
+func (device *BlockDevice) Save() config.DeviceState {
 	ds := device.GenericDevice.Save()
 	ds.Type = string(device.DeviceType())
 
-	drive := device.BlockDrive
-	if drive != nil {
-		ds.BlockDrive = &persistapi.BlockDrive{
-			File:     drive.File,
-			Format:   drive.Format,
-			ID:       drive.ID,
-			Index:    drive.Index,
-			MmioAddr: drive.MmioAddr,
-			PCIPath:  drive.PCIPath,
-			SCSIAddr: drive.SCSIAddr,
-			NvdimmID: drive.NvdimmID,
-			VirtPath: drive.VirtPath,
-			DevNo:    drive.DevNo,
-			Pmem:     drive.Pmem,
-		}
-	}
+	ds.BlockDrive = device.BlockDrive
+
 	return ds
 }
 
 // Load loads DeviceState and converts it to specific device
-func (device *BlockDevice) Load(ds persistapi.DeviceState) {
+func (device *BlockDevice) Load(ds config.DeviceState) {
 	device.GenericDevice = &GenericDevice{}
 	device.GenericDevice.Load(ds)
 
-	bd := ds.BlockDrive
-	if bd == nil {
-		return
-	}
-	device.BlockDrive = &config.BlockDrive{
-		File:     bd.File,
-		Format:   bd.Format,
-		ID:       bd.ID,
-		Index:    bd.Index,
-		MmioAddr: bd.MmioAddr,
-		PCIPath:  bd.PCIPath,
-		SCSIAddr: bd.SCSIAddr,
-		NvdimmID: bd.NvdimmID,
-		VirtPath: bd.VirtPath,
-		DevNo:    bd.DevNo,
-		Pmem:     bd.Pmem,
-	}
+	device.BlockDrive = ds.BlockDrive
 }
 
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation

src/runtime/pkg/device/drivers/generic.go

@@ -10,9 +10,8 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 )
 
 // GenericDevice refers to a device that is neither a VFIO device, block device or VhostUserDevice.
@@ -128,8 +127,8 @@ func (device *GenericDevice) bumpAttachCount(attach bool) (skip bool, err error)
 }
 
 // Save converts Device to DeviceState
-func (device *GenericDevice) Save() persistapi.DeviceState {
-	dss := persistapi.DeviceState{
+func (device *GenericDevice) Save() config.DeviceState {
+	dss := config.DeviceState{
 		ID:          device.ID,
 		Type:        string(device.DeviceType()),
 		RefCount:    device.RefCount,
@@ -148,7 +147,7 @@ func (device *GenericDevice) Save() persistapi.DeviceState {
 }
 
 // Load loads DeviceState and converts it to specific device
-func (device *GenericDevice) Load(ds persistapi.DeviceState) {
+func (device *GenericDevice) Load(ds config.DeviceState) {
 	device.ID = ds.ID
 	device.RefCount = ds.RefCount
 	device.AttachCount = ds.AttachCount

src/runtime/pkg/device/drivers/generic_test.go

@@ -8,7 +8,7 @@ package drivers
 import (
 	"testing"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/stretchr/testify/assert"
 )
 

src/runtime/pkg/device/drivers/utils.go

@@ -12,8 +12,8 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/sirupsen/logrus"
 )
 
@@ -45,9 +45,8 @@ func deviceLogger() *logrus.Entry {
 	return api.DeviceLogger()
 }
 
-// Identify PCIe device by /sys/bus/pci/slots/xx/max_bus_speed, sample content "8.0 GT/s PCIe"
-// The /sys/bus/pci/slots/xx/address contains bdf, sample content "0000:04:00"
-// bdf format: bus:slot.function
+// Identify PCIe device by reading the size of the PCI config space
+// Plain PCI device have 256 bytes of config space where PCIe devices have 4K
 func isPCIeDevice(bdf string) bool {
 	if len(strings.Split(bdf, ":")) == 2 {
 		bdf = PCIDomain + ":" + bdf

src/runtime/pkg/device/drivers/vfio.go

@@ -16,9 +16,8 @@ import (
 
 	"github.com/sirupsen/logrus"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
 
@@ -174,26 +173,21 @@ func (device *VFIODevice) GetDeviceInfo() interface{} {
 }
 
 // Save converts Device to DeviceState
-func (device *VFIODevice) Save() persistapi.DeviceState {
+func (device *VFIODevice) Save() config.DeviceState {
 	ds := device.GenericDevice.Save()
 	ds.Type = string(device.DeviceType())
 
 	devs := device.VfioDevs
 	for _, dev := range devs {
 		if dev != nil {
-			ds.VFIODevs = append(ds.VFIODevs, &persistapi.VFIODev{
-				ID:       dev.ID,
-				Type:     uint32(dev.Type),
-				BDF:      dev.BDF,
-				SysfsDev: dev.SysfsDev,
-			})
+			ds.VFIODevs = append(ds.VFIODevs, dev)
 		}
 	}
 	return ds
 }
 
 // Load loads DeviceState and converts it to specific device
-func (device *VFIODevice) Load(ds persistapi.DeviceState) {
+func (device *VFIODevice) Load(ds config.DeviceState) {
 	device.GenericDevice = &GenericDevice{}
 	device.GenericDevice.Load(ds)
 
@@ -222,6 +216,7 @@ func getVFIODetails(deviceFileName, iommuDevicesPath string) (deviceBDF, deviceS
 		// Get sysfsdev of device eg. /sys/devices/pci0000:00/0000:00:02.0/f79944e4-5a3d-11e8-99ce-479cbab002e4
 		sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName)
 		deviceSysfsDev, err = getSysfsDev(sysfsDevStr)
+		deviceBDF = getBDF(getMediatedBDF(deviceSysfsDev))
 	default:
 		err = fmt.Errorf("Incorrect tokens found while parsing vfio details: %s", deviceFileName)
 	}
@@ -229,10 +224,23 @@ func getVFIODetails(deviceFileName, iommuDevicesPath string) (deviceBDF, deviceS
 	return deviceBDF, deviceSysfsDev, vfioDeviceType, err
 }
 
+// getMediatedBDF returns the BDF of a VF
+// Expected input string format is /sys/devices/pci0000:d7/BDF0/BDF1/.../MDEVBDF/UUID
+func getMediatedBDF(deviceSysfsDev string) string {
+	tokens := strings.SplitN(deviceSysfsDev, "/", -1)
+	if len(tokens) < 4 {
+		return ""
+	}
+	return tokens[len(tokens)-2]
+}
+
 // getBDF returns the BDF of pci device
 // Expected input string format is [<domain>]:[<bus>][<slot>].[<func>] eg. 0000:02:10.0
 func getBDF(deviceSysStr string) string {
 	tokens := strings.SplitN(deviceSysStr, ":", 2)
+	if len(tokens) == 1 {
+		return ""
+	}
 	return tokens[1]
 }
 

src/runtime/pkg/device/drivers/vfio_test.go

@@ -9,7 +9,7 @@ package drivers
 import (
 	"testing"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -46,4 +46,5 @@ func TestGetVFIODetails(t *testing.T) {
 			assert.Nil(t, err)
 		}
 	}
+
 }

src/runtime/pkg/device/drivers/vhost_user_blk.go

@@ -9,9 +9,8 @@ package drivers
 import (
 	"context"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 	"github.com/sirupsen/logrus"
 )
@@ -156,40 +155,19 @@ func (device *VhostUserBlkDevice) GetDeviceInfo() interface{} {
 }
 
 // Save converts Device to DeviceState
-func (device *VhostUserBlkDevice) Save() persistapi.DeviceState {
+func (device *VhostUserBlkDevice) Save() config.DeviceState {
 	ds := device.GenericDevice.Save()
 	ds.Type = string(device.DeviceType())
+	ds.VhostUserDev = device.VhostUserDeviceAttrs
 
-	vAttr := device.VhostUserDeviceAttrs
-	if vAttr != nil {
-		ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{
-			DevID:      vAttr.DevID,
-			SocketPath: vAttr.SocketPath,
-			Type:       string(vAttr.Type),
-			PCIPath:    vAttr.PCIPath,
-			Index:      vAttr.Index,
-		}
-	}
 	return ds
 }
 
 // Load loads DeviceState and converts it to specific device
-func (device *VhostUserBlkDevice) Load(ds persistapi.DeviceState) {
+func (device *VhostUserBlkDevice) Load(ds config.DeviceState) {
 	device.GenericDevice = &GenericDevice{}
 	device.GenericDevice.Load(ds)
-
-	dev := ds.VhostUserDev
-	if dev == nil {
-		return
-	}
-
-	device.VhostUserDeviceAttrs = &config.VhostUserDeviceAttrs{
-		DevID:      dev.DevID,
-		SocketPath: dev.SocketPath,
-		Type:       config.DeviceType(dev.Type),
-		PCIPath:    dev.PCIPath,
-		Index:      dev.Index,
-	}
+	device.VhostUserDeviceAttrs = ds.VhostUserDev
 }
 
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation

src/runtime/pkg/device/drivers/vhost_user_fs.go

@@ -9,8 +9,8 @@ import (
 	"context"
 	"encoding/hex"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
 

src/runtime/pkg/device/drivers/vhost_user_net.go

@@ -10,16 +10,15 @@ import (
 	"context"
 	"encoding/hex"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
 
 // VhostUserNetDevice is a network vhost-user based device
 type VhostUserNetDevice struct {
 	*GenericDevice
-	config.VhostUserDeviceAttrs
+	*config.VhostUserDeviceAttrs
 }
 
 //
@@ -71,38 +70,25 @@ func (device *VhostUserNetDevice) DeviceType() config.DeviceType {
 // GetDeviceInfo returns device information used for creating
 func (device *VhostUserNetDevice) GetDeviceInfo() interface{} {
 	device.Type = device.DeviceType()
-	return &device.VhostUserDeviceAttrs
+	return device.VhostUserDeviceAttrs
 }
 
 // Save converts Device to DeviceState
-func (device *VhostUserNetDevice) Save() persistapi.DeviceState {
+func (device *VhostUserNetDevice) Save() config.DeviceState {
 	ds := device.GenericDevice.Save()
 	ds.Type = string(device.DeviceType())
-	ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{
-		DevID:      device.DevID,
-		SocketPath: device.SocketPath,
-		Type:       string(device.Type),
-		MacAddress: device.MacAddress,
-	}
+
+	ds.VhostUserDev = device.VhostUserDeviceAttrs
+
 	return ds
 }
 
 // Load loads DeviceState and converts it to specific device
-func (device *VhostUserNetDevice) Load(ds persistapi.DeviceState) {
+func (device *VhostUserNetDevice) Load(ds config.DeviceState) {
 	device.GenericDevice = &GenericDevice{}
 	device.GenericDevice.Load(ds)
 
-	dev := ds.VhostUserDev
-	if dev == nil {
-		return
-	}
-
-	device.VhostUserDeviceAttrs = config.VhostUserDeviceAttrs{
-		DevID:      dev.DevID,
-		SocketPath: dev.SocketPath,
-		Type:       config.DeviceType(dev.Type),
-		MacAddress: dev.MacAddress,
-	}
+	device.VhostUserDeviceAttrs = ds.VhostUserDev
 }
 
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation

src/runtime/pkg/device/drivers/vhost_user_scsi.go

@@ -10,16 +10,15 @@ import (
 	"context"
 	"encoding/hex"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
 
 // VhostUserSCSIDevice is a SCSI vhost-user based device
 type VhostUserSCSIDevice struct {
 	*GenericDevice
-	config.VhostUserDeviceAttrs
+	*config.VhostUserDeviceAttrs
 }
 
 //
@@ -71,38 +70,24 @@ func (device *VhostUserSCSIDevice) DeviceType() config.DeviceType {
 // GetDeviceInfo returns device information used for creating
 func (device *VhostUserSCSIDevice) GetDeviceInfo() interface{} {
 	device.Type = device.DeviceType()
-	return &device.VhostUserDeviceAttrs
+	return device.VhostUserDeviceAttrs
 }
 
 // Save converts Device to DeviceState
-func (device *VhostUserSCSIDevice) Save() persistapi.DeviceState {
+func (device *VhostUserSCSIDevice) Save() config.DeviceState {
 	ds := device.GenericDevice.Save()
 	ds.Type = string(device.DeviceType())
-	ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{
-		DevID:      device.DevID,
-		SocketPath: device.SocketPath,
-		Type:       string(device.Type),
-		MacAddress: device.MacAddress,
-	}
+	ds.VhostUserDev = device.VhostUserDeviceAttrs
+
 	return ds
 }
 
 // Load loads DeviceState and converts it to specific device
-func (device *VhostUserSCSIDevice) Load(ds persistapi.DeviceState) {
+func (device *VhostUserSCSIDevice) Load(ds config.DeviceState) {
 	device.GenericDevice = &GenericDevice{}
 	device.GenericDevice.Load(ds)
 
-	dev := ds.VhostUserDev
-	if dev == nil {
-		return
-	}
-
-	device.VhostUserDeviceAttrs = config.VhostUserDeviceAttrs{
-		DevID:      dev.DevID,
-		SocketPath: dev.SocketPath,
-		Type:       config.DeviceType(dev.Type),
-		MacAddress: dev.MacAddress,
-	}
+	device.VhostUserDeviceAttrs = ds.VhostUserDev
 }
 
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation

src/runtime/pkg/device/manager/manager.go

@@ -14,10 +14,9 @@ import (
 
 	"github.com/sirupsen/logrus"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers"
-	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
 
@@ -242,7 +241,7 @@ func (dm *deviceManager) IsDeviceAttached(id string) bool {
 }
 
 // LoadDevices load devices from persist state
-func (dm *deviceManager) LoadDevices(devStates []persistapi.DeviceState) {
+func (dm *deviceManager) LoadDevices(devStates []config.DeviceState) {
 	dm.Lock()
 	defer dm.Unlock()
 

src/runtime/pkg/device/manager/manager_linux_test.go

@@ -13,10 +13,10 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers"
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers"
 	"github.com/stretchr/testify/assert"
 
 	"golang.org/x/sys/unix"