Merge pull request #2976 from bergwolf/2.3.0-rc0-branch-bump

# Kata Containers 2.3.0-rc0

.github/workflows/kata-deploy-test.yaml

@@ -5,26 +5,12 @@ on:
 name: test-kata-deploy
 
 jobs:
-  check_comments:
-    if: ${{ github.event.issue.pull_request }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check for Command
-        id: command
-        uses: kata-containers/slash-command-action@v1
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          command: "test_kata_deploy"
-          reaction: "true"
-          reaction-type: "eyes"
-          allow-edits: "false"
-          permission-level: admin
-      - name: verify command arg is kata-deploy
-        run: |
-           echo "The command was '${{ steps.command.outputs.command-name }}' with arguments '${{ steps.command.outputs.command-arguments }}'"
-
   create-and-test-container:
-    needs: check_comments
+    if: |
+      github.event.issue.pull_request
+      && github.event_name == 'issue_comment'
+      && github.event.action == 'created'
+      && startsWith(github.event.comment.body, '/test_kata_deploy')
     runs-on: ubuntu-latest
     steps:
       - name: get-PR-ref

.github/workflows/release.yaml

@@ -149,3 +149,30 @@ jobs:
           tar -cvzf "${tarball}" src/agent/.cargo/config src/agent/vendor
           GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}" 
           popd
+
+  upload-libseccomp-tarball:
+    needs: upload-cargo-vendored-tarball
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: download-and-upload-tarball
+        env:
+          GITHUB_TOKEN: ${{ secrets.GIT_UPLOAD_TOKEN }}
+        run: |
+          pushd $GITHUB_WORKSPACE
+          ./ci/install_yq.sh
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          versions_yaml="versions.yaml"
+          version=$(yq read ${versions_yaml} "externals.libseccomp.version")
+          repo_url=$(yq read ${versions_yaml} "externals.libseccomp.url")
+          download_url="${repo_url}/releases/download/v${version}"
+          tarball="libseccomp-${version}.tar.gz"
+          asc="${tarball}.asc"
+          curl -sSLO "${download_url}/${tarball}"
+          curl -sSLO "${download_url}/${asc}"
+          # "-m" option should be empty to re-use the existing release title
+          # without opening a text editor.
+          # For the details, check https://hub.github.com/hub-release.1.html.
+          hub release edit -m "" -a "${tarball}" "${tag}"
+          hub release edit -m "" -a "${asc}" "${tag}"
+          popd

.github/workflows/static-checks.yaml

@@ -67,6 +67,14 @@ jobs:
         PATH=$PATH:"$HOME/.cargo/bin"
         rustup target add x86_64-unknown-linux-musl
         rustup component add rustfmt clippy
+    - name: Setup seccomp
+      run: |
+        libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
+        gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
+        echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
+        echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
+        echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
     # Check whether the vendored code is up-to-date & working as the first thing
     - name: Check vendored code
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}

VERSION

@@ -1 +1 @@
-2.3.0-alpha2
+2.3.0-rc0

ci/go-no-os-exit.sh

@@ -1,30 +0,0 @@
-#!/bin/bash
-# Copyright (c) 2018 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-# Check there are no os.Exit() calls creeping into the code
-# We don't use that exit path in the Kata codebase.
-
-# Allow the path to check to be over-ridden.
-# Default to the current directory.
-go_packages=${1:-.}
-
-echo "Checking for no os.Exit() calls for package [${go_packages}]"
-
-candidates=`go list -f '{{.Dir}}/*.go' $go_packages`
-for f in $candidates; do
-	filename=`basename $f`
-	# skip all go test files
-	[[ $filename == *_test.go ]] && continue
-	# skip exit.go where, the only file we should call os.Exit() from.
-	[[ $filename == "exit.go" ]] && continue
-	files="$f $files"
-done
-
-[ -z "$files" ] && echo "No files to check, skipping" && exit 0
-
-if egrep -n '\<os\.Exit\>' $files; then
-	echo "Direct calls to os.Exit() are forbidden, please use exit() so atexit() works"
-	exit 1
-fi

ci/install_libseccomp.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+#
+# Copyright 2021 Sony Group Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -o errexit
+
+cidir=$(dirname "$0")
+source "${cidir}/lib.sh"
+
+clone_tests_repo
+
+source "${tests_repo_dir}/.ci/lib.sh"
+
+# The following variables if set on the environment will change the behavior
+# of gperf and libseccomp configure scripts, that may lead this script to
+# fail. So let's ensure they are unset here.
+unset PREFIX DESTDIR
+
+arch=$(uname -m)
+workdir="$(mktemp -d --tmpdir build-libseccomp.XXXXX)"
+
+# Variables for libseccomp
+# Currently, specify the libseccomp version directly without using `versions.yaml`
+# because the current Snap workflow is incomplete.
+# After solving the issue, replace this code by using the `versions.yaml`.
+# libseccomp_version=$(get_version "externals.libseccomp.version")
+# libseccomp_url=$(get_version "externals.libseccomp.url")
+libseccomp_version="2.5.1"
+libseccomp_url="https://github.com/seccomp/libseccomp"
+libseccomp_tarball="libseccomp-${libseccomp_version}.tar.gz"
+libseccomp_tarball_url="${libseccomp_url}/releases/download/v${libseccomp_version}/${libseccomp_tarball}"
+cflags="-O2"
+
+# Variables for gperf
+# Currently, specify the gperf version directly without using `versions.yaml`
+# because the current Snap workflow is incomplete.
+# After solving the issue, replace this code by using the `versions.yaml`.
+# gperf_version=$(get_version "externals.gperf.version")
+# gperf_url=$(get_version "externals.gperf.url")
+gperf_version="3.1"
+gperf_url="https://ftp.gnu.org/gnu/gperf"
+gperf_tarball="gperf-${gperf_version}.tar.gz"
+gperf_tarball_url="${gperf_url}/${gperf_tarball}"
+
+# We need to build the libseccomp library from sources to create a static library for the musl libc.
+# However, ppc64le and s390x have no musl targets in Rust. Hence, we do not set cflags for the musl libc.
+if ([ "${arch}" != "ppc64le" ] && [ "${arch}" != "s390x" ]); then
+    # Set FORTIFY_SOURCE=1 because the musl-libc does not have some functions about FORTIFY_SOURCE=2
+    cflags="-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=1 -O2"
+fi
+
+die() {
+    msg="$*"
+    echo "[Error] ${msg}" >&2
+    exit 1
+}
+
+finish() {
+    rm -rf "${workdir}"
+}
+
+trap finish EXIT
+
+build_and_install_gperf() {
+    echo "Build and install gperf version ${gperf_version}"
+    mkdir -p "${gperf_install_dir}"
+    curl -sLO "${gperf_tarball_url}"
+    tar -xf "${gperf_tarball}"
+    pushd "gperf-${gperf_version}"
+    ./configure --prefix="${gperf_install_dir}"
+    make
+    make install
+    export PATH=$PATH:"${gperf_install_dir}"/bin
+    popd
+    echo "Gperf installed successfully"
+}
+
+build_and_install_libseccomp() {
+    echo "Build and install libseccomp version ${libseccomp_version}"
+    mkdir -p "${libseccomp_install_dir}"
+    curl -sLO "${libseccomp_tarball_url}"
+    tar -xf "${libseccomp_tarball}"
+    pushd "libseccomp-${libseccomp_version}"
+    ./configure --prefix="${libseccomp_install_dir}" CFLAGS="${cflags}" --enable-static
+    make
+    make install
+    popd
+    echo "Libseccomp installed successfully"
+}
+
+main() {
+    local libseccomp_install_dir="${1:-}"
+    local gperf_install_dir="${2:-}"
+
+    if [ -z "${libseccomp_install_dir}" ] || [ -z "${gperf_install_dir}" ]; then
+        die "Usage: ${0} <libseccomp-install-dir> <gperf-install-dir>"
+    fi
+
+    pushd "$workdir"
+    # gperf is required for building the libseccomp.
+    build_and_install_gperf
+    build_and_install_libseccomp
+    popd
+}
+
+main "$@"

ci/install_rust.sh

@@ -12,5 +12,5 @@ source "${cidir}/lib.sh"
 clone_tests_repo
 
 pushd ${tests_repo_dir}
-.ci/install_rust.sh
+.ci/install_rust.sh ${1:-}
 popd

docs/Developer-Guide.md

@@ -86,6 +86,16 @@ One of the `initrd` and `image` options in Kata runtime config file **MUST** be
 The main difference between the options is that the size of `initrd`(10MB+) is significantly smaller than
 rootfs `image`(100MB+).
 
+## Enable seccomp
+
+Enable seccomp as follows:
+
+```
+$ sudo sed -i '/^disable_guest_seccomp/ s/true/false/' /etc/kata-containers/configuration.toml
+```
+
+This will pass container seccomp profiles to the kata agent.
+
 ## Enable full debug
 
 Enable full debug as follows:
@@ -216,6 +226,18 @@ $ go get -d -u github.com/kata-containers/kata-containers
 $ cd $GOPATH/src/github.com/kata-containers/kata-containers/src/agent && make
 ```
 
+The agent is built with seccomp capability by default.
+If you want to build the agent without the seccomp capability, you need to run `make` with `SECCOMP=no` as follows.
+
+```
+$ make -C $GOPATH/src/github.com/kata-containers/kata-containers/src/agent SECCOMP=no
+```
+
+> **Note:**
+>
+> - If you enable seccomp in the main configuration file but build the agent without seccomp capability,
+>   the runtime exits conservatively with an error message.
+
 ## Get the osbuilder
 
 ```
@@ -234,9 +256,21 @@ the following example.
 $ export ROOTFS_DIR=${GOPATH}/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder/rootfs
 $ sudo rm -rf ${ROOTFS_DIR}
 $ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder
-$ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
+$ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true ./rootfs.sh ${distro}'
+```
+
+You MUST choose a distribution (e.g., `ubuntu`) for `${distro}`.
+You can get a supported distributions list in the Kata Containers by running the following.
+
+```
+$ ./rootfs.sh -l
+```
+
+If you want to build the agent without seccomp capability, you need to run the `rootfs.sh` script with `SECCOMP=no` as follows.
+
+```
+$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
 ```
-You MUST choose one of `alpine`, `centos`, `clearlinux`, `debian`, `euleros`, `fedora`, `suse`, and `ubuntu` for `${distro}`. By default `seccomp` packages are not included in the rootfs image. Set `SECCOMP` to `yes` to include them.
 
 > **Note:**
 >
@@ -272,6 +306,7 @@ $ script -fec 'sudo -E USE_DOCKER=true ./image_builder.sh ${ROOTFS_DIR}'
 > - If you do *not* wish to build under Docker, remove the `USE_DOCKER`
 >   variable in the previous command and ensure the `qemu-img` command is
 >   available on your system.
+>   - If `qemu-img` is not installed, you will likely see errors such as `ERROR: File /dev/loop19p1 is not a block device` and `losetup: /tmp/tmp.bHz11oY851: Warning: file is smaller than 512 bytes; the loop device may be useless or invisible for system tools`. These can be mitigated by installing the `qemu-img` command (available in the `qemu-img` package on Fedora or the `qemu-utils` package on Debian).
 
 
 ### Install the rootfs image
@@ -290,12 +325,23 @@ $ (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
 $ export ROOTFS_DIR="${GOPATH}/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder/rootfs"
 $ sudo rm -rf ${ROOTFS_DIR}
 $ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder
-$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
+$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true ./rootfs.sh ${distro}'
 ```
 `AGENT_INIT` controls if the guest image uses the Kata agent as the guest `init` process. When you create an initrd image,
-always set `AGENT_INIT` to `yes`. By default `seccomp` packages are not included in the initrd image. Set `SECCOMP` to `yes` to include them.
+always set `AGENT_INIT` to `yes`.
 
-You MUST choose one of `alpine`, `centos`, `clearlinux`, `euleros`, and `fedora` for `${distro}`.
+You MUST choose a distribution (e.g., `ubuntu`) for `${distro}`.
+You can get a supported distributions list in the Kata Containers by running the following.
+
+```
+$ ./rootfs.sh -l
+```
+
+If you want to build the agent without seccomp capability, you need to run the `rootfs.sh` script with `SECCOMP=no` as follows.
+
+```
+$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
+```
 
 > **Note:**
 >

docs/README.md

@@ -11,6 +11,10 @@ For details of the other Kata Containers repositories, see the
 
 * [Installation guides](./install/README.md): Install and run Kata Containers with Docker or Kubernetes
 
+## Tracing
+
+See the [tracing documentation](tracing.md).
+
 ## More User Guides
 
 * [Upgrading](Upgrading.md): how to upgrade from [Clear Containers](https://github.com/clearcontainers) and [runV](https://github.com/hyperhq/runv) to [Kata Containers](https://github.com/kata-containers) and how to upgrade an existing Kata Containers system to the latest version.

docs/design/kata-design-requirements.md

@@ -30,7 +30,7 @@ The Kata Containers runtime **MUST** implement the following command line option
 The Kata Containers project **MUST** provide two interfaces for CRI shims to manage hardware
 virtualization based Kubernetes pods and containers:
 -  An OCI and `runc` compatible command line interface, as described in the previous section.
-This interface is used by implementations such as [`CRI-O`](http://cri-o.io) and [`cri-containerd`](https://github.com/containerd/cri-containerd), for example.
+This interface is used by implementations such as [`CRI-O`](http://cri-o.io) and [`containerd`](https://github.com/containerd/containerd), for example.
 - A hardware virtualization runtime library API for CRI shims to consume and provide a more
 CRI native implementation. The [`frakti`](https://github.com/kubernetes/frakti) CRI shim is an example of such a consumer.
 

docs/how-to/README.md

@@ -5,7 +5,7 @@
 - [Run Kata containers with `crictl`](run-kata-with-crictl.md)
 - [Run Kata Containers with Kubernetes](run-kata-with-k8s.md)
 - [How to use Kata Containers and Containerd](containerd-kata.md)
-- [How to use Kata Containers and CRI (containerd plugin) with Kubernetes](how-to-use-k8s-with-cri-containerd-and-kata.md)
+- [How to use Kata Containers and CRI (containerd) with Kubernetes](how-to-use-k8s-with-cri-containerd-and-kata.md)
 - [Kata Containers and service mesh for Kubernetes](service-mesh.md)
 - [How to import Kata Containers logs into Fluentd](how-to-import-kata-logs-with-fluentd.md)
 

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -34,8 +34,6 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.agent.enable_tracing` | `boolean` | enable tracing for the agent |
 | `io.katacontainers.config.agent.container_pipe_size` | uint32 | specify the size of the std(in/out) pipes created for containers |
 | `io.katacontainers.config.agent.kernel_modules` | string | the list of kernel modules and their parameters that will be loaded in the guest kernel. Semicolon separated list of kernel modules and their parameters. These modules will be loaded in the guest kernel using `modprobe`(8). E.g., `e1000e InterruptThrottleRate=3000,3000,3000 EEE=1; i915 enable_ppgtt=0` |
-| `io.katacontainers.config.agent.trace_mode` | string | the trace mode for the agent |
-| `io.katacontainers.config.agent.trace_type` | string | the trace type for the agent |
 
 ## Hypervisor Options
 | Key | Value Type | Comments |

docs/how-to/how-to-use-k8s-with-cri-containerd-and-kata.md

@@ -3,7 +3,7 @@
 This document describes how to set up a single-machine Kubernetes (k8s) cluster.
 
 The Kubernetes cluster will use the
-[CRI containerd plugin](https://github.com/containerd/containerd/tree/main/pkg/cri) and
+[CRI containerd](https://github.com/containerd/containerd/) and
 [Kata Containers](https://katacontainers.io) to launch untrusted workloads.
 
 ## Requirements
@@ -71,12 +71,12 @@ $ for service in ${services}; do
     service_dir="/etc/systemd/system/${service}.service.d/"
     sudo mkdir -p ${service_dir}
 
-    cat << EOT | sudo tee "${service_dir}/proxy.conf"
+    cat << EOF | sudo tee "${service_dir}/proxy.conf"
 [Service]
 Environment="HTTP_PROXY=${http_proxy}"
 Environment="HTTPS_PROXY=${https_proxy}"
 Environment="NO_PROXY=${no_proxy}"
-EOT
+EOF
 done
 
 $ sudo systemctl daemon-reload
@@ -172,7 +172,7 @@ If a pod has the `runtimeClassName` set to `kata`, the CRI plugin runs the pod w
 - Create an pod configuration that using Kata Containers runtime
 
   ```bash
-  $ cat << EOT | tee nginx-kata.yaml
+  $ cat << EOF | tee nginx-kata.yaml
   apiVersion: v1
   kind: Pod
   metadata:
@@ -183,7 +183,7 @@ If a pod has the `runtimeClassName` set to `kata`, the CRI plugin runs the pod w
     - name: nginx
       image: nginx
       
-  EOT
+  EOF
   ```
 
 - Create the pod

docs/how-to/how-to-use-kata-containers-with-acrn.md

@@ -22,7 +22,7 @@ This document requires the presence of the ACRN hypervisor and Kata Containers o
 
 - ACRN supported [Hardware](https://projectacrn.github.io/latest/hardware.html#supported-hardware).
   > **Note:** Please make sure to have a minimum of 4 logical processors (HT) or cores.
-- ACRN [software](https://projectacrn.github.io/latest/tutorials/kbl-nuc-sdc.html#use-the-script-to-set-up-acrn-automatically) setup.
+- ACRN [software](https://projectacrn.github.io/latest/tutorials/run_kata_containers.html) setup.
 - For networking, ACRN supports either MACVTAP or TAP. If MACVTAP is not enabled in the Service OS, please follow the below steps to update the kernel:
 
   ```sh

docs/how-to/privileged.md

@@ -16,9 +16,9 @@ from the host, a potentially undesirable side-effect that decreases the security
 
 The following sections document how to configure this behavior in different container runtimes.
 
-#### Containerd and CRI
+#### Containerd
 
-The Containerd CRI allows configuring the privileged host devices behavior for each runtime in the CRI config. This is
+The Containerd allows configuring the privileged host devices behavior for each runtime in the containerd config. This is
 done with the `privileged_without_host_devices` option. Setting this to `true` will disable hot plugging of the host 
 devices into the guest, even when privileged is enabled.
 
@@ -41,7 +41,7 @@ See below example config:
 ```
 
  - [Kata Containers with Containerd and CRI documentation](how-to-use-k8s-with-cri-containerd-and-kata.md)
- - [Containerd CRI config documentation](https://github.com/containerd/cri/blob/master/docs/config.md)
+ - [Containerd CRI config documentation](https://github.com/containerd/containerd/blob/main/docs/cri/config.md)
 
 #### CRI-O
 

docs/how-to/run-kata-with-k8s.md

@@ -9,7 +9,7 @@ Kubernetes CRI (Container Runtime Interface) implementations allow using any
 OCI-compatible runtime with Kubernetes, such as the Kata Containers runtime.
 
 Kata Containers support both the [CRI-O](https://github.com/kubernetes-incubator/cri-o) and
-[CRI-containerd](https://github.com/containerd/cri) CRI implementations.
+[containerd](https://github.com/containerd/containerd) CRI implementations.
 
 After choosing one CRI implementation, you must make the appropriate configuration
 to ensure it integrates with Kata Containers.
@@ -111,11 +111,7 @@ manage_ns_lifecycle = true
 ```
 
 
-### containerd with CRI plugin
-
-If you select containerd with `cri` plugin, follow the "Getting Started for Developers"
-instructions [here](https://github.com/containerd/cri#getting-started-for-developers)
-to properly install it.
+### containerd
 
 To customize containerd to select Kata Containers runtime, follow our
 "Configure containerd to use Kata Containers" internal documentation
@@ -160,7 +156,7 @@ $ sudo systemctl restart kubelet
 # If using CRI-O
 $ sudo kubeadm init --ignore-preflight-errors=all --cri-socket /var/run/crio/crio.sock --pod-network-cidr=10.244.0.0/16
 
-# If using CRI-containerd
+# If using containerd
 $ sudo kubeadm init --ignore-preflight-errors=all --cri-socket /run/containerd/containerd.sock --pod-network-cidr=10.244.0.0/16
 
 $ export KUBECONFIG=/etc/kubernetes/admin.conf

docs/how-to/service-mesh.md

@@ -34,7 +34,7 @@ as the proxy starts.
 
 Follow the [instructions](../install/README.md)
 to get Kata Containers properly installed and configured with Kubernetes.
-You can choose between CRI-O and CRI-containerd, both are supported
+You can choose between CRI-O and containerd, both are supported
 through this document.
 
 For both cases, select the workloads as _trusted_ by default. This way,
@@ -159,7 +159,7 @@ containers with `privileged: true` to `privileged: false`.
 There is no difference between Istio and Linkerd in this section. It is
 about which CRI implementation you use.
 
-For both CRI-O and CRI-containerd, you have to add an annotation indicating
+For both CRI-O and containerd, you have to add an annotation indicating
 the workload for this deployment is not _trusted_, which will trigger
 `kata-runtime` to be called instead of `runc`.
 
@@ -193,9 +193,9 @@ spec:
 ...
 ```
 
-__CRI-containerd:__
+__containerd:__
 
-Add the following annotation for CRI-containerd
+Add the following annotation for containerd
 ```yaml
 io.kubernetes.cri.untrusted-workload: "true"
 ```

docs/tracing.md

@@ -0,0 +1,214 @@
+# Overview
+
+This document explains how to trace Kata Containers components.
+
+# Introduction
+
+The Kata Containers runtime and agent are able to generate
+[OpenTelemetry][opentelemetry] trace spans, which allow the administrator to
+observe what those components are doing and how much time they are spending on
+each operation.
+
+# OpenTelemetry summary
+
+An OpenTelemetry-enabled application creates a number of trace "spans". A span
+contains the following attributes:
+
+- A name
+- A pair of timestamps (recording the start time and end time of some operation)
+- A reference to the span's parent span
+
+All spans need to be *finished*, or *completed*, to allow the OpenTelemetry
+framework to generate the final trace information (by effectively closing the
+transaction encompassing the initial (root) span and all its children).
+
+For Kata, the root span represents the total amount of time taken to run a
+particular component from startup to its shutdown (the "run time").
+
+# Architecture
+
+## Runtime tracing architecture
+
+The runtime, which runs in the host environment, has been modified to
+optionally generate trace spans which are sent to a trace collector on the
+host.
+
+## Agent tracing architecture
+
+An OpenTelemetry system (such as [Jaeger][jaeger-tracing]) uses a collector to
+gather up trace spans from the application for viewing and processing. For an
+application to use the collector, it must run in the same context as
+the collector.
+
+This poses a problem for tracing the Kata Containers agent since it does not
+run in the same context as the collector: it runs inside a virtual machine (VM).
+
+To allow spans from the agent to be sent to the trace collector, Kata provides
+a [trace forwarder][trace-forwarder] component. This runs in the same context
+as the collector (generally on the host system) and listens on a
+[`VSOCK`][vsock] channel for traces generated by the agent, forwarding them on
+to the trace collector.
+
+> **Note:**
+>
+> This design supports agent tracing without having to make changes to the
+> image, but also means that [custom images][osbuilder] can also benefit from
+> agent tracing.
+
+The following diagram summarises the architecture used to trace the Kata
+Containers agent:
+
+```
++--------------------------------------------+
+| Host                                       |
+|                                            |
+| +---------------+                          |
+| | OpenTelemetry |                          |
+| | Trace         |                          |
+| | Collector     |                          |
+| +---------------+                          |
+|       ^                  +---------------+ |
+|       | spans            | Kata VM       | |
+| +-----+-----+            |               | |
+| | Kata      |    spans   o     +-------+ | |
+| | Trace     |<-----------------| Kata  | | |
+| | Forwarder |    VSOCK   o     | Agent | | |
+| +-----------+    Channel |     +-------+ | |
+|                          +---------------+ |
++--------------------------------------------+
+```
+
+# Agent tracing prerequisites
+
+- You must have a trace collector running.
+
+  Although the collector normally runs on the host, it can also be run from
+  inside a Docker image configured to expose the appropriate host ports to the
+  collector.
+
+  The [Jaeger "all-in-one" Docker image][jaeger-all-in-one] method
+  is the quickest and simplest way to run the collector for testing.
+
+- If you wish to trace the agent, you must start the
+  [trace forwarder][trace-forwarder].
+
+> **Notes:**
+>
+> - If agent tracing is enabled but the forwarder is not running,
+>   the agent will log an error (signalling that it cannot generate trace
+>   spans), but continue to work as normal.
+>
+> - The trace forwarder requires a trace collector (such as Jaeger) to be
+>   running before it is started. If a collector is not running, the trace
+>   forwarder will exit with an error.
+
+# Enable tracing
+
+By default, tracing is disabled for all components. To enable _any_ form of
+tracing an `enable_tracing` option must be enabled for at least one component.
+
+> **Note:** 
+>
+> Enabling this option will only allow tracing for subsequently
+> started containers.
+
+## Enable runtime tracing
+
+To enable runtime tracing, set the tracing option as shown:
+
+```toml
+[runtime]
+enable_tracing = true
+```
+
+## Enable agent tracing
+
+To enable agent tracing, set the tracing option as shown:
+
+```toml
+[agent.kata]
+enable_tracing = true
+```
+
+> **Note:**
+>
+> If both agent tracing and runtime tracing are enabled, the resulting trace
+> spans will be "collated": expanding individual runtime spans in the Jaeger
+> web UI will show the agent trace spans resulting from the runtime
+> operation.
+
+# Appendices
+
+## Agent tracing requirements
+
+### Host environment
+
+- The host kernel must support the VSOCK socket type.
+
+  This will be available if the kernel is built with the
+  `CONFIG_VHOST_VSOCK` configuration option.
+
+- The VSOCK kernel module must be loaded:
+
+   ```
+   $ sudo modprobe vhost_vsock
+   ```
+
+### Guest environment
+
+- The guest kernel must support the VSOCK socket type:
+
+  This will be available if the kernel is built with the
+  `CONFIG_VIRTIO_VSOCKETS` configuration option.
+
+  > **Note:** The default Kata Containers guest kernel provides this feature.
+
+## Agent tracing limitations
+
+- Agent tracing is only "completed" when the workload and the Kata agent
+  process have exited.
+
+  Although trace information *can* be inspected before the workload and agent
+  have exited, it is incomplete. This is shown as `<trace-without-root-span>`
+  in the Jaeger web UI.
+
+  If the workload is still running, the trace transaction -- which spans the entire
+  runtime of the Kata agent -- will not have been completed. To view the complete
+  trace details, wait for the workload to end, or stop the container.
+
+## Performance impact
+
+[OpenTelemetry][opentelemetry] is designed for high performance. It combines
+the best of two previous generation projects (OpenTracing and OpenCensus) and
+uses a very efficient mechanism to capture trace spans. Further, the trace
+points inserted into the agent are generated dynamically at compile time. This
+is advantageous since new versions of the agent will automatically benefit
+from improvements in the tracing infrastructure. Overall, the impact of
+enabling runtime and agent tracing should be extremely low.
+
+## Agent shutdown behaviour
+ 
+In normal operation, the Kata runtime manages the VM shutdown and performs
+certain optimisations to speed up this process. However, if agent tracing is
+enabled, the agent itself is responsible for shutting down the VM. This it to
+ensure all agent trace transactions are completed. This means there will be a
+small performance impact for container shutdown when agent tracing is enabled
+as the runtime must wait for the VM to shutdown fully.
+
+## Set up a tracing development environment
+
+If you want to debug, further develop, or test tracing,
+[enabling full debug][enable-full-debug]
+is highly recommended. For working with the agent, you may also wish to
+[enable a debug console][setup-debug-console]
+to allow you to access the VM environment.
+
+[agent-ctl]: https://github.com/kata-containers/kata-containers/blob/main/tools/agent-ctl
+[enable-full-debug]: https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md#enable-full-debug
+[jaeger-all-in-one]: https://www.jaegertracing.io/docs/getting-started/
+[jaeger-tracing]: https://www.jaegertracing.io
+[opentelemetry]: https://opentelemetry.io
+[osbuilder]: https://github.com/kata-containers/kata-containers/blob/main/tools/osbuilder
+[setup-debug-console]: https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md#set-up-a-debug-console
+[trace-forwarder]: https://github.com/kata-containers/kata-containers/blob/main/src/trace-forwarder
+[vsock]: https://wiki.qemu.org/Features/VirtioVsock

docs/use-cases/using-Intel-SGX-and-kata.md

@@ -1,107 +1,113 @@
 # Kata Containers with SGX
 
-Intel® Software Guard Extensions (SGX) is a set of instructions that increases the security
+Intel Software Guard Extensions (SGX) is a set of instructions that increases the security
 of applications code and data, giving them more protections from disclosure or modification.
 
-> **Note:** At the time of writing this document, SGX patches have not landed on the Linux kernel
-> project, so specific versions for guest and host kernels must be installed to enable SGX.
+This document guides you to run containers with SGX enclaves with Kata Containers in Kubernetes.
 
-## Check if SGX is enabled
+## Preconditions
 
-Run the following command to check if your host supports SGX.
+* Intel SGX capable bare metal nodes
+* Host kernel Linux 5.13 or later with SGX and SGX KVM enabled:
 
 ```sh
-$ grep -o sgx /proc/cpuinfo
+$ grep SGX /boot/config-`uname -r`
+CONFIG_X86_SGX=y
+CONFIG_X86_SGX_KVM=y
 ```
 
-Continue to the following section if the output of the above command is empty,
-otherwise continue to section [Install Guest kernel with SGX support](#install-guest-kernel-with-sgx-support)
+* Kubernetes cluster configured with:
+   * [`kata-deploy`](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy) based Kata Containers installation
+   * [Intel SGX Kubernetes device plugin](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/main/cmd/sgx_plugin#deploying-with-pre-built-images)
 
-## Install Host kernel with SGX support
+> Note: Kata Containers supports creating VM sandboxes with Intel® SGX enabled
+> using [cloud-hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor/) VMM only. QEMU support is waiting to get the
+> Intel SGX enabled QEMU upstream release.
 
-The following commands were tested on Fedora 32, they might work on other distros too.
+## Installation
+
+### Kata Containers Guest Kernel
+
+Follow the instructions to [setup](../../tools/packaging/kernel/README.md#setup-kernel-source-code) and [build](../../tools/packaging/kernel/README.md#build-the-kernel) the experimental guest kernel. Then, install as:
 
 ```sh
-$ git clone --depth=1 https://github.com/intel/kvm-sgx
-$ pushd kvm-sgx
-$ cp /boot/config-$(uname -r) .config
-$ yes "" | make oldconfig
-$ # In the following step, enable: INTEL_SGX and INTEL_SGX_VIRTUALIZATION
-$ make menuconfig
-$ make -j$(($(nproc)-1)) bzImage
-$ make -j$(($(nproc)-1)) modules
-$ sudo make modules_install
-$ sudo make install
-$ popd
-$ sudo reboot
+$ sudo cp kata-linux-experimental-*/vmlinux /opt/kata/share/kata-containers/vmlinux.sgx
+$ sudo sed -i 's|vmlinux.container|vmlinux.sgx|g' \
+  /opt/kata/share/defaults/kata-containers/configuration-clh.toml
 ```
 
-> **Notes:**
-> * Run: `mokutil --sb-state` to check whether secure boot is enabled, if so, you will need to sign the kernel.
-> * You'll lose SGX support when a new distro kernel is installed and the system rebooted.
-
-Once you have restarted your system with the new brand Linux Kernel with SGX support, run
-the following command to make sure it's enabled. If the output is empty, go to the BIOS
-setup and enable SGX manually.
-
-```sh
-$ grep -o sgx /proc/cpuinfo
-```
-
-## Install Guest kernel with SGX support
-
-Install the guest kernel in the Kata Containers directory, this way it can be used to run
-Kata Containers.
-
-```sh
-$ curl -LOk https://github.com/devimc/kvm-sgx/releases/download/v0.0.1/kata-virtiofs-sgx.tar.gz
-$ sudo tar -xf kata-virtiofs-sgx.tar.gz -C /usr/share/kata-containers/
-$ sudo sed -i 's|kernel =|kernel = "/usr/share/kata-containers/vmlinux-virtiofs-sgx.container"|g' \
-  /usr/share/defaults/kata-containers/configuration.toml
-```
-
-## Run Kata Containers with SGX enabled
+### Kata Containers Configuration
 
 Before running a Kata Container make sure that your version of `crio` or `containerd`
 supports annotations.
+
 For `containerd` check in `/etc/containerd/config.toml` that the list of `pod_annotations` passed
 to the `sandbox` are: `["io.katacontainers.*", "sgx.intel.com/epc"]`.
 
-> `sgx.yaml`
+## Usage
+
+With the following sample job deployed using `kubectl apply -f`:
+
 ```yaml
-apiVersion: v1
-kind: Pod
+apiVersion: batch/v1
+kind: Job
 metadata:
-  name: sgx
-  annotations:
-    sgx.intel.com/epc: "32Mi"
+  name: oesgx-demo-job
+  labels:
+    jobgroup: oesgx-demo
 spec:
-  terminationGracePeriodSeconds: 0
-  runtimeClassName: kata
-  containers:
-  - name: c1
-    image: busybox
-    command:
-        - sh
-    stdin: true
-    tty: true
-    volumeMounts:
-    - mountPath: /dev/sgx/
-      name: test-volume
-  volumes:
-  - name: test-volume
-    hostPath:
-      path: /dev/sgx/
-      type: Directory
+  template:
+    metadata:
+      labels:
+        jobgroup: oesgx-demo
+    spec:
+      runtimeClassName: kata-clh
+      initContainers:
+        - name: init-sgx
+          image: busybox
+          command: ['sh', '-c', 'mkdir /dev/sgx; ln -s /dev/sgx_enclave /dev/sgx/enclave; ln -s /dev/sgx_provision /dev/sgx/provision']
+          volumeMounts:
+          - mountPath: /dev
+            name: dev-mount
+      restartPolicy: Never
+      containers:
+        -
+          name: eosgx-demo-job-1
+          image: oeciteam/oe-helloworld:latest
+          imagePullPolicy: IfNotPresent
+          securityContext:
+            readOnlyRootFilesystem: true
+            capabilities:
+              add: ["IPC_LOCK"]
+          resources:
+            limits:
+              sgx.intel.com/epc: "512Ki"
+      volumes:
+        - name: dev-mount
+          hostPath:
+            path: /dev
 ```
 
+You'll see the enclave output:
+
 ```sh
-$ kubectl apply -f sgx.yaml
-$ kubectl exec -ti sgx ls /dev/sgx/
-enclave    provision
+$ kubectl logs oesgx-demo-job-wh42g
+Hello world from the enclave
+Enclave called into host to print: Hello World!
 ```
 
-The output of the latest command shouldn't be empty, otherwise check
-your system environment to make sure SGX is fully supported.
+### Notes
 
-[1]: github.com/cloud-hypervisor/cloud-hypervisor/
+* The Kata VM's SGX Encrypted Page Cache (EPC) memory size is based on the sum of `sgx.intel.com/epc`
+resource requests within the pod.
+* `init-sgx` can be removed from the YAML configuration file if the Kata rootfs is modified with the
+necessary udev rules.
+   See the [note on SGX backwards compatibility](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/main/cmd/sgx_plugin#backwards-compatibility-note).
+* Intel SGX DCAP attestation is known to work from Kata sandboxes but it comes with one limitation: If
+the Intel SGX `aesm` daemon runs on the bare metal node and DCAP `out-of-proc` attestation is used,
+containers within the Kata sandbox cannot get the access to the host's `/var/run/aesmd/aesm.sock`
+because socket passthrough is not supported. An alternative is to deploy the `aesm` daemon as a side-car
+container.
+* Projects like [Gramine Shielded Containers (GSC)](https://gramine-gsc.readthedocs.io/en/latest/) are
+also known to work. For GSC specifically, the Kata guest kernel needs to have the `CONFIG_NUMA=y`
+enabled and at least one CPU online when running the GSC container.

pkg/logging/Cargo.toml

@@ -12,7 +12,7 @@ serde_json = "1.0.39"
 # - Dynamic keys required to allow HashMap keys to be slog::Serialized.
 # - The 'max_*' features allow changing the log level at runtime
 #   (by stopping the compiler from removing log calls).
-slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_info"] }
+slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug"] }
 slog-json = "2.3.0"
 slog-async = "2.3.0"
 slog-scope = "4.1.2"

snap/snapcraft.yaml

@@ -59,7 +59,7 @@ parts:
 
       yq_version=3.4.1
       yq_url="https://${yq_pkg}/releases/download/${yq_version}/yq_${goos}_${goarch}"
-      curl -o "${yq_path}" -LSsf "${yq_url}"
+      curl -o "${yq_path}" -L "${yq_url}"
       chmod +x "${yq_path}"
 
       kata_dir=gopath/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
@@ -139,7 +139,7 @@ parts:
       cp kata-containers*.img ${kata_image_dir}
 
   runtime:
-    after: [godeps, image]
+    after: [godeps, image, cloud-hypervisor]
     plugin: nil
     build-attributes: [no-patchelf]
     override-build: |
@@ -185,6 +185,7 @@ parts:
       - flex
     override-build: |
       yq=${SNAPCRAFT_STAGE}/yq
+      export PATH="${PATH}:${SNAPCRAFT_STAGE}"
       export GOPATH=${SNAPCRAFT_STAGE}/gopath
       kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
       versions_file="${kata_dir}/versions.yaml"
@@ -199,10 +200,17 @@ parts:
       kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
 
       cd ${kata_dir}/tools/packaging/kernel
+      kernel_dir_prefix="kata-linux-"
 
       # Setup and build kernel
-      ./build-kernel.sh -v ${kernel_version} -d setup
-      kernel_dir_prefix="kata-linux-"
+      if [ "$(uname -m)" = "x86_64" ]; then
+        kernel_version="$(${yq} r $versions_file assets.kernel-experimental.tag)"
+        kernel_version=${kernel_version#v}
+        kernel_dir_prefix="kata-linux-experimental-"
+        ./build-kernel.sh -e -v ${kernel_version} -d setup
+      else
+        ./build-kernel.sh -v ${kernel_version} -d setup
+      fi
       cd ${kernel_dir_prefix}*
       make -j $(($(nproc)-1)) EXTRAVERSION=".container"
 
@@ -327,6 +335,22 @@ parts:
       # Hack: move qemu to /
       "snap/kata-containers/current/": "./"
 
+  cloud-hypervisor:
+    plugin: nil
+    after: [godeps]
+    override-build: |
+      export GOPATH=${SNAPCRAFT_STAGE}/gopath
+      yq=${SNAPCRAFT_STAGE}/yq
+      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
+      versions_file="${kata_dir}/versions.yaml"
+      version="$(${yq} r ${versions_file} assets.hypervisor.cloud_hypervisor.version)"
+      url="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/${version}"
+      curl -L ${url}/cloud-hypervisor-static -o cloud-hypervisor
+      curl -LO ${url}/clh-remote
+
+      install -D cloud-hypervisor ${SNAPCRAFT_PART_INSTALL}/usr/bin/cloud-hypervisor
+      install -D clh-remote ${SNAPCRAFT_PART_INSTALL}/usr/bin/clh-remote
+
 apps:
   runtime:
     command: usr/bin/kata-runtime

src/agent/Cargo.lock

@@ -546,6 +546,7 @@ dependencies = [
  "scopeguard",
  "serde",
  "serde_json",
+ "serial_test",
  "slog",
  "slog-scope",
  "slog-stdlog",
@@ -593,6 +594,24 @@ dependencies = [
  "rle-decode-fast",
 ]
 
+[[package]]
+name = "libseccomp"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36ad71a5b66ceef3acfe6a3178b29b4da063f8bcb2c36dab666d52a7a9cfdb86"
+dependencies = [
+ "libc",
+ "libseccomp-sys",
+ "nix 0.17.0",
+ "pkg-config",
+]
+
+[[package]]
+name = "libseccomp-sys"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "539912de229a4fc16e507e8df12a394038a524a5b5b6c92045ad344472aac475"
+
 [[package]]
 name = "lock_api"
 version = "0.4.4"
@@ -762,6 +781,19 @@ dependencies = [
  "void",
 ]
 
+[[package]]
+name = "nix"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50e4785f2c3b7589a0d0c1dd60285e1188adac4006e8abd6dd578e1567027363"
+dependencies = [
+ "bitflags",
+ "cc",
+ "cfg-if 0.1.10",
+ "libc",
+ "void",
+]
+
 [[package]]
 name = "nix"
 version = "0.19.1"
@@ -976,6 +1008,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
+[[package]]
+name = "pkg-config"
+version = "0.3.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c"
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.10"
@@ -1283,6 +1321,7 @@ dependencies = [
  "inotify",
  "lazy_static",
  "libc",
+ "libseccomp",
  "nix 0.21.0",
  "oci",
  "path-absolutize",

src/agent/Cargo.toml

@@ -6,7 +6,6 @@ edition = "2018"
 
 [dependencies]
 oci = { path = "oci" }
-logging = { path = "../../pkg/logging" }
 rustjail = { path = "rustjail" }
 protocols = { path = "protocols" }
 lazy_static = "1.3.0"
@@ -20,6 +19,7 @@ scan_fmt = "0.2.3"
 scopeguard = "1.0.0"
 thiserror = "1.0.26"
 regex = "1"
+serial_test = "0.5.1"
 
 # Async helpers
 async-trait = "0.1.42"
@@ -35,11 +35,10 @@ rtnetlink = "0.8.0"
 netlink-packet-utils = "0.4.1"
 ipnetwork = "0.17.0"
 
-# slog:
-# - Dynamic keys required to allow HashMap keys to be slog::Serialized.
-# - The 'max_*' features allow changing the log level at runtime
-#   (by stopping the compiler from removing log calls).
-slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_info"] }
+# Note: this crate sets the slog 'max_*' features which allows the log level
+# to be modified at runtime.
+logging = { path = "../../pkg/logging" }
+slog = "2.5.2"
 slog-scope = "4.1.2"
 
 # Redirect ttrpc log calls
@@ -74,3 +73,6 @@ members = [
 
 [profile.release]
 lto = true
+
+[features]
+seccomp = ["rustjail/seccomp"]

src/agent/Makefile

@@ -27,6 +27,20 @@ COMMIT_MSG = $(if $(COMMIT),$(COMMIT),unknown)
 # Exported to allow cargo to see it
 export VERSION_COMMIT := $(if $(COMMIT),$(VERSION)-$(COMMIT),$(VERSION))
 
+EXTRA_RUSTFEATURES :=
+
+##VAR SECCOMP=yes|no define if agent enables seccomp feature
+SECCOMP := yes
+
+# Enable seccomp feature of rust build
+ifeq ($(SECCOMP),yes)
+    override EXTRA_RUSTFEATURES += seccomp
+endif
+
+ifneq ($(EXTRA_RUSTFEATURES),)
+    override EXTRA_RUSTFEATURES := --features $(EXTRA_RUSTFEATURES)
+endif
+
 include ../../utils.mk
 
 TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET)
@@ -90,15 +104,14 @@ default: $(TARGET) show-header
 $(TARGET): $(GENERATED_CODE) $(TARGET_PATH)
 
 $(TARGET_PATH): $(SOURCES) | show-summary
-	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE)
+	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
 
 $(GENERATED_FILES): %: %.in
 	@sed $(foreach r,$(GENERATED_REPLACEMENTS),-e 's|@$r@|$($r)|g') "$<" > "$@"
 
 ##TARGET optimize: optimized  build
 optimize: $(SOURCES) | show-summary show-header
-	@RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny-warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE)
-
+	@RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny-warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
 
 ##TARGET clippy: run clippy linter
 clippy: $(GENERATED_CODE)
@@ -127,7 +140,7 @@ vendor:
 
 #TARGET test: run cargo tests
 test:
-	@cargo test --all --target $(TRIPLE) -- --nocapture
+	@cargo test --all --target $(TRIPLE) $(EXTRA_RUSTFEATURES) -- --nocapture
 
 ##TARGET check: run test
 check: clippy format

src/agent/README.md

@@ -19,6 +19,7 @@ After that, we drafted the initial code here, and any contributions are welcome.
 | I/O stream              | :white_check_mark: |
 | Cgroups                 | :white_check_mark: |
 | Capabilities, `rlimit`, readonly path, masked path, users | :white_check_mark: |
+| Seccomp                 | :white_check_mark: |
 | container stats (`stats_container`)                     | :white_check_mark: |
 | Hooks                   | :white_check_mark: |
 | **Agent Features & APIs** |

src/agent/rustjail/Cargo.toml

@@ -30,7 +30,11 @@ tokio = { version = "1.2.0", features = ["sync", "io-util", "process", "time", "
 futures = "0.3"
 async-trait = "0.1.31"
 inotify = "0.9.2"
+libseccomp = { version = "0.1.3", optional = true }
 
 [dev-dependencies]
 serial_test = "0.5.0"
 tempfile = "3.1.0"
+
+[features]
+seccomp = ["libseccomp"]

src/agent/rustjail/src/container.rs

@@ -25,6 +25,8 @@ use crate::cgroups::mock::Manager as FsManager;
 use crate::cgroups::Manager;
 use crate::log_child;
 use crate::process::Process;
+#[cfg(feature = "seccomp")]
+use crate::seccomp;
 use crate::specconv::CreateOpts;
 use crate::{mount, validator};
 
@@ -151,7 +153,7 @@ lazy_static! {
             },
             LinuxDevice {
                 path: "/dev/full".to_string(),
-                r#type: String::from("c"),
+                r#type: "c".to_string(),
                 major: 1,
                 minor: 7,
                 file_mode: Some(0o666),
@@ -593,11 +595,22 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         })?;
     }
 
-    // NoNewPeiviledges, Drop capabilities
+    // NoNewPrivileges
     if oci_process.no_new_privileges {
         capctl::prctl::set_no_new_privs().map_err(|_| anyhow!("cannot set no new privileges"))?;
     }
 
+    // Without NoNewPrivileges, we need to set seccomp
+    // before dropping capabilities because the calling thread
+    // must have the CAP_SYS_ADMIN.
+    #[cfg(feature = "seccomp")]
+    if !oci_process.no_new_privileges {
+        if let Some(ref scmp) = linux.seccomp {
+            seccomp::init_seccomp(scmp)?;
+        }
+    }
+
+    // Drop capabilities
     if oci_process.capabilities.is_some() {
         let c = oci_process.capabilities.as_ref().unwrap();
         capabilities::drop_privileges(cfd_log, c)?;
@@ -641,7 +654,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     let exec_file = Path::new(&args[0]);
     log_child!(cfd_log, "process command: {:?}", &args);
     if !exec_file.exists() {
-        find_file(exec_file).ok_or_else(|| anyhow!("the file {} is not exist", &args[0]))?;
+        find_file(exec_file).ok_or_else(|| anyhow!("the file {} was not found", &args[0]))?;
     }
 
     // notify parent that the child's ready to start
@@ -669,6 +682,16 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         unistd::read(fd, &mut buf)?;
     }
 
+    // With NoNewPrivileges, we should set seccomp as close to
+    // do_exec as possible in order to reduce the amount of
+    // system calls in the seccomp profiles.
+    #[cfg(feature = "seccomp")]
+    if oci_process.no_new_privileges {
+        if let Some(ref scmp) = linux.seccomp {
+            seccomp::init_seccomp(scmp)?;
+        }
+    }
+
     do_exec(&args);
 }
 

src/agent/rustjail/src/lib.rs

@@ -34,6 +34,8 @@ pub mod container;
 pub mod mount;
 pub mod pipestream;
 pub mod process;
+#[cfg(feature = "seccomp")]
+pub mod seccomp;
 pub mod specconv;
 pub mod sync;
 pub mod sync_with_async;

src/agent/rustjail/src/mount.rs

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use anyhow::{anyhow, bail, Context, Result};
+use anyhow::{anyhow, Context, Result};
 use libc::uid_t;
 use nix::errno::Errno;
 use nix::fcntl::{self, OFlag};
@@ -19,7 +19,7 @@ use std::fs::{self, OpenOptions};
 use std::mem::MaybeUninit;
 use std::os::unix;
 use std::os::unix::io::RawFd;
-use std::path::{Path, PathBuf};
+use std::path::{Component, Path, PathBuf};
 
 use path_absolutize::*;
 use std::fs::File;
@@ -828,18 +828,35 @@ fn default_symlinks() -> Result<()> {
     }
     Ok(())
 }
+
+fn dev_rel_path(path: &str) -> Option<&Path> {
+    let path = Path::new(path);
+
+    if !path.starts_with("/dev")
+        || path == Path::new("/dev")
+        || path.components().any(|c| c == Component::ParentDir)
+    {
+        return None;
+    }
+    path.strip_prefix("/").ok()
+}
+
 fn create_devices(devices: &[LinuxDevice], bind: bool) -> Result<()> {
-    let op: fn(&LinuxDevice) -> Result<()> = if bind { bind_dev } else { mknod_dev };
+    let op: fn(&LinuxDevice, &Path) -> Result<()> = if bind { bind_dev } else { mknod_dev };
     let old = stat::umask(Mode::from_bits_truncate(0o000));
     for dev in DEFAULT_DEVICES.iter() {
-        op(dev)?;
+        let path = Path::new(&dev.path[1..]);
+        op(dev, path).context(format!("Creating container device {:?}", dev))?;
     }
     for dev in devices {
-        if !dev.path.starts_with("/dev") || dev.path.contains("..") {
+        let path = dev_rel_path(&dev.path).ok_or_else(|| {
             let msg = format!("{} is not a valid device path", dev.path);
-            bail!(anyhow!(msg));
+            anyhow!(msg)
+        })?;
+        if let Some(dir) = path.parent() {
+            fs::create_dir_all(dir).context(format!("Creating container device {:?}", dev))?;
         }
-        op(dev)?;
+        op(dev, path).context(format!("Creating container device {:?}", dev))?;
     }
     stat::umask(old);
     Ok(())
@@ -861,21 +878,21 @@ lazy_static! {
     };
 }
 
-fn mknod_dev(dev: &LinuxDevice) -> Result<()> {
+fn mknod_dev(dev: &LinuxDevice, relpath: &Path) -> Result<()> {
     let f = match LINUXDEVICETYPE.get(dev.r#type.as_str()) {
         Some(v) => v,
         None => return Err(anyhow!("invalid spec".to_string())),
     };
 
     stat::mknod(
-        &dev.path[1..],
+        relpath,
         *f,
         Mode::from_bits_truncate(dev.file_mode.unwrap_or(0)),
         nix::sys::stat::makedev(dev.major as u64, dev.minor as u64),
     )?;
 
     unistd::chown(
-        &dev.path[1..],
+        relpath,
         Some(Uid::from_raw(dev.uid.unwrap_or(0) as uid_t)),
         Some(Gid::from_raw(dev.gid.unwrap_or(0) as uid_t)),
     )?;
@@ -883,9 +900,9 @@ fn mknod_dev(dev: &LinuxDevice) -> Result<()> {
     Ok(())
 }
 
-fn bind_dev(dev: &LinuxDevice) -> Result<()> {
+fn bind_dev(dev: &LinuxDevice, relpath: &Path) -> Result<()> {
     let fd = fcntl::open(
-        &dev.path[1..],
+        relpath,
         OFlag::O_RDWR | OFlag::O_CREAT,
         Mode::from_bits_truncate(0o644),
     )?;
@@ -894,7 +911,7 @@ fn bind_dev(dev: &LinuxDevice) -> Result<()> {
 
     mount(
         Some(&*dev.path),
-        &dev.path[1..],
+        relpath,
         None::<&str>,
         MsFlags::MS_BIND,
         None::<&str>,
@@ -1258,11 +1275,12 @@ mod tests {
             uid: Some(unistd::getuid().as_raw()),
             gid: Some(unistd::getgid().as_raw()),
         };
+        let path = Path::new("fifo");
 
-        let ret = mknod_dev(&dev);
+        let ret = mknod_dev(&dev, path);
         assert!(ret.is_ok(), "Should pass. Got: {:?}", ret);
 
-        let ret = stat::stat("fifo");
+        let ret = stat::stat(path);
         assert!(ret.is_ok(), "Should pass. Got: {:?}", ret);
     }
     #[test]
@@ -1379,4 +1397,26 @@ mod tests {
             assert!(result == t.result, "{}", msg);
         }
     }
+
+    #[test]
+    fn test_dev_rel_path() {
+        // Valid device paths
+        assert_eq!(dev_rel_path("/dev/sda").unwrap(), Path::new("dev/sda"));
+        assert_eq!(dev_rel_path("//dev/sda").unwrap(), Path::new("dev/sda"));
+        assert_eq!(
+            dev_rel_path("/dev/vfio/99").unwrap(),
+            Path::new("dev/vfio/99")
+        );
+        assert_eq!(dev_rel_path("/dev/...").unwrap(), Path::new("dev/..."));
+        assert_eq!(dev_rel_path("/dev/a..b").unwrap(), Path::new("dev/a..b"));
+        assert_eq!(dev_rel_path("/dev//foo").unwrap(), Path::new("dev/foo"));
+
+        // Bad device paths
+        assert!(dev_rel_path("/devfoo").is_none());
+        assert!(dev_rel_path("/etc/passwd").is_none());
+        assert!(dev_rel_path("/dev/../etc/passwd").is_none());
+        assert!(dev_rel_path("dev/foo").is_none());
+        assert!(dev_rel_path("").is_none());
+        assert!(dev_rel_path("/dev").is_none());
+    }
 }

src/agent/rustjail/src/process.rs

@@ -24,6 +24,16 @@ use tokio::io::{split, ReadHalf, WriteHalf};
 use tokio::sync::Mutex;
 use tokio::sync::Notify;
 
+macro_rules! close_process_stream {
+    ($self: ident, $stream:ident, $stream_type: ident) => {
+        if $self.$stream.is_some() {
+            $self.close_stream(StreamType::$stream_type);
+            let _ = unistd::close($self.$stream.unwrap());
+            $self.$stream = None;
+        }
+    };
+}
+
 #[derive(Debug, PartialEq, Eq, Hash, Clone)]
 pub enum StreamType {
     Stdin,
@@ -147,6 +157,22 @@ impl Process {
         notify.notify_one();
     }
 
+    pub fn close_stdin(&mut self) {
+        close_process_stream!(self, term_master, TermMaster);
+        close_process_stream!(self, parent_stdin, ParentStdin);
+
+        self.notify_term_close();
+    }
+
+    pub fn cleanup_process_stream(&mut self) {
+        close_process_stream!(self, parent_stdin, ParentStdin);
+        close_process_stream!(self, parent_stdout, ParentStdout);
+        close_process_stream!(self, parent_stderr, ParentStderr);
+        close_process_stream!(self, term_master, TermMaster);
+
+        self.notify_term_close();
+    }
+
     fn get_fd(&self, stream_type: &StreamType) -> Option<RawFd> {
         match stream_type {
             StreamType::Stdin => self.stdin,

src/agent/rustjail/src/seccomp.rs

@@ -0,0 +1,237 @@
+// Copyright 2021 Sony Group Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use anyhow::{anyhow, Result};
+use libseccomp::*;
+use oci::{LinuxSeccomp, LinuxSeccompArg};
+use std::str::FromStr;
+
+fn get_filter_attr_from_flag(flag: &str) -> Result<ScmpFilterAttr> {
+    match flag {
+        "SECCOMP_FILTER_FLAG_TSYNC" => Ok(ScmpFilterAttr::CtlTsync),
+        "SECCOMP_FILTER_FLAG_LOG" => Ok(ScmpFilterAttr::CtlLog),
+        "SECCOMP_FILTER_FLAG_SPEC_ALLOW" => Ok(ScmpFilterAttr::CtlSsb),
+        _ => Err(anyhow!("Invalid seccomp flag")),
+    }
+}
+
+// get_rule_conditions gets rule conditions for a system call from the args.
+fn get_rule_conditions(args: &[LinuxSeccompArg]) -> Result<Vec<ScmpArgCompare>> {
+    let mut conditions: Vec<ScmpArgCompare> = Vec::new();
+
+    for arg in args {
+        if arg.op.is_empty() {
+            return Err(anyhow!("seccomp opreator is required"));
+        }
+
+        let cond = ScmpArgCompare::new(
+            arg.index,
+            ScmpCompareOp::from_str(&arg.op)?,
+            arg.value,
+            Some(arg.value_two),
+        );
+
+        conditions.push(cond);
+    }
+
+    Ok(conditions)
+}
+
+// init_seccomp creates a seccomp filter and loads it for the current process
+// including all the child processes.
+pub fn init_seccomp(scmp: &LinuxSeccomp) -> Result<()> {
+    let def_action = ScmpAction::from_str(scmp.default_action.as_str(), Some(libc::EPERM as u32))?;
+
+    // Create a new filter context
+    let mut filter = ScmpFilterContext::new_filter(def_action)?;
+
+    // Add extra architectures
+    for arch in &scmp.architectures {
+        let scmp_arch = ScmpArch::from_str(arch)?;
+        filter.add_arch(scmp_arch)?;
+    }
+
+    // Unset no new privileges bit
+    filter.set_no_new_privs_bit(false)?;
+
+    // Add a rule for each system call
+    for syscall in &scmp.syscalls {
+        if syscall.names.is_empty() {
+            return Err(anyhow!("syscall name is required"));
+        }
+
+        let action = ScmpAction::from_str(&syscall.action, Some(syscall.errno_ret))?;
+        if action == def_action {
+            continue;
+        }
+
+        for name in &syscall.names {
+            let syscall_num = get_syscall_from_name(name, None)?;
+
+            if syscall.args.is_empty() {
+                filter.add_rule(action, syscall_num, None)?;
+            } else {
+                let conditions = get_rule_conditions(&syscall.args)?;
+                filter.add_rule(action, syscall_num, Some(&conditions))?;
+            }
+        }
+    }
+
+    // Set filter attributes for each seccomp flag
+    for flag in &scmp.flags {
+        let scmp_attr = get_filter_attr_from_flag(flag)?;
+        filter.set_filter_attr(scmp_attr, 1)?;
+    }
+
+    // Load the filter
+    filter.load()?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::skip_if_not_root;
+    use libc::{dup3, process_vm_readv, EPERM, O_CLOEXEC};
+    use std::io::Error;
+    use std::ptr::null;
+
+    macro_rules! syscall_assert {
+        ($e1: expr, $e2: expr) => {
+            let mut errno: i32 = 0;
+            if $e1 < 0 {
+                errno = -Error::last_os_error().raw_os_error().unwrap();
+            }
+            assert_eq!(errno, $e2);
+        };
+    }
+
+    #[test]
+    fn test_get_filter_attr_from_flag() {
+        skip_if_not_root!();
+
+        assert_eq!(
+            get_filter_attr_from_flag("SECCOMP_FILTER_FLAG_TSYNC").unwrap(),
+            ScmpFilterAttr::CtlTsync
+        );
+
+        assert_eq!(get_filter_attr_from_flag("ERROR").is_err(), true);
+    }
+
+    #[test]
+    fn test_init_seccomp() {
+        skip_if_not_root!();
+
+        let data = r#"{
+            "defaultAction": "SCMP_ACT_ALLOW",
+            "architectures": [
+            ],
+            "flags": [
+                "SECCOMP_FILTER_FLAG_LOG"
+            ],
+            "syscalls": [
+                {
+                   "names": [
+                        "dup3"
+                    ],
+                    "action": "SCMP_ACT_ERRNO"
+                },
+                {
+                   "names": [
+                        "process_vm_readv"
+                    ],
+                    "action": "SCMP_ACT_ERRNO",
+                    "errnoRet": 111,
+                    "args": [
+                        {
+                            "index": 0,
+                            "value": 10,
+                            "op": "SCMP_CMP_EQ"
+                        }
+                    ]
+                },
+                {
+                   "names": [
+                        "process_vm_readv"
+                    ],
+                    "action": "SCMP_ACT_ERRNO",
+                    "errnoRet": 111,
+                    "args": [
+                        {
+                            "index": 0,
+                            "value": 20,
+                            "op": "SCMP_CMP_EQ"
+                        }
+                    ]
+                },
+                {
+                   "names": [
+                        "process_vm_readv"
+                    ],
+                    "action": "SCMP_ACT_ERRNO",
+                    "errnoRet": 222,
+                    "args": [
+                        {
+                            "index": 0,
+                            "value": 30,
+                            "op": "SCMP_CMP_EQ"
+                        },
+                        {
+                            "index": 2,
+                            "value": 40,
+                            "op": "SCMP_CMP_EQ"
+                        }
+                    ]
+                }
+            ]
+        }"#;
+
+        let mut scmp: oci::LinuxSeccomp = serde_json::from_str(data).unwrap();
+        let mut arch: Vec<oci::Arch>;
+
+        if cfg!(target_endian = "little") {
+            // For little-endian architectures
+            arch = vec![
+                "SCMP_ARCH_X86".to_string(),
+                "SCMP_ARCH_X32".to_string(),
+                "SCMP_ARCH_X86_64".to_string(),
+                "SCMP_ARCH_AARCH64".to_string(),
+                "SCMP_ARCH_ARM".to_string(),
+                "SCMP_ARCH_PPC64LE".to_string(),
+            ];
+        } else {
+            // For big-endian architectures
+            arch = vec!["SCMP_ARCH_S390X".to_string()];
+        }
+
+        scmp.architectures.append(&mut arch);
+
+        init_seccomp(&scmp).unwrap();
+
+        // Basic syscall with simple rule
+        syscall_assert!(unsafe { dup3(0, 1, O_CLOEXEC) }, -EPERM);
+
+        // Syscall with permitted arguments
+        syscall_assert!(unsafe { process_vm_readv(1, null(), 0, null(), 0, 0) }, 0);
+
+        // Multiple arguments with OR rules with ERRNO
+        syscall_assert!(
+            unsafe { process_vm_readv(10, null(), 0, null(), 0, 0) },
+            -111
+        );
+        syscall_assert!(
+            unsafe { process_vm_readv(20, null(), 0, null(), 0, 0) },
+            -111
+        );
+
+        // Multiple arguments with AND rules with ERRNO
+        syscall_assert!(unsafe { process_vm_readv(30, null(), 0, null(), 0, 0) }, 0);
+        syscall_assert!(
+            unsafe { process_vm_readv(30, null(), 40, null(), 0, 0) },
+            -222
+        );
+    }
+}

src/agent/samples/configuration-all-endpoints.toml

@@ -4,18 +4,38 @@ server_addr = 'vsock://8:2048'
 
 [endpoints]
 # All endpoints are allowed
-allowed = [ "CreateContainer", "StartContainer", "RemoveContainer",
-            "ExecProcess",  "SignalProcess", "WaitProcess",
-            "UpdateContainer", "StatsContainer", "PauseContainer", "ResumeContainer",
-            "WriteStdin", "ReadStdout", "ReadStderr", "CloseStdin", "TtyWinResize",
-            "UpdateInterface", "UpdateRoutes", "ListInterfaces", "ListRoutes", "AddARPNeighbors",
-            "StartTracing", "StopTracing", "GetMetrics",
-            "CreateSandbox", "DestroySandbox",
-            "OnlineCPUMem",
-            "ReseedRandomDev",
-            "GetGuestDetails",
-            "MemHotplugByProbe",
-            "SetGuestDateTime",
-            "CopyFile",
-            "GetOOMEvent",
-            "AddSwap"]
+allowed = [
+        "AddARPNeighborsRequest",
+        "AddSwapRequest",
+        "CloseStdinRequest",
+        "CopyFileRequest",
+        "CreateContainerRequest",
+        "CreateSandboxRequest",
+        "DestroySandboxRequest",
+        "ExecProcessRequest",
+        "GetMetricsRequest",
+        "GetOOMEventRequest",
+        "GuestDetailsRequest",
+        "ListInterfacesRequest",
+        "ListRoutesRequest",
+        "MemHotplugByProbeRequest",
+        "OnlineCPUMemRequest",
+        "PauseContainerRequest",
+        "PullImageRequest",
+        "ReadStreamRequest",
+        "RemoveContainerRequest",
+        "ReseedRandomDevRequest",
+        "ResumeContainerRequest",
+        "SetGuestDateTimeRequest",
+        "SignalProcessRequest",
+        "StartContainerRequest",
+        "StartTracingRequest",
+        "StatsContainerRequest",
+        "StopTracingRequest",
+        "TtyWinResizeRequest",
+        "UpdateContainerRequest",
+        "UpdateInterfaceRequest",
+        "UpdateRoutesRequest",
+        "WaitProcessRequest",
+        "WriteStreamRequest"
+]

src/agent/src/config.rs

@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 //
-use crate::tracer;
+use crate::rpc;
 use anyhow::{bail, ensure, Context, Result};
 use serde::Deserialize;
 use std::collections::HashSet;
@@ -33,7 +33,7 @@ const VSOCK_PORT: u16 = 1024;
 // Environment variables used for development and testing
 const SERVER_ADDR_ENV_VAR: &str = "KATA_AGENT_SERVER_ADDR";
 const LOG_LEVEL_ENV_VAR: &str = "KATA_AGENT_LOG_LEVEL";
-const TRACE_TYPE_ENV_VAR: &str = "KATA_AGENT_TRACE_TYPE";
+const TRACING_ENV_VAR: &str = "KATA_AGENT_TRACING";
 
 const ERR_INVALID_LOG_LEVEL: &str = "invalid log level";
 const ERR_INVALID_LOG_LEVEL_PARAM: &str = "invalid log level parameter";
@@ -73,8 +73,9 @@ pub struct AgentConfig {
     pub container_pipe_size: i32,
     pub server_addr: String,
     pub unified_cgroup_hierarchy: bool,
-    pub tracing: tracer::TraceType,
+    pub tracing: bool,
     pub endpoints: AgentEndpoints,
+    pub supports_seccomp: bool,
 }
 
 #[derive(Debug, Deserialize)]
@@ -88,7 +89,7 @@ pub struct AgentConfigBuilder {
     pub container_pipe_size: Option<i32>,
     pub server_addr: Option<String>,
     pub unified_cgroup_hierarchy: Option<bool>,
-    pub tracing: Option<tracer::TraceType>,
+    pub tracing: Option<bool>,
     pub endpoints: Option<EndpointsConfig>,
 }
 
@@ -148,8 +149,9 @@ impl Default for AgentConfig {
             container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
             server_addr: format!("{}:{}", VSOCK_ADDR, VSOCK_PORT),
             unified_cgroup_hierarchy: false,
-            tracing: tracer::TraceType::Disabled,
+            tracing: false,
             endpoints: Default::default(),
+            supports_seccomp: rpc::have_seccomp(),
         }
     }
 }
@@ -213,11 +215,11 @@ impl AgentConfig {
             // Support "bare" tracing option for backwards compatibility with
             // Kata 1.x.
             if param == &TRACE_MODE_OPTION {
-                config.tracing = tracer::TraceType::Isolated;
+                config.tracing = true;
                 continue;
             }
 
-            parse_cmdline_param!(param, TRACE_MODE_OPTION, config.tracing, get_trace_type);
+            parse_cmdline_param!(param, TRACE_MODE_OPTION, config.tracing, get_bool_value);
 
             // parse cmdline options
             parse_cmdline_param!(param, LOG_LEVEL_OPTION, config.log_level, get_log_level);
@@ -277,10 +279,10 @@ impl AgentConfig {
             }
         }
 
-        if let Ok(value) = env::var(TRACE_TYPE_ENV_VAR) {
-            if let Ok(result) = value.parse::<tracer::TraceType>() {
-                config.tracing = result;
-            }
+        if let Ok(value) = env::var(TRACING_ENV_VAR) {
+            let name_value = format!("{}={}", TRACING_ENV_VAR, value);
+
+            config.tracing = get_bool_value(&name_value)?;
         }
 
         // We did not get a configuration file: allow all endpoints.
@@ -343,25 +345,6 @@ fn get_log_level(param: &str) -> Result<slog::Level> {
     logrus_to_slog_level(fields[1])
 }
 
-#[instrument]
-fn get_trace_type(param: &str) -> Result<tracer::TraceType> {
-    ensure!(!param.is_empty(), "invalid trace type parameter");
-
-    let fields: Vec<&str> = param.split('=').collect();
-    ensure!(
-        fields[0] == TRACE_MODE_OPTION,
-        "invalid trace type key name"
-    );
-
-    if fields.len() == 1 {
-        return Ok(tracer::TraceType::Isolated);
-    }
-
-    let result = fields[1].parse::<tracer::TraceType>()?;
-
-    Ok(result)
-}
-
 #[instrument]
 fn get_hotplug_timeout(param: &str) -> Result<time::Duration> {
     let fields: Vec<&str> = param.split('=').collect();
@@ -446,10 +429,6 @@ mod tests {
     use std::time;
     use tempfile::tempdir;
 
-    const ERR_INVALID_TRACE_TYPE_PARAM: &str = "invalid trace type parameter";
-    const ERR_INVALID_TRACE_TYPE: &str = "invalid trace type";
-    const ERR_INVALID_TRACE_TYPE_KEY: &str = "invalid trace type key name";
-
     // Parameters:
     //
     // 1: expected Result
@@ -500,7 +479,7 @@ mod tests {
             container_pipe_size: i32,
             server_addr: &'a str,
             unified_cgroup_hierarchy: bool,
-            tracing: tracer::TraceType,
+            tracing: bool,
         }
 
         impl Default for TestData<'_> {
@@ -515,7 +494,7 @@ mod tests {
                     container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
                     server_addr: TEST_SERVER_ADDR,
                     unified_cgroup_hierarchy: false,
-                    tracing: tracer::TraceType::Disabled,
+                    tracing: false,
                 }
             }
         }
@@ -774,49 +753,115 @@ mod tests {
             },
             TestData {
                 contents: "trace",
-                tracing: tracer::TraceType::Disabled,
+                tracing: false,
                 ..Default::default()
             },
             TestData {
                 contents: ".trace",
-                tracing: tracer::TraceType::Disabled,
+                tracing: false,
                 ..Default::default()
             },
             TestData {
                 contents: "agent.tracer",
-                tracing: tracer::TraceType::Disabled,
+                tracing: false,
                 ..Default::default()
             },
             TestData {
                 contents: "agent.trac",
-                tracing: tracer::TraceType::Disabled,
+                tracing: false,
                 ..Default::default()
             },
             TestData {
                 contents: "agent.trace",
-                tracing: tracer::TraceType::Isolated,
+                tracing: true,
                 ..Default::default()
             },
             TestData {
-                contents: "agent.trace=isolated",
-                tracing: tracer::TraceType::Isolated,
+                contents: "agent.trace=true",
+                tracing: true,
                 ..Default::default()
             },
             TestData {
-                contents: "agent.trace=disabled",
-                tracing: tracer::TraceType::Disabled,
+                contents: "agent.trace=false",
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.trace=0",
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.trace=1",
+                tracing: true,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.trace=a",
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.trace=foo",
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.trace=.",
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.trace=,",
+                tracing: false,
                 ..Default::default()
             },
             TestData {
                 contents: "",
-                env_vars: vec!["KATA_AGENT_TRACE_TYPE=isolated"],
-                tracing: tracer::TraceType::Isolated,
+                env_vars: vec!["KATA_AGENT_TRACING="],
+                tracing: false,
                 ..Default::default()
             },
             TestData {
                 contents: "",
-                env_vars: vec!["KATA_AGENT_TRACE_TYPE=disabled"],
-                tracing: tracer::TraceType::Disabled,
+                env_vars: vec!["KATA_AGENT_TRACING=''"],
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_TRACING=0"],
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_TRACING=."],
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_TRACING=,"],
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_TRACING=foo"],
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_TRACING=1"],
+                tracing: true,
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_TRACING=true"],
+                tracing: true,
                 ..Default::default()
             },
         ];
@@ -1302,64 +1347,6 @@ Caused by:
         }
     }
 
-    #[test]
-    fn test_get_trace_type() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            param: &'a str,
-            result: Result<tracer::TraceType>,
-        }
-
-        let tests = &[
-            TestData {
-                param: "",
-                result: Err(anyhow!(ERR_INVALID_TRACE_TYPE_PARAM)),
-            },
-            TestData {
-                param: "agent.tracer",
-                result: Err(anyhow!(ERR_INVALID_TRACE_TYPE_KEY)),
-            },
-            TestData {
-                param: "agent.trac",
-                result: Err(anyhow!(ERR_INVALID_TRACE_TYPE_KEY)),
-            },
-            TestData {
-                param: "agent.trace=",
-                result: Err(anyhow!(ERR_INVALID_TRACE_TYPE)),
-            },
-            TestData {
-                param: "agent.trace==",
-                result: Err(anyhow!(ERR_INVALID_TRACE_TYPE)),
-            },
-            TestData {
-                param: "agent.trace=foo",
-                result: Err(anyhow!(ERR_INVALID_TRACE_TYPE)),
-            },
-            TestData {
-                param: "agent.trace",
-                result: Ok(tracer::TraceType::Isolated),
-            },
-            TestData {
-                param: "agent.trace=isolated",
-                result: Ok(tracer::TraceType::Isolated),
-            },
-            TestData {
-                param: "agent.trace=disabled",
-                result: Ok(tracer::TraceType::Disabled),
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let result = get_trace_type(d.param);
-
-            let msg = format!("{}: result: {:?}", msg, result);
-
-            assert_result!(d.result, result, msg);
-        }
-    }
-
     #[test]
     fn test_config_builder_from_string() {
         let config = AgentConfig::from_str(

src/agent/src/device.rs

@@ -7,7 +7,10 @@ use libc::{c_uint, major, minor};
 use nix::sys::stat;
 use regex::Regex;
 use std::collections::HashMap;
+use std::ffi::OsStr;
+use std::fmt;
 use std::fs;
+use std::os::unix::ffi::OsStrExt;
 use std::os::unix::fs::MetadataExt;
 use std::path::Path;
 use std::str::FromStr;
@@ -46,6 +49,9 @@ pub const DRIVER_LOCAL_TYPE: &str = "local";
 pub const DRIVER_WATCHABLE_BIND_TYPE: &str = "watchable-bind";
 // VFIO device to be bound to a guest kernel driver
 pub const DRIVER_VFIO_GK_TYPE: &str = "vfio-gk";
+// VFIO device to be bound to vfio-pci and made available inside the
+// container as a VFIO device node
+pub const DRIVER_VFIO_TYPE: &str = "vfio";
 
 #[derive(Debug)]
 struct DevIndexEntry {
@@ -62,6 +68,83 @@ pub fn online_device(path: &str) -> Result<()> {
     Ok(())
 }
 
+// Force a given PCI device to bind to the given driver, does
+// basically the same thing as
+//    driverctl set-override <PCI address> <driver>
+#[instrument]
+pub fn pci_driver_override<T, U>(syspci: T, dev: pci::Address, drv: U) -> Result<()>
+where
+    T: AsRef<OsStr> + std::fmt::Debug,
+    U: AsRef<OsStr> + std::fmt::Debug,
+{
+    let syspci = Path::new(&syspci);
+    let drv = drv.as_ref();
+    info!(sl!(), "rebind_pci_driver: {} => {:?}", dev, drv);
+
+    let devpath = syspci.join("devices").join(dev.to_string());
+    let overridepath = &devpath.join("driver_override");
+
+    fs::write(overridepath, drv.as_bytes())?;
+
+    let drvpath = &devpath.join("driver");
+    let need_unbind = match fs::read_link(drvpath) {
+        Ok(d) if d.file_name() == Some(drv) => return Ok(()), // Nothing to do
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => false, // No current driver
+        Err(e) => return Err(anyhow!("Error checking driver on {}: {}", dev, e)),
+        Ok(_) => true, // Current driver needs unbinding
+    };
+    if need_unbind {
+        let unbindpath = &drvpath.join("unbind");
+        fs::write(unbindpath, dev.to_string())?;
+    }
+    let probepath = syspci.join("drivers_probe");
+    fs::write(probepath, dev.to_string())?;
+    Ok(())
+}
+
+// Represents an IOMMU group
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct IommuGroup(u32);
+
+impl fmt::Display for IommuGroup {
+    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+        write!(f, "{}", self.0)
+    }
+}
+
+// Determine the IOMMU group of a PCI device
+#[instrument]
+fn pci_iommu_group<T>(syspci: T, dev: pci::Address) -> Result<Option<IommuGroup>>
+where
+    T: AsRef<OsStr> + std::fmt::Debug,
+{
+    let syspci = Path::new(&syspci);
+    let grouppath = syspci
+        .join("devices")
+        .join(dev.to_string())
+        .join("iommu_group");
+
+    match fs::read_link(&grouppath) {
+        // Device has no group
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
+        Err(e) => Err(anyhow!("Error reading link {:?}: {}", &grouppath, e)),
+        Ok(group) => {
+            if let Some(group) = group.file_name() {
+                if let Some(group) = group.to_str() {
+                    if let Ok(group) = group.parse::<u32>() {
+                        return Ok(Some(IommuGroup(group)));
+                    }
+                }
+            }
+            Err(anyhow!(
+                "Unexpected IOMMU group link {:?} => {:?}",
+                grouppath,
+                group
+            ))
+        }
+    }
+}
+
 // pcipath_to_sysfs fetches the sysfs path for a PCI path, relative to
 // the sysfs path for the PCI host bridge, based on the PCI path
 // provided.
@@ -296,6 +379,33 @@ pub async fn wait_for_pci_device(
     Ok(addr)
 }
 
+#[derive(Debug)]
+struct VfioMatcher {
+    syspath: String,
+}
+
+impl VfioMatcher {
+    fn new(grp: IommuGroup) -> VfioMatcher {
+        VfioMatcher {
+            syspath: format!("/devices/virtual/vfio/{}", grp),
+        }
+    }
+}
+
+impl UeventMatcher for VfioMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.devpath == self.syspath
+    }
+}
+
+#[instrument]
+async fn get_vfio_device_name(sandbox: &Arc<Mutex<Sandbox>>, grp: IommuGroup) -> Result<String> {
+    let matcher = VfioMatcher::new(grp);
+
+    let uev = wait_for_uevent(sandbox, matcher).await?;
+    Ok(format!("{}/{}", SYSTEM_DEV_PATH, &uev.devname))
+}
+
 /// Scan SCSI bus for the given SCSI address(SCSI-Id and LUN)
 #[instrument]
 fn scan_scsi_bus(scsi_addr: &str) -> Result<()> {
@@ -326,24 +436,27 @@ fn scan_scsi_bus(scsi_addr: &str) -> Result<()> {
     Ok(())
 }
 
-// update_spec_device_list takes a device description provided by the caller,
-// trying to find it on the guest. Once this device has been identified, the
-// "real" information that can be read from inside the VM is used to update
-// the same device in the list of devices provided through the OCI spec.
-// This is needed to update information about minor/major numbers that cannot
-// be predicted from the caller.
+// update_spec_device updates the device list in the OCI spec to make
+// it include details appropriate for the VM, instead of the host.  It
+// is given the host path to the device (to locate the device in the
+// original OCI spec) and the VM path which it uses to determine the
+// VM major/minor numbers, and the final path with which to present
+// the device in the (inner) container
 #[instrument]
-fn update_spec_device_list(device: &Device, spec: &mut Spec, devidx: &DevIndex) -> Result<()> {
+fn update_spec_device(
+    spec: &mut Spec,
+    devidx: &DevIndex,
+    host_path: &str,
+    vm_path: &str,
+    final_path: &str,
+) -> Result<()> {
     let major_id: c_uint;
     let minor_id: c_uint;
 
     // If no container_path is provided, we won't be able to match and
     // update the device in the OCI spec device list. This is an error.
-    if device.container_path.is_empty() {
-        return Err(anyhow!(
-            "container_path cannot empty for device {:?}",
-            device
-        ));
+    if host_path.is_empty() {
+        return Err(anyhow!("Host path cannot empty for device"));
     }
 
     let linux = spec
@@ -351,11 +464,11 @@ fn update_spec_device_list(device: &Device, spec: &mut Spec, devidx: &DevIndex)
         .as_mut()
         .ok_or_else(|| anyhow!("Spec didn't container linux field"))?;
 
-    if !Path::new(&device.vm_path).exists() {
-        return Err(anyhow!("vm_path:{} doesn't exist", device.vm_path));
+    if !Path::new(vm_path).exists() {
+        return Err(anyhow!("vm_path:{} doesn't exist", vm_path));
     }
 
-    let meta = fs::metadata(&device.vm_path)?;
+    let meta = fs::metadata(vm_path)?;
     let dev_id = meta.rdev();
     unsafe {
         major_id = major(dev_id);
@@ -364,24 +477,27 @@ fn update_spec_device_list(device: &Device, spec: &mut Spec, devidx: &DevIndex)
 
     info!(
         sl!(),
-        "got the device: dev_path: {}, major: {}, minor: {}\n", &device.vm_path, major_id, minor_id
+        "update_spec_device(): vm_path={}, major: {}, minor: {}\n", vm_path, major_id, minor_id
     );
 
-    if let Some(idxdata) = devidx.0.get(device.container_path.as_str()) {
+    if let Some(idxdata) = devidx.0.get(host_path) {
         let dev = &mut linux.devices[idxdata.idx];
         let host_major = dev.major;
         let host_minor = dev.minor;
 
         dev.major = major_id as i64;
         dev.minor = minor_id as i64;
+        dev.path = final_path.to_string();
 
         info!(
             sl!(),
-            "change the device from major: {} minor: {} to vm device major: {} minor: {}",
+            "change the device from path: {} major: {} minor: {} to vm device path: {} major: {} minor: {}",
+            host_path,
             host_major,
             host_minor,
-            major_id,
-            minor_id
+            dev.path,
+            dev.major,
+            dev.minor,
         );
 
         // Resources must be updated since they are used to identify
@@ -402,7 +518,7 @@ fn update_spec_device_list(device: &Device, spec: &mut Spec, devidx: &DevIndex)
     } else {
         Err(anyhow!(
             "Should have found a matching device {} in the spec",
-            device.vm_path
+            vm_path
         ))
     }
 }
@@ -420,7 +536,13 @@ async fn virtiommio_blk_device_handler(
         return Err(anyhow!("Invalid path for virtio mmio blk device"));
     }
 
-    update_spec_device_list(device, spec, devidx)
+    update_spec_device(
+        spec,
+        devidx,
+        &device.container_path,
+        &device.vm_path,
+        &device.container_path,
+    )
 }
 
 // device.Id should be a PCI path string
@@ -436,7 +558,13 @@ async fn virtio_blk_device_handler(
 
     dev.vm_path = get_virtio_blk_pci_device_name(sandbox, &pcipath).await?;
 
-    update_spec_device_list(&dev, spec, devidx)
+    update_spec_device(
+        spec,
+        devidx,
+        &dev.container_path,
+        &dev.vm_path,
+        &dev.container_path,
+    )
 }
 
 // device.id should be a CCW path string
@@ -451,7 +579,13 @@ async fn virtio_blk_ccw_device_handler(
     let mut dev = device.clone();
     let ccw_device = ccw::Device::from_str(&device.id)?;
     dev.vm_path = get_virtio_blk_ccw_device_name(sandbox, &ccw_device).await?;
-    update_spec_device_list(&dev, spec, devidx)
+    update_spec_device(
+        spec,
+        devidx,
+        &dev.container_path,
+        &dev.vm_path,
+        &dev.container_path,
+    )
 }
 
 #[cfg(not(target_arch = "s390x"))]
@@ -475,7 +609,13 @@ async fn virtio_scsi_device_handler(
 ) -> Result<()> {
     let mut dev = device.clone();
     dev.vm_path = get_scsi_device_name(sandbox, &device.id).await?;
-    update_spec_device_list(&dev, spec, devidx)
+    update_spec_device(
+        spec,
+        devidx,
+        &dev.container_path,
+        &dev.vm_path,
+        &dev.container_path,
+    )
 }
 
 #[instrument]
@@ -489,7 +629,13 @@ async fn virtio_nvdimm_device_handler(
         return Err(anyhow!("Invalid path for nvdimm device"));
     }
 
-    update_spec_device_list(device, spec, devidx)
+    update_spec_device(
+        spec,
+        devidx,
+        &device.container_path,
+        &device.vm_path,
+        &device.container_path,
+    )
 }
 
 fn split_vfio_option(opt: &str) -> Option<(&str, &str)> {
@@ -507,19 +653,54 @@ fn split_vfio_option(opt: &str) -> Option<(&str, &str)> {
 // Each option should have the form "DDDD:BB:DD.F=<pcipath>"
 //     DDDD:BB:DD.F is the device's PCI address in the host
 //     <pcipath> is a PCI path to the device in the guest (see pci.rs)
-async fn vfio_gk_device_handler(
+async fn vfio_device_handler(
     device: &Device,
-    _: &mut Spec,
+    spec: &mut Spec,
     sandbox: &Arc<Mutex<Sandbox>>,
-    _: &DevIndex,
+    devidx: &DevIndex,
 ) -> Result<()> {
+    let vfio_in_guest = device.field_type != DRIVER_VFIO_GK_TYPE;
+    let mut group = None;
+
     for opt in device.options.iter() {
         let (_, pcipath) =
             split_vfio_option(opt).ok_or_else(|| anyhow!("Malformed VFIO option {:?}", opt))?;
         let pcipath = pci::Path::from_str(pcipath)?;
 
-        wait_for_pci_device(sandbox, &pcipath).await?;
+        let guestdev = wait_for_pci_device(sandbox, &pcipath).await?;
+        if vfio_in_guest {
+            pci_driver_override(SYSFS_BUS_PCI_PATH, guestdev, "vfio-pci")?;
+
+            let devgroup = pci_iommu_group(SYSFS_BUS_PCI_PATH, guestdev)?;
+            if devgroup.is_none() {
+                // Devices must have an IOMMU group to be usable via VFIO
+                return Err(anyhow!("{} has no IOMMU group", guestdev));
+            }
+
+            if group.is_some() && group != devgroup {
+                // If PCI devices associated with the same VFIO device
+                // (and therefore group) in the host don't end up in
+                // the same group in the guest, something has gone
+                // horribly wrong
+                return Err(anyhow!(
+                    "{} is not in guest IOMMU group {}",
+                    guestdev,
+                    group.unwrap()
+                ));
+            }
+
+            group = devgroup;
+        }
     }
+
+    if vfio_in_guest {
+        // If there are any devices at all, logic above ensures that group is not None
+        let group = group.unwrap();
+        let vmpath = get_vfio_device_name(sandbox, group).await?;
+
+        update_spec_device(spec, devidx, &device.container_path, &vmpath, &vmpath)?;
+    }
+
     Ok(())
 }
 
@@ -592,7 +773,9 @@ async fn add_device(
         DRIVER_MMIO_BLK_TYPE => virtiommio_blk_device_handler(device, spec, sandbox, devidx).await,
         DRIVER_NVDIMM_TYPE => virtio_nvdimm_device_handler(device, spec, sandbox, devidx).await,
         DRIVER_SCSI_TYPE => virtio_scsi_device_handler(device, spec, sandbox, devidx).await,
-        DRIVER_VFIO_GK_TYPE => vfio_gk_device_handler(device, spec, sandbox, devidx).await,
+        DRIVER_VFIO_GK_TYPE | DRIVER_VFIO_TYPE => {
+            vfio_device_handler(device, spec, sandbox, devidx).await
+        }
         _ => Err(anyhow!("Unknown device type {}", device.field_type)),
     }
 }
@@ -657,28 +840,28 @@ mod tests {
     }
 
     #[test]
-    fn test_update_spec_device_list() {
+    fn test_update_spec_device() {
         let (major, minor) = (7, 2);
-        let mut device = Device::default();
         let mut spec = Spec::default();
 
         // container_path empty
+        let container_path = "";
+        let vm_path = "";
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_err());
 
-        device.container_path = "/dev/null".to_string();
-
         // linux is empty
+        let container_path = "/dev/null";
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_err());
 
         spec.linux = Some(Linux::default());
 
         // linux.devices is empty
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_err());
 
         spec.linux.as_mut().unwrap().devices = vec![oci::LinuxDevice {
@@ -690,26 +873,32 @@ mod tests {
 
         // vm_path empty
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_err());
 
-        device.vm_path = "/dev/null".to_string();
+        let vm_path = "/dev/null";
 
         // guest and host path are not the same
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
-        assert!(res.is_err(), "device={:?} spec={:?}", device, spec);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
+        assert!(
+            res.is_err(),
+            "container_path={:?} vm_path={:?} spec={:?}",
+            container_path,
+            vm_path,
+            spec
+        );
 
-        spec.linux.as_mut().unwrap().devices[0].path = device.container_path.clone();
+        spec.linux.as_mut().unwrap().devices[0].path = container_path.to_string();
 
         // spec.linux.resources is empty
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_ok());
 
         // update both devices and cgroup lists
         spec.linux.as_mut().unwrap().devices = vec![oci::LinuxDevice {
-            path: device.container_path.clone(),
+            path: container_path.to_string(),
             major,
             minor,
             ..oci::LinuxDevice::default()
@@ -725,12 +914,12 @@ mod tests {
         });
 
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_ok());
     }
 
     #[test]
-    fn test_update_spec_device_list_guest_host_conflict() {
+    fn test_update_spec_device_guest_host_conflict() {
         let null_rdev = fs::metadata("/dev/null").unwrap().rdev();
         let zero_rdev = fs::metadata("/dev/zero").unwrap().rdev();
         let full_rdev = fs::metadata("/dev/full").unwrap().rdev();
@@ -781,20 +970,14 @@ mod tests {
         };
         let devidx = DevIndex::new(&spec);
 
-        let dev_a = Device {
-            container_path: "/dev/a".to_string(),
-            vm_path: "/dev/zero".to_string(),
-            ..Device::default()
-        };
+        let container_path_a = "/dev/a";
+        let vm_path_a = "/dev/zero";
 
         let guest_major_a = stat::major(zero_rdev) as i64;
         let guest_minor_a = stat::minor(zero_rdev) as i64;
 
-        let dev_b = Device {
-            container_path: "/dev/b".to_string(),
-            vm_path: "/dev/full".to_string(),
-            ..Device::default()
-        };
+        let container_path_b = "/dev/b";
+        let vm_path_b = "/dev/full";
 
         let guest_major_b = stat::major(full_rdev) as i64;
         let guest_minor_b = stat::minor(full_rdev) as i64;
@@ -811,7 +994,13 @@ mod tests {
         assert_eq!(Some(host_major_b), specresources.devices[1].major);
         assert_eq!(Some(host_minor_b), specresources.devices[1].minor);
 
-        let res = update_spec_device_list(&dev_a, &mut spec, &devidx);
+        let res = update_spec_device(
+            &mut spec,
+            &devidx,
+            container_path_a,
+            vm_path_a,
+            container_path_a,
+        );
         assert!(res.is_ok());
 
         let specdevices = &spec.linux.as_ref().unwrap().devices;
@@ -826,7 +1015,13 @@ mod tests {
         assert_eq!(Some(host_major_b), specresources.devices[1].major);
         assert_eq!(Some(host_minor_b), specresources.devices[1].minor);
 
-        let res = update_spec_device_list(&dev_b, &mut spec, &devidx);
+        let res = update_spec_device(
+            &mut spec,
+            &devidx,
+            container_path_b,
+            vm_path_b,
+            container_path_b,
+        );
         assert!(res.is_ok());
 
         let specdevices = &spec.linux.as_ref().unwrap().devices;
@@ -843,7 +1038,7 @@ mod tests {
     }
 
     #[test]
-    fn test_update_spec_device_list_char_block_conflict() {
+    fn test_update_spec_device_char_block_conflict() {
         let null_rdev = fs::metadata("/dev/null").unwrap().rdev();
 
         let guest_major = stat::major(null_rdev) as i64;
@@ -892,11 +1087,8 @@ mod tests {
         };
         let devidx = DevIndex::new(&spec);
 
-        let dev = Device {
-            container_path: "/dev/char".to_string(),
-            vm_path: "/dev/null".to_string(),
-            ..Device::default()
-        };
+        let container_path = "/dev/char";
+        let vm_path = "/dev/null";
 
         let specresources = spec.linux.as_ref().unwrap().resources.as_ref().unwrap();
         assert_eq!(Some(host_major), specresources.devices[0].major);
@@ -904,7 +1096,7 @@ mod tests {
         assert_eq!(Some(host_major), specresources.devices[1].major);
         assert_eq!(Some(host_minor), specresources.devices[1].minor);
 
-        let res = update_spec_device_list(&dev, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_ok());
 
         // Only the char device, not the block device should be updated
@@ -915,6 +1107,43 @@ mod tests {
         assert_eq!(Some(host_minor), specresources.devices[1].minor);
     }
 
+    #[test]
+    fn test_update_spec_device_final_path() {
+        let null_rdev = fs::metadata("/dev/null").unwrap().rdev();
+        let guest_major = stat::major(null_rdev) as i64;
+        let guest_minor = stat::minor(null_rdev) as i64;
+
+        let host_path = "/dev/host";
+        let host_major: i64 = 99;
+        let host_minor: i64 = 99;
+
+        let mut spec = Spec {
+            linux: Some(Linux {
+                devices: vec![oci::LinuxDevice {
+                    path: host_path.to_string(),
+                    r#type: "c".to_string(),
+                    major: host_major,
+                    minor: host_minor,
+                    ..oci::LinuxDevice::default()
+                }],
+                ..Linux::default()
+            }),
+            ..Spec::default()
+        };
+        let devidx = DevIndex::new(&spec);
+
+        let vm_path = "/dev/null";
+        let final_path = "/dev/final";
+
+        let res = update_spec_device(&mut spec, &devidx, host_path, vm_path, final_path);
+        assert!(res.is_ok());
+
+        let specdevices = &spec.linux.as_ref().unwrap().devices;
+        assert_eq!(guest_major, specdevices[0].major);
+        assert_eq!(guest_minor, specdevices[0].minor);
+        assert_eq!(final_path, specdevices[0].path);
+    }
+
     #[test]
     fn test_pcipath_to_sysfs() {
         let testdir = tempdir().expect("failed to create tmpdir");
@@ -1142,6 +1371,27 @@ mod tests {
         assert!(!matcher_a.is_match(&uev_b));
     }
 
+    #[tokio::test]
+    async fn test_vfio_matcher() {
+        let grpa = IommuGroup(1);
+        let grpb = IommuGroup(22);
+
+        let mut uev_a = crate::uevent::Uevent::default();
+        uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
+        uev_a.devname = format!("vfio/{}", grpa);
+        uev_a.devpath = format!("/devices/virtual/vfio/{}", grpa);
+        let matcher_a = VfioMatcher::new(grpa);
+
+        let mut uev_b = uev_a.clone();
+        uev_b.devpath = format!("/devices/virtual/vfio/{}", grpb);
+        let matcher_b = VfioMatcher::new(grpb);
+
+        assert!(matcher_a.is_match(&uev_a));
+        assert!(matcher_b.is_match(&uev_b));
+        assert!(!matcher_b.is_match(&uev_a));
+        assert!(!matcher_a.is_match(&uev_b));
+    }
+
     #[test]
     fn test_split_vfio_option() {
         assert_eq!(
@@ -1151,4 +1401,81 @@ mod tests {
         assert_eq!(split_vfio_option("0000:01:00.0=02/01=rubbish"), None);
         assert_eq!(split_vfio_option("0000:01:00.0"), None);
     }
+
+    #[test]
+    fn test_pci_driver_override() {
+        let testdir = tempdir().expect("failed to create tmpdir");
+        let syspci = testdir.path(); // Path to mock /sys/bus/pci
+
+        let dev0 = pci::Address::new(0, 0, pci::SlotFn::new(0, 0).unwrap());
+        let dev0path = syspci.join("devices").join(dev0.to_string());
+        let dev0drv = dev0path.join("driver");
+        let dev0override = dev0path.join("driver_override");
+
+        let drvapath = syspci.join("drivers").join("drv_a");
+        let drvaunbind = drvapath.join("unbind");
+
+        let probepath = syspci.join("drivers_probe");
+
+        // Start mocking dev0 as being unbound
+        fs::create_dir_all(&dev0path).unwrap();
+
+        pci_driver_override(syspci, dev0, "drv_a").unwrap();
+        assert_eq!(fs::read_to_string(&dev0override).unwrap(), "drv_a");
+        assert_eq!(fs::read_to_string(&probepath).unwrap(), dev0.to_string());
+
+        // Now mock dev0 already being attached to drv_a
+        fs::create_dir_all(&drvapath).unwrap();
+        std::os::unix::fs::symlink(&drvapath, dev0drv).unwrap();
+        std::fs::remove_file(&probepath).unwrap();
+
+        pci_driver_override(syspci, dev0, "drv_a").unwrap(); // no-op
+        assert_eq!(fs::read_to_string(&dev0override).unwrap(), "drv_a");
+        assert!(!probepath.exists());
+
+        // Now try binding to a different driver
+        pci_driver_override(syspci, dev0, "drv_b").unwrap();
+        assert_eq!(fs::read_to_string(&dev0override).unwrap(), "drv_b");
+        assert_eq!(fs::read_to_string(&probepath).unwrap(), dev0.to_string());
+        assert_eq!(fs::read_to_string(&drvaunbind).unwrap(), dev0.to_string());
+    }
+
+    #[test]
+    fn test_pci_iommu_group() {
+        let testdir = tempdir().expect("failed to create tmpdir"); // mock /sys
+        let syspci = testdir.path().join("bus").join("pci");
+
+        // Mock dev0, which has no group
+        let dev0 = pci::Address::new(0, 0, pci::SlotFn::new(0, 0).unwrap());
+        let dev0path = syspci.join("devices").join(dev0.to_string());
+
+        fs::create_dir_all(&dev0path).unwrap();
+
+        // Test dev0
+        assert!(pci_iommu_group(&syspci, dev0).unwrap().is_none());
+
+        // Mock dev1, which is in group 12
+        let dev1 = pci::Address::new(0, 1, pci::SlotFn::new(0, 0).unwrap());
+        let dev1path = syspci.join("devices").join(dev1.to_string());
+        let dev1group = dev1path.join("iommu_group");
+
+        fs::create_dir_all(&dev1path).unwrap();
+        std::os::unix::fs::symlink("../../../kernel/iommu_groups/12", &dev1group).unwrap();
+
+        // Test dev1
+        assert_eq!(
+            pci_iommu_group(&syspci, dev1).unwrap(),
+            Some(IommuGroup(12))
+        );
+
+        // Mock dev2, which has a bogus group (dir instead of symlink)
+        let dev2 = pci::Address::new(0, 2, pci::SlotFn::new(0, 0).unwrap());
+        let dev2path = syspci.join("devices").join(dev2.to_string());
+        let dev2group = dev2path.join("iommu_group");
+
+        fs::create_dir_all(&dev2group).unwrap();
+
+        // Test dev2
+        assert!(pci_iommu_group(&syspci, dev2).is_err());
+    }
 }

src/agent/src/linux_abi.rs

@@ -83,6 +83,8 @@ pub const SYSFS_MEMORY_ONLINE_PATH: &str = "/sys/devices/system/memory";
 
 pub const SYSFS_SCSI_HOST_PATH: &str = "/sys/class/scsi_host";
 
+pub const SYSFS_BUS_PCI_PATH: &str = "/sys/bus/pci";
+
 pub const SYSFS_CGROUPPATH: &str = "/sys/fs/cgroup";
 pub const SYSFS_ONLINE_FILE: &str = "online";
 
@@ -94,6 +96,7 @@ pub const SYSTEM_DEV_PATH: &str = "/dev";
 // Linux UEvent related consts.
 pub const U_EVENT_ACTION: &str = "ACTION";
 pub const U_EVENT_ACTION_ADD: &str = "add";
+pub const U_EVENT_ACTION_REMOVE: &str = "remove";
 pub const U_EVENT_DEV_PATH: &str = "DEVPATH";
 pub const U_EVENT_SUB_SYSTEM: &str = "SUBSYSTEM";
 pub const U_EVENT_SEQ_NUM: &str = "SEQNUM";

src/agent/src/main.rs

@@ -196,8 +196,8 @@ async fn real_main() -> std::result::Result<(), Box<dyn std::error::Error>> {
         ttrpc_log_guard = Ok(slog_stdlog::init().map_err(|e| e)?);
     }
 
-    if config.tracing != tracer::TraceType::Disabled {
-        let _ = tracer::setup_tracing(NAME, &logger, &config)?;
+    if config.tracing {
+        tracer::setup_tracing(NAME, &logger)?;
     }
 
     let root_span = span!(tracing::Level::TRACE, "root-span");
@@ -229,23 +229,29 @@ async fn real_main() -> std::result::Result<(), Box<dyn std::error::Error>> {
     // Wait for all threads to finish
     let results = join_all(tasks).await;
 
-    for result in results {
-        if let Err(e) = result {
-            return Err(anyhow!(e).into());
-        }
-    }
-
     // force flushing spans
     drop(span_guard);
     drop(root_span);
 
-    if config.tracing != tracer::TraceType::Disabled {
+    if config.tracing {
         tracer::end_tracing();
     }
 
     eprintln!("{} shutdown complete", NAME);
 
-    Ok(())
+    let mut wait_errors: Vec<tokio::task::JoinError> = vec![];
+    for result in results {
+        if let Err(e) = result {
+            eprintln!("wait task error: {:#?}", e);
+            wait_errors.push(e);
+        }
+    }
+
+    if wait_errors.is_empty() {
+        Ok(())
+    } else {
+        Err(anyhow!("wait all tasks failed: {:#?}", wait_errors).into())
+    }
 }
 
 fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {

src/agent/src/netlink.rs

@@ -6,6 +6,7 @@
 use anyhow::{anyhow, Context, Result};
 use futures::{future, StreamExt, TryStreamExt};
 use ipnetwork::{IpNetwork, Ipv4Network, Ipv6Network};
+use nix::errno::Errno;
 use protobuf::RepeatedField;
 use protocols::types::{ARPNeighbor, IPAddress, IPFamily, Interface, Route};
 use rtnetlink::{new_connection, packet, IpVersion};
@@ -363,14 +364,17 @@ impl Handle {
                     request = request.gateway(ip);
                 }
 
-                request.execute().await.with_context(|| {
-                    format!(
-                        "Failed to add IP v6 route (src: {}, dst: {}, gtw: {})",
-                        route.get_source(),
-                        route.get_dest(),
-                        route.get_gateway()
-                    )
-                })?;
+                if let Err(rtnetlink::Error::NetlinkError(message)) = request.execute().await {
+                    if Errno::from_i32(message.code.abs()) != Errno::EEXIST {
+                        return Err(anyhow!(
+                            "Failed to add IP v6 route (src: {}, dst: {}, gtw: {},Err: {})",
+                            route.get_source(),
+                            route.get_dest(),
+                            route.get_gateway(),
+                            message
+                        ));
+                    }
+                }
             } else {
                 let dest_addr = if !route.dest.is_empty() {
                     Ipv4Network::from_str(&route.dest)?
@@ -401,7 +405,17 @@ impl Handle {
                     request = request.gateway(ip);
                 }
 
-                request.execute().await?;
+                if let Err(rtnetlink::Error::NetlinkError(message)) = request.execute().await {
+                    if Errno::from_i32(message.code.abs()) != Errno::EEXIST {
+                        return Err(anyhow!(
+                            "Failed to add IP v4 route (src: {}, dst: {}, gtw: {},Err: {})",
+                            route.get_source(),
+                            route.get_dest(),
+                            route.get_gateway(),
+                            message
+                        ));
+                    }
+                }
             }
         }
 

src/agent/src/rpc.rs

@@ -433,7 +433,7 @@ impl AgentService {
             .get_container(&cid)
             .ok_or_else(|| anyhow!("Invalid container id"))?;
 
-        let mut p = match ctr.processes.get_mut(&pid) {
+        let p = match ctr.processes.get_mut(&pid) {
             Some(p) => p,
             None => {
                 // Lost race, pick up exit code from channel
@@ -444,7 +444,7 @@ impl AgentService {
 
         // need to close all fd
         // ignore errors for some fd might be closed by stream
-        let _ = cleanup_process(&mut p);
+        p.cleanup_process_stream();
 
         resp.status = p.exit_code;
         // broadcast exit code to all parallel watchers
@@ -776,19 +776,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
             )
         })?;
 
-        if p.term_master.is_some() {
-            p.close_stream(StreamType::TermMaster);
-            let _ = unistd::close(p.term_master.unwrap());
-            p.term_master = None;
-        }
-
-        if p.parent_stdin.is_some() {
-            p.close_stream(StreamType::ParentStdin);
-            let _ = unistd::close(p.parent_stdin.unwrap());
-            p.parent_stdin = None;
-        }
-
-        p.notify_term_close();
+        p.close_stdin();
 
         Ok(Empty::new())
     }
@@ -1334,11 +1322,19 @@ fn get_memory_info(block_size: bool, hotplug: bool) -> Result<(u64, bool)> {
     Ok((size, plug))
 }
 
+pub fn have_seccomp() -> bool {
+    if cfg!(feature = "seccomp") {
+        return true;
+    }
+
+    false
+}
+
 fn get_agent_details() -> AgentDetails {
     let mut detail = AgentDetails::new();
 
     detail.set_version(AGENT_VERSION.to_string());
-    detail.set_supports_seccomp(false);
+    detail.set_supports_seccomp(have_seccomp());
     detail.init_daemon = unistd::getpid() == Pid::from_raw(1);
 
     detail.device_handlers = RepeatedField::new();
@@ -1653,11 +1649,6 @@ fn setup_bundle(cid: &str, spec: &mut Spec) -> Result<PathBuf> {
         readonly: spec_root.readonly,
     });
 
-    info!(
-        sl!(),
-        "{:?}",
-        spec.process.as_ref().unwrap().console_size.as_ref()
-    );
     let _ = spec.save(config_path.to_str().unwrap());
 
     let olddir = unistd::getcwd().context("cannot getcwd")?;
@@ -1666,37 +1657,6 @@ fn setup_bundle(cid: &str, spec: &mut Spec) -> Result<PathBuf> {
     Ok(olddir)
 }
 
-fn cleanup_process(p: &mut Process) -> Result<()> {
-    if p.parent_stdin.is_some() {
-        p.close_stream(StreamType::ParentStdin);
-        unistd::close(p.parent_stdin.unwrap())?;
-    }
-
-    if p.parent_stdout.is_some() {
-        p.close_stream(StreamType::ParentStdout);
-        unistd::close(p.parent_stdout.unwrap())?;
-    }
-
-    if p.parent_stderr.is_some() {
-        p.close_stream(StreamType::ParentStderr);
-        unistd::close(p.parent_stderr.unwrap())?;
-    }
-
-    if p.term_master.is_some() {
-        p.close_stream(StreamType::TermMaster);
-        unistd::close(p.term_master.unwrap())?;
-    }
-
-    p.notify_term_close();
-
-    p.parent_stdin = None;
-    p.parent_stdout = None;
-    p.parent_stderr = None;
-    p.term_master = None;
-
-    Ok(())
-}
-
 fn load_kernel_module(module: &protocols::agent::KernelModule) -> Result<()> {
     if module.name.is_empty() {
         return Err(anyhow!("Kernel module name is empty"));

src/agent/src/sandbox.rs

@@ -464,7 +464,10 @@ mod tests {
         baremount(src, dst, "bind", MsFlags::MS_BIND, "", logger)
     }
 
+    use serial_test::serial;
+
     #[tokio::test]
+    #[serial]
     async fn set_sandbox_storage() {
         let logger = slog::Logger::root(slog::Discard, o!());
         let mut s = Sandbox::new(&logger).unwrap();
@@ -499,6 +502,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn remove_sandbox_storage() {
         skip_if_not_root!();
 
@@ -555,6 +559,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn unset_and_remove_sandbox_storage() {
         skip_if_not_root!();
 
@@ -606,6 +611,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn unset_sandbox_storage() {
         let logger = slog::Logger::root(slog::Discard, o!());
         let mut s = Sandbox::new(&logger).unwrap();
@@ -689,6 +695,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn get_container_entry_exist() {
         skip_if_not_root!();
         let logger = slog::Logger::root(slog::Discard, o!());
@@ -702,6 +709,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn get_container_no_entry() {
         let logger = slog::Logger::root(slog::Discard, o!());
         let mut s = Sandbox::new(&logger).unwrap();
@@ -711,6 +719,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn add_and_get_container() {
         skip_if_not_root!();
         let logger = slog::Logger::root(slog::Discard, o!());
@@ -722,6 +731,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn update_shared_pidns() {
         skip_if_not_root!();
         let logger = slog::Logger::root(slog::Discard, o!());
@@ -740,6 +750,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn add_guest_hooks() {
         let logger = slog::Logger::root(slog::Discard, o!());
         let mut s = Sandbox::new(&logger).unwrap();
@@ -763,6 +774,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn test_sandbox_set_destroy() {
         let logger = slog::Logger::root(slog::Discard, o!());
         let mut s = Sandbox::new(&logger).unwrap();

src/agent/src/tracer.rs

@@ -3,61 +3,17 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use crate::config::AgentConfig;
 use anyhow::Result;
 use opentelemetry::sdk::propagation::TraceContextPropagator;
 use opentelemetry::{global, sdk::trace::Config, trace::TracerProvider};
-use serde::Deserialize;
 use slog::{info, o, Logger};
 use std::collections::HashMap;
-use std::error::Error;
-use std::fmt;
-use std::str::FromStr;
 use tracing_opentelemetry::OpenTelemetryLayer;
 use tracing_subscriber::layer::SubscriberExt;
 use tracing_subscriber::Registry;
 use ttrpc::r#async::TtrpcContext;
 
-#[derive(Debug, Deserialize, PartialEq)]
-pub enum TraceType {
-    Disabled,
-    Isolated,
-}
-
-#[derive(Debug)]
-pub struct TraceTypeError {
-    details: String,
-}
-
-impl TraceTypeError {
-    fn new(msg: &str) -> TraceTypeError {
-        TraceTypeError {
-            details: msg.into(),
-        }
-    }
-}
-
-impl Error for TraceTypeError {}
-
-impl fmt::Display for TraceTypeError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}", self.details)
-    }
-}
-
-impl FromStr for TraceType {
-    type Err = TraceTypeError;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        match s {
-            "isolated" => Ok(TraceType::Isolated),
-            "disabled" => Ok(TraceType::Disabled),
-            _ => Err(TraceTypeError::new("invalid trace type")),
-        }
-    }
-}
-
-pub fn setup_tracing(name: &'static str, logger: &Logger, _agent_cfg: &AgentConfig) -> Result<()> {
+pub fn setup_tracing(name: &'static str, logger: &Logger) -> Result<()> {
     let logger = logger.new(o!("subsystem" => "vsock-tracer"));
 
     let exporter = vsock_exporter::Exporter::builder()

src/agent/src/uevent.rs

@@ -97,10 +97,18 @@ impl Uevent {
         })
     }
 
+    #[instrument]
+    async fn process_remove(&self, logger: &Logger, sandbox: &Arc<Mutex<Sandbox>>) {
+        let mut sb = sandbox.lock().await;
+        sb.uevent_map.remove(&self.devpath);
+    }
+
     #[instrument]
     async fn process(&self, logger: &Logger, sandbox: &Arc<Mutex<Sandbox>>) {
         if self.action == U_EVENT_ACTION_ADD {
             return self.process_add(logger, sandbox).await;
+        } else if self.action == U_EVENT_ACTION_REMOVE {
+            return self.process_remove(logger, sandbox).await;
         }
         debug!(*logger, "ignoring event"; "uevent" => format!("{:?}", self));
     }

src/agent/src/watcher.rs

@@ -979,7 +979,10 @@ mod tests {
         );
     }
 
+    use serial_test::serial;
+
     #[tokio::test]
+    #[serial]
     async fn create_tmpfs() {
         skip_if_not_root!();
 
@@ -994,6 +997,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn spawn_thread() {
         skip_if_not_root!();
 
@@ -1023,6 +1027,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn verify_container_cleanup_watching() {
         skip_if_not_root!();
 

src/agent/vsock-exporter/Cargo.toml

@@ -15,6 +15,6 @@ serde = { version = "1.0.126", features = ["derive"] }
 tokio-vsock = "0.3.1"
 bincode = "1.3.3"
 byteorder = "1.4.3"
-slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_info"] }
+slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug"] }
 async-trait = "0.1.50"
 tokio = "1.2.0"

src/runtime/Makefile

@@ -190,6 +190,7 @@ DEFVALIDVHOSTUSERSTOREPATHS := [\"$(DEFVHOSTUSERSTOREPATH)\"]
 DEFFILEMEMBACKEND := ""
 DEFVALIDFILEMEMBACKENDS := [\"$(DEFFILEMEMBACKEND)\"]
 DEFMSIZE9P := 8192
+DEFVFIOMODE := guest-kernel
 
 # Default cgroup model
 DEFSANDBOXCGROUPONLY ?= false
@@ -459,6 +460,7 @@ USER_VARS += DEFENTROPYSOURCE
 USER_VARS += DEFVALIDENTROPYSOURCES
 USER_VARS += DEFSANDBOXCGROUPONLY
 USER_VARS += DEFBINDMOUNTS
+USER_VARS += DEFVFIOMODE
 USER_VARS += FEATURE_SELINUX
 USER_VARS += BUILDFLAGS
 
@@ -531,7 +533,7 @@ $(NETMON_RUNTIME_OUTPUT): $(SOURCES) VERSION
 runtime: $(RUNTIME_OUTPUT) $(CONFIGS)
 .DEFAULT: default
 
-build: default
+build: all
 
 #Install an executable file
 # params:
@@ -573,10 +575,11 @@ $(MONITOR_OUTPUT): $(SOURCES) $(GENERATED_FILES) $(MAKEFILE_LIST) .git-commit
 
 .PHONY: \
 	check \
-	check-go-static \
 	coverage \
 	default \
 	install \
+	lint \
+	pre-commit \
 	show-header \
 	show-summary \
 	show-variables \
@@ -595,8 +598,6 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit
 
 generate-config: $(CONFIGS)
 
-check: check-go-static
-
 test: install-hook go-test
 
 install-hook:
@@ -610,15 +611,34 @@ go-test: $(GENERATED_FILES)
 	go clean -testcache
 	go test -v -mod=vendor ./...
 
-check-go-static:
-	$(QUIET_CHECK)../../ci/go-no-os-exit.sh ./cmd/kata-runtime
-	$(QUIET_CHECK)../../ci/go-no-os-exit.sh ./virtcontainers
+fast-test: $(GENERATED_FILES)
+	go clean -testcache
+	for s in $$(go list ./...); do if ! go test -failfast -v -mod=vendor -p 1 $$s; then break; fi; done
+
+GOLANGCI_LINT_FILE := ../../../tests/.ci/.golangci.yml
+GOLANGCI_LINT_NAME = golangci-lint
+GOLANGCI_LINT_CMD := $(shell command -v $(GOLANGCI_LINT_NAME) 2>/dev/null)
+lint: all
+	if [ -z $(GOLANGCI_LINT_CMD) ] ; \
+	then \
+		echo "ERROR: command $(GOLANGCI_LINT_NAME) not found. Please install it first." >&2; exit 1; \
+	fi
+
+	if [ -f $(GOLANGCI_LINT_FILE) ] ; \
+	then \
+		echo "running $(GOLANGCI_LINT_NAME)..."; \
+		$(GOLANGCI_LINT_NAME) run -c $(GOLANGCI_LINT_FILE) ; \
+	else \
+		echo "ERROR: file $(GOLANGCI_LINT_FILE) not found. You should clone https://github.com/kata-containers/tests to run $(GOLANGCI_LINT_NAME) locally." >&2; exit 1; \
+	fi;
+
+pre-commit: lint fast-test
 
 coverage:
 	go test -v -mod=vendor -covermode=atomic -coverprofile=coverage.txt ./...
 	go tool cover -html=coverage.txt -o coverage.html
 
-install: default install-runtime install-containerd-shim-v2 install-monitor install-netmon
+install: all install-runtime install-containerd-shim-v2 install-monitor install-netmon
 
 install-bin: $(BINLIST)
 	$(QUIET_INST)$(foreach f,$(BINLIST),$(call INSTALL_EXEC,$f,$(BINDIR)))
@@ -675,6 +695,9 @@ show-usage: show-header
 	@printf "\n"
 	@printf "\tbuild                      : standard build (build everything).\n"
 	@printf "\ttest                       : run tests.\n"
+	@printf "\tpre-commit                 : run $(GOLANGCI_LINT_NAME) and tests locally.\n"
+	@printf "\tlint                       : run $(GOLANGCI_LINT_NAME).\n"
+	@printf "\tfast-test                  : run tests with failfast option.\n"
 	@printf "\tcheck                      : run code checks.\n"
 	@printf "\tclean                      : remove built files.\n"
 	@printf "\tcontainerd-shim-v2         : only build containerd shim v2.\n"

src/runtime/cmd/kata-runtime/exit.go

@@ -1,28 +0,0 @@
-// Copyright (c) 2017 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-package main
-
-import "os"
-
-var atexitFuncs []func()
-
-var exitFunc = os.Exit
-
-// atexit registers a function f that will be run when exit is called. The
-// handlers so registered will be called the in reverse order of their
-// registration.
-func atexit(f func()) {
-	atexitFuncs = append(atexitFuncs, f)
-}
-
-// exit calls all atexit handlers before exiting the process with status.
-func exit(status int) {
-	for i := len(atexitFuncs) - 1; i >= 0; i-- {
-		f := atexitFuncs[i]
-		f()
-	}
-	exitFunc(status)
-}

src/runtime/cmd/kata-runtime/exit_test.go

@@ -1,42 +0,0 @@
-// Copyright (c) 2017 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-package main
-
-import (
-	"os"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-var testFoo string
-
-func testFunc() {
-	testFoo = "bar"
-}
-
-func TestExit(t *testing.T) {
-	assert := assert.New(t)
-
-	var testExitStatus int
-	exitFunc = func(status int) {
-		testExitStatus = status
-	}
-
-	defer func() {
-		exitFunc = os.Exit
-	}()
-
-	// test with no atexit functions added.
-	exit(1)
-	assert.Equal(testExitStatus, 1)
-
-	// test with a function added to the atexit list.
-	atexit(testFunc)
-	exit(0)
-	assert.Equal(testFoo, "bar")
-	assert.Equal(testExitStatus, 0)
-}

src/runtime/cmd/kata-runtime/kata-env.go

@@ -29,7 +29,7 @@ import (
 //
 // XXX: Increment for every change to the output format
 // (meaning any change to the EnvInfo type).
-const formatVersion = "1.0.25"
+const formatVersion = "1.0.26"
 
 // MetaInfo stores information on the format of the output itself
 type MetaInfo struct {
@@ -108,6 +108,7 @@ type HypervisorInfo struct {
 	EntropySource        string
 	SharedFS             string
 	VirtioFSDaemon       string
+	SocketPath           string
 	Msize9p              uint32
 	MemorySlots          uint32
 	PCIeRootPort         uint32
@@ -117,10 +118,8 @@ type HypervisorInfo struct {
 
 // AgentInfo stores agent details
 type AgentInfo struct {
-	TraceMode string
-	TraceType string
-	Debug     bool
-	Trace     bool
+	Debug bool
+	Trace bool
 }
 
 // DistroInfo stores host operating system distribution details.
@@ -131,13 +130,14 @@ type DistroInfo struct {
 
 // HostInfo stores host details
 type HostInfo struct {
-	Kernel             string
-	Architecture       string
-	Distro             DistroInfo
-	CPU                CPUInfo
-	Memory             MemoryInfo
-	VMContainerCapable bool
-	SupportVSocks      bool
+	AvailableGuestProtections []string
+	Kernel                    string
+	Architecture              string
+	Distro                    DistroInfo
+	CPU                       CPUInfo
+	Memory                    MemoryInfo
+	VMContainerCapable        bool
+	SupportVSocks             bool
 }
 
 // NetmonInfo stores netmon details
@@ -157,11 +157,11 @@ type EnvInfo struct {
 	Meta       MetaInfo
 	Image      ImageInfo
 	Initrd     InitrdInfo
-	Agent      AgentInfo
 	Hypervisor HypervisorInfo
-	Netmon     NetmonInfo
 	Runtime    RuntimeInfo
+	Netmon     NetmonInfo
 	Host       HostInfo
+	Agent      AgentInfo
 }
 
 func getMetaInfo() MetaInfo {
@@ -242,14 +242,17 @@ func getHostInfo() (HostInfo, error) {
 
 	memoryInfo := getMemoryInfo()
 
+	availableGuestProtection := vc.AvailableGuestProtections()
+
 	host := HostInfo{
-		Kernel:             hostKernelVersion,
-		Architecture:       arch,
-		Distro:             hostDistro,
-		CPU:                hostCPU,
-		Memory:             memoryInfo,
-		VMContainerCapable: hostVMContainerCapable,
-		SupportVSocks:      supportVSocks,
+		Kernel:                    hostKernelVersion,
+		Architecture:              arch,
+		Distro:                    hostDistro,
+		CPU:                       hostCPU,
+		Memory:                    memoryInfo,
+		AvailableGuestProtections: availableGuestProtection,
+		VMContainerCapable:        hostVMContainerCapable,
+		SupportVSocks:             supportVSocks,
 	}
 
 	return host, nil
@@ -303,13 +306,11 @@ func getAgentInfo(config oci.RuntimeConfig) (AgentInfo, error) {
 	agentConfig := config.AgentConfig
 	agent.Debug = agentConfig.Debug
 	agent.Trace = agentConfig.Trace
-	agent.TraceMode = agentConfig.TraceMode
-	agent.TraceType = agentConfig.TraceType
 
 	return agent, nil
 }
 
-func getHypervisorInfo(config oci.RuntimeConfig) HypervisorInfo {
+func getHypervisorInfo(config oci.RuntimeConfig) (HypervisorInfo, error) {
 	hypervisorPath := config.HypervisorConfig.HypervisorPath
 
 	version, err := getCommandVersion(hypervisorPath)
@@ -317,6 +318,19 @@ func getHypervisorInfo(config oci.RuntimeConfig) HypervisorInfo {
 		version = unknown
 	}
 
+	hypervisorType := config.HypervisorType
+
+	socketPath := unknown
+
+	// It is only reliable to make this call as root since a
+	// non-privileged user may not have access to /dev/vhost-vsock.
+	if os.Geteuid() == 0 {
+		socketPath, err = vc.GetHypervisorSocketTemplate(hypervisorType, &config.HypervisorConfig)
+		if err != nil {
+			return HypervisorInfo{}, err
+		}
+	}
+
 	return HypervisorInfo{
 		Debug:             config.HypervisorConfig.Debug,
 		MachineType:       config.HypervisorConfig.HypervisorMachineType,
@@ -331,7 +345,8 @@ func getHypervisorInfo(config oci.RuntimeConfig) HypervisorInfo {
 
 		HotplugVFIOOnRootBus: config.HypervisorConfig.HotplugVFIOOnRootBus,
 		PCIeRootPort:         config.HypervisorConfig.PCIeRootPort,
-	}
+		SocketPath:           socketPath,
+	}, nil
 }
 
 func getEnvInfo(configFile string, config oci.RuntimeConfig) (env EnvInfo, err error) {
@@ -356,7 +371,10 @@ func getEnvInfo(configFile string, config oci.RuntimeConfig) (env EnvInfo, err e
 		return EnvInfo{}, err
 	}
 
-	hypervisor := getHypervisorInfo(config)
+	hypervisor, err := getHypervisorInfo(config)
+	if err != nil {
+		return EnvInfo{}, err
+	}
 
 	image := ImageInfo{
 		Path: config.HypervisorConfig.ImagePath,

src/runtime/cmd/kata-runtime/kata-env_test.go

@@ -184,10 +184,6 @@ func getExpectedAgentDetails(config oci.RuntimeConfig) (AgentInfo, error) {
 	return AgentInfo{
 		Debug: agentConfig.Debug,
 		Trace: agentConfig.Trace,
-
-		// No trace mode/type set by default
-		TraceMode: "",
-		TraceType: "",
 	}, nil
 }
 
@@ -281,7 +277,7 @@ VERSION_ID="%s"
 }
 
 func getExpectedHypervisor(config oci.RuntimeConfig) HypervisorInfo {
-	return HypervisorInfo{
+	info := HypervisorInfo{
 		Version:           testHypervisorVersion,
 		Path:              config.HypervisorConfig.HypervisorPath,
 		MachineType:       config.HypervisorConfig.HypervisorMachineType,
@@ -296,6 +292,16 @@ func getExpectedHypervisor(config oci.RuntimeConfig) HypervisorInfo {
 		HotplugVFIOOnRootBus: config.HypervisorConfig.HotplugVFIOOnRootBus,
 		PCIeRootPort:         config.HypervisorConfig.PCIeRootPort,
 	}
+
+	if os.Geteuid() == 0 {
+		// This assumes the test hypervisor is a non-hybrid-vsock
+		// one (such as QEMU).
+		info.SocketPath = ""
+	} else {
+		info.SocketPath = unknown
+	}
+
+	return info
 }
 
 func getExpectedImage(config oci.RuntimeConfig) ImageInfo {
@@ -677,14 +683,10 @@ func TestEnvGetAgentInfo(t *testing.T) {
 	assert.True(t, agent.Debug)
 
 	agentConfig.Trace = true
-	agentConfig.TraceMode = "traceMode"
-	agentConfig.TraceType = "traceType"
 	config.AgentConfig = agentConfig
 	agent, err = getAgentInfo(config)
 	assert.NoError(t, err)
 	assert.True(t, agent.Trace)
-	assert.Equal(t, agent.TraceMode, "traceMode")
-	assert.Equal(t, agent.TraceType, "traceType")
 }
 
 func testEnvShowTOMLSettings(t *testing.T, tmpdir string, tmpfile *os.File) error {
@@ -1015,12 +1017,58 @@ func TestGetHypervisorInfo(t *testing.T) {
 	_, config, err := makeRuntimeConfig(tmpdir)
 	assert.NoError(err)
 
-	info := getHypervisorInfo(config)
+	info, err := getHypervisorInfo(config)
+	assert.NoError(err)
 	assert.Equal(info.Version, testHypervisorVersion)
 
 	err = os.Remove(config.HypervisorConfig.HypervisorPath)
 	assert.NoError(err)
 
-	info = getHypervisorInfo(config)
+	info, err = getHypervisorInfo(config)
+	assert.NoError(err)
 	assert.Equal(info.Version, unknown)
 }
+
+func TestGetHypervisorInfoSocket(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	_, config, err := makeRuntimeConfig(tmpdir)
+	assert.NoError(err)
+
+	type TestHypervisorDetails struct {
+		hType       vc.HypervisorType
+		hybridVsock bool
+	}
+
+	hypervisors := []TestHypervisorDetails{
+		{vc.AcrnHypervisor, false},
+		{vc.ClhHypervisor, true},
+		{vc.FirecrackerHypervisor, true},
+		{vc.MockHypervisor, false},
+		{vc.QemuHypervisor, false},
+	}
+
+	for i, details := range hypervisors {
+		msg := fmt.Sprintf("hypervisor[%d]: %+v", i, details)
+
+		config.HypervisorType = details.hType
+
+		info, err := getHypervisorInfo(config)
+		assert.NoError(err, msg)
+
+		if os.Geteuid() == 0 {
+			if !details.hybridVsock {
+				assert.Equal(info.SocketPath, "", msg)
+			} else {
+				assert.NotEmpty(info.SocketPath, msg)
+				assert.True(strings.HasPrefix(info.SocketPath, "/"), msg)
+			}
+		} else {
+			assert.Equal(info.SocketPath, unknown, msg)
+		}
+	}
+}

src/runtime/cmd/kata-runtime/main.go

@@ -30,12 +30,11 @@ import (
 	"github.com/urfave/cli"
 )
 
-// specConfig is the name of the file holding the containers configuration
-const specConfig = "config.json"
-
 // arch is the architecture for the running program
 const arch = goruntime.GOARCH
 
+var exitFunc = os.Exit
+
 var usage = fmt.Sprintf(`%s runtime
 
 %s is a command line program for running applications packaged
@@ -331,7 +330,7 @@ func handleShowConfig(context *cli.Context) {
 			fmt.Fprintf(defaultOutputFile, "%s\n", file)
 		}
 
-		exit(0)
+		exitFunc(0)
 	}
 }
 
@@ -451,7 +450,7 @@ func userWantsUsage(context *cli.Context) bool {
 func fatal(err error) {
 	kataLog.Error(err)
 	fmt.Fprintln(defaultErrorFile, err)
-	exit(1)
+	exitFunc(1)
 }
 
 type fatalWriter struct {

src/runtime/cmd/kata-runtime/main_test.go

@@ -13,7 +13,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path"
 	"path/filepath"
 	"regexp"
@@ -22,9 +21,7 @@ import (
 
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/vcmock"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -38,19 +35,12 @@ const (
 	testFileMode    = os.FileMode(0640)
 	testExeFileMode = os.FileMode(0750)
 
-	// small docker image used to create root filesystems from
-	testDockerImage = "busybox"
-
-	testBundle  = "bundle"
 	testConsole = "/dev/pts/999"
 )
 
 var (
 	// package variables set by calling TestMain()
-	testDir       = ""
-	testBundleDir = ""
-	tc            ktu.TestConstraint
-	ctrEngine     = katautils.CtrEngine{}
+	tc ktu.TestConstraint
 )
 
 // testingImpl is a concrete mock RVC implementation used for testing
@@ -79,57 +69,6 @@ func init() {
 	fmt.Printf("INFO: switching to fake virtcontainers implementation for testing\n")
 	vci = testingImpl
 
-	var err error
-
-	fmt.Printf("INFO: creating test directory\n")
-	testDir, err = ioutil.TempDir("", fmt.Sprintf("%s-", katautils.NAME))
-	if err != nil {
-		panic(fmt.Sprintf("ERROR: failed to create test directory: %v", err))
-	}
-	fmt.Printf("INFO: test directory is %v\n", testDir)
-
-	var output string
-	for _, name := range katautils.DockerLikeCtrEngines {
-		fmt.Printf("INFO: checking for container engine: %s\n", name)
-
-		output, err = ctrEngine.Init(name)
-		if err == nil {
-			break
-		}
-	}
-
-	if ctrEngine.Name == "" {
-		panic(fmt.Sprintf("ERROR: Docker-like container engine not accessible to current user: %v (error %v)",
-			output, err))
-	}
-
-	// Do this now to avoid hitting the test timeout value due to
-	// slow network response.
-	fmt.Printf("INFO: ensuring required container image (%v) is available\n", testDockerImage)
-	// Only hit the network if the image doesn't exist locally
-	_, err = ctrEngine.Inspect(testDockerImage)
-	if err == nil {
-		fmt.Printf("INFO: container image %v already exists locally\n", testDockerImage)
-	} else {
-		fmt.Printf("INFO: pulling container image %v\n", testDockerImage)
-		_, err = ctrEngine.Pull(testDockerImage)
-		if err != nil {
-			panic(err)
-		}
-	}
-
-	testBundleDir = filepath.Join(testDir, testBundle)
-	err = os.MkdirAll(testBundleDir, testDirMode)
-	if err != nil {
-		panic(fmt.Sprintf("ERROR: failed to create bundle directory %v: %v", testBundleDir, err))
-	}
-
-	fmt.Printf("INFO: creating OCI bundle in %v for tests to use\n", testBundleDir)
-	err = realMakeOCIBundle(testBundleDir)
-	if err != nil {
-		panic(fmt.Sprintf("ERROR: failed to create OCI bundle: %v", err))
-	}
-
 	tc = ktu.NewTestConstraint(false)
 }
 
@@ -143,8 +82,6 @@ func resetCLIGlobals() {
 func runUnitTests(m *testing.M) {
 	ret := m.Run()
 
-	os.RemoveAll(testDir)
-
 	os.Exit(ret)
 }
 
@@ -156,7 +93,7 @@ func TestMain(m *testing.M) {
 	if path.Base(os.Args[0]) == katautils.NAME+".coverage" ||
 		path.Base(os.Args[0]) == katautils.NAME {
 		main()
-		exit(0)
+		exitFunc(0)
 	}
 
 	runUnitTests(m)
@@ -232,146 +169,6 @@ func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConf
 	}, nil
 }
 
-// createOCIConfig creates an OCI configuration (spec) file in
-// the bundle directory specified (which must exist).
-func createOCIConfig(bundleDir string) error {
-	if bundleDir == "" {
-		return errors.New("BUG: Need bundle directory")
-	}
-
-	if !katautils.FileExists(bundleDir) {
-		return fmt.Errorf("BUG: Bundle directory %s does not exist", bundleDir)
-	}
-
-	var configCmd string
-
-	// Search for a suitable version of runc to use to generate
-	// the OCI config file.
-	for _, cmd := range []string{"docker-runc", "runc"} {
-		fullPath, err := exec.LookPath(cmd)
-		if err == nil {
-			configCmd = fullPath
-			break
-		}
-	}
-
-	if configCmd == "" {
-		return errors.New("Cannot find command to generate OCI config file")
-	}
-
-	_, err := utils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
-	if err != nil {
-		return err
-	}
-
-	specFile := filepath.Join(bundleDir, specConfig)
-	if !katautils.FileExists(specFile) {
-		return fmt.Errorf("generated OCI config file does not exist: %v", specFile)
-	}
-
-	return nil
-}
-
-// createRootfs creates a minimal root filesystem below the specified
-// directory.
-func createRootfs(dir string) error {
-	err := os.MkdirAll(dir, testDirMode)
-	if err != nil {
-		return err
-	}
-
-	container, err := ctrEngine.Create(testDockerImage)
-	if err != nil {
-		return err
-	}
-
-	err = ctrEngine.GetRootfs(container, dir)
-	if err != nil {
-		return err
-	}
-
-	// Clean up
-	_, err = ctrEngine.Rm(container)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// realMakeOCIBundle will create an OCI bundle (including the "config.json"
-// config file) in the directory specified (which must already exist).
-//
-// XXX: Note that tests should *NOT* call this function - they should
-// XXX: instead call makeOCIBundle().
-func realMakeOCIBundle(bundleDir string) error {
-	if bundleDir == "" {
-		return errors.New("BUG: Need bundle directory")
-	}
-
-	if !katautils.FileExists(bundleDir) {
-		return fmt.Errorf("BUG: Bundle directory %v does not exist", bundleDir)
-	}
-
-	err := createOCIConfig(bundleDir)
-	if err != nil {
-		return err
-	}
-
-	// Note the unusual parameter (a directory, not the config
-	// file to parse!)
-	spec, err := compatoci.ParseConfigJSON(bundleDir)
-	if err != nil {
-		return err
-	}
-
-	// Determine the rootfs directory name the OCI config refers to
-	ociRootPath := spec.Root.Path
-
-	rootfsDir := filepath.Join(bundleDir, ociRootPath)
-
-	if strings.HasPrefix(ociRootPath, "/") {
-		return fmt.Errorf("Cannot handle absolute rootfs as bundle must be unique to each test")
-	}
-
-	err = createRootfs(rootfsDir)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// Create an OCI bundle in the specified directory.
-//
-// Note that the directory will be created, but it's parent is expected to exist.
-//
-// This function works by copying the already-created test bundle. Ideally,
-// the bundle would be recreated for each test, but createRootfs() uses
-// docker which on some systems is too slow, resulting in the tests timing
-// out.
-func makeOCIBundle(bundleDir string) error {
-	from := testBundleDir
-	to := bundleDir
-
-	// only the basename of bundleDir needs to exist as bundleDir
-	// will get created by cp(1).
-	base := filepath.Dir(bundleDir)
-
-	for _, dir := range []string{from, base} {
-		if !katautils.FileExists(dir) {
-			return fmt.Errorf("BUG: directory %v should exist", dir)
-		}
-	}
-
-	output, err := utils.RunCommandFull([]string{"cp", "-a", from, to}, true)
-	if err != nil {
-		return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output)
-	}
-
-	return nil
-}
-
 func createCLIContextWithApp(flagSet *flag.FlagSet, app *cli.App) *cli.Context {
 	ctx := cli.NewContext(app, flagSet, nil)
 
@@ -390,69 +187,6 @@ func createCLIContext(flagset *flag.FlagSet) *cli.Context {
 	return createCLIContextWithApp(flagset, cli.NewApp())
 }
 
-func TestMakeOCIBundle(t *testing.T) {
-	assert := assert.New(t)
-
-	tmpdir, err := ioutil.TempDir(testDir, "")
-	assert.NoError(err)
-	defer os.RemoveAll(tmpdir)
-
-	bundleDir := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundleDir)
-	assert.NoError(err)
-
-	specFile := filepath.Join(bundleDir, specConfig)
-	assert.True(katautils.FileExists(specFile))
-}
-
-func TestCreateOCIConfig(t *testing.T) {
-	assert := assert.New(t)
-
-	tmpdir, err := ioutil.TempDir(testDir, "")
-	assert.NoError(err)
-	defer os.RemoveAll(tmpdir)
-
-	bundleDir := filepath.Join(tmpdir, "bundle")
-
-	err = createOCIConfig(bundleDir)
-	// ENOENT
-	assert.Error(err)
-
-	err = os.MkdirAll(bundleDir, testDirMode)
-	assert.NoError(err)
-
-	err = createOCIConfig(bundleDir)
-	assert.NoError(err)
-
-	specFile := filepath.Join(bundleDir, specConfig)
-	assert.True(katautils.FileExists(specFile))
-}
-
-func TestCreateRootfs(t *testing.T) {
-	assert := assert.New(t)
-
-	tmpdir, err := ioutil.TempDir(testDir, "")
-	assert.NoError(err)
-	defer os.RemoveAll(tmpdir)
-
-	rootfsDir := filepath.Join(tmpdir, "rootfs")
-	assert.False(katautils.FileExists(rootfsDir))
-
-	err = createRootfs(rootfsDir)
-	assert.NoError(err)
-
-	// non-comprehensive list of expected directories
-	expectedDirs := []string{"bin", "dev", "etc", "usr", "var"}
-
-	assert.True(katautils.FileExists(rootfsDir))
-
-	for _, dir := range expectedDirs {
-		dirPath := filepath.Join(rootfsDir, dir)
-		assert.True(katautils.FileExists(dirPath))
-	}
-}
-
 func TestMainUserWantsUsage(t *testing.T) {
 	assert := assert.New(t)
 
@@ -525,7 +259,7 @@ func TestMainBeforeSubCommands(t *testing.T) {
 func TestMainBeforeSubCommandsInvalidLogFile(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir(testDir, "")
+	tmpdir, err := ioutil.TempDir("", "katatest")
 	assert.NoError(err)
 	defer os.RemoveAll(tmpdir)
 
@@ -548,7 +282,7 @@ func TestMainBeforeSubCommandsInvalidLogFile(t *testing.T) {
 func TestMainBeforeSubCommandsInvalidLogFormat(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir(testDir, "")
+	tmpdir, err := ioutil.TempDir("", "katatest")
 	assert.NoError(err)
 	defer os.RemoveAll(tmpdir)
 
@@ -577,7 +311,7 @@ func TestMainBeforeSubCommandsInvalidLogFormat(t *testing.T) {
 func TestMainBeforeSubCommandsLoadConfigurationFail(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir(testDir, "")
+	tmpdir, err := ioutil.TempDir("", "katatest")
 	assert.NoError(err)
 	defer os.RemoveAll(tmpdir)
 
@@ -612,7 +346,7 @@ func TestMainBeforeSubCommandsLoadConfigurationFail(t *testing.T) {
 func TestMainBeforeSubCommandsShowCCConfigPaths(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir(testDir, "")
+	tmpdir, err := ioutil.TempDir("", "katatest")
 	assert.NoError(err)
 	defer os.RemoveAll(tmpdir)
 
@@ -676,7 +410,7 @@ func TestMainBeforeSubCommandsShowCCConfigPaths(t *testing.T) {
 func TestMainFatal(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir(testDir, "")
+	tmpdir, err := ioutil.TempDir("", "katatest")
 	assert.NoError(err)
 	defer os.RemoveAll(tmpdir)
 
@@ -900,7 +634,7 @@ func TestMainCreateRuntime(t *testing.T) {
 func TestMainVersionPrinter(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir("", "")
+	tmpdir, err := ioutil.TempDir("", "katatest")
 	assert.NoError(err)
 	defer os.RemoveAll(tmpdir)
 

src/runtime/cmd/kata-runtime/utils_test.go

@@ -18,7 +18,7 @@ import (
 )
 
 func TestFileExists(t *testing.T) {
-	dir, err := ioutil.TempDir(testDir, "")
+	dir, err := ioutil.TempDir("", "katatest")
 	if err != nil {
 		t.Fatal(err)
 	}

src/runtime/config/configuration-acrn.toml.in

@@ -124,24 +124,17 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_ACRN@"
 
 # Enable agent tracing.
 #
-# If enabled, the default trace mode is "dynamic" and the
-# default trace type is "isolated". The trace mode and type are set
-# explicity with the `trace_type=` and `trace_mode=` options.
+# If enabled, the agent will generate OpenTelemetry trace spans.
 #
 # Notes:
 #
-# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly
-#   setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing`
-#   will NOT activate agent tracing.
-#
-# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for
-#   full details.
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
 #
 # (default: disabled)
 #enable_tracing = true
-#
-#trace_mode = "dynamic"
-#trace_type = "isolated"
 
 # Enable debug console.
 

src/runtime/config/configuration-clh.toml.in

@@ -105,8 +105,7 @@ virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@
 virtio_fs_cache = "@DEFVIRTIOFSCACHE@"
 
 # Block storage driver to be used for the hypervisor in case the container
-# rootfs is backed by a block device. This is virtio-scsi, virtio-blk
-# or nvdimm.
+# rootfs is backed by a block device. This is virtio-blk.
 block_device_driver = "virtio-blk"
 
 # Enable huge pages for VM RAM, default false
@@ -114,6 +113,9 @@ block_device_driver = "virtio-blk"
 # being allocated using huge pages.
 #enable_hugepages = true
 
+# Disable the 'seccomp' feature from Cloud Hypervisor, default false
+# disable_seccomp = true
+
 # This option changes the default hypervisor and kernel parameters
 # to enable debug output where available.
 #
@@ -144,24 +146,17 @@ block_device_driver = "virtio-blk"
 
 # Enable agent tracing.
 #
-# If enabled, the default trace mode is "dynamic" and the
-# default trace type is "isolated". The trace mode and type are set
-# explicity with the `trace_type=` and `trace_mode=` options.
+# If enabled, the agent will generate OpenTelemetry trace spans.
 #
 # Notes:
 #
-# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly
-#   setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing`
-#   will NOT activate agent tracing.
-#
-# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for
-#   full details.
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
 #
 # (default: disabled)
 #enable_tracing = true
-#
-#trace_mode = "dynamic"
-#trace_type = "isolated"
 
 # Enable debug console.
 
@@ -267,6 +262,27 @@ sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
 # These will not be exposed to the container workloads, and are only provided for potential guest services.
 sandbox_bind_mounts=@DEFBINDMOUNTS@
 
+# VFIO Mode
+# Determines how VFIO devices should be be presented to the container.
+# Options:
+#
+#  - vfio
+#    Matches behaviour of OCI runtimes (e.g. runc) as much as
+#    possible.  VFIO devices will appear in the container as VFIO
+#    character devices under /dev/vfio.  The exact names may differ
+#    from the host (they need to match the VM's IOMMU group numbers
+#    rather than the host's)
+#
+#  - guest-kernel
+#    This is a Kata-specific behaviour that's useful in certain cases.
+#    The VFIO device is managed by whatever driver in the VM kernel
+#    claims it.  This means it will appear as one or more device nodes
+#    or network interfaces depending on the nature of the device.
+#    Using this mode requires specially built workloads that know how
+#    to locate the relevant device interfaces within the VM.
+#
+vfio_mode="@DEFVFIOMODE@"
+
 # Enabled experimental feature list, format: ["a", "b"].
 # Experimental features are features not stable enough for production,
 # they may break compatibility, and are prepared for a big version bump.

src/runtime/config/configuration-fc.toml.in

@@ -246,24 +246,17 @@ valid_entropy_sources = @DEFVALIDENTROPYSOURCES@
 
 # Enable agent tracing.
 #
-# If enabled, the default trace mode is "dynamic" and the
-# default trace type is "isolated". The trace mode and type are set
-# explicity with the `trace_type=` and `trace_mode=` options.
+# If enabled, the agent will generate OpenTelemetry trace spans.
 #
 # Notes:
 #
-# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly
-#   setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing`
-#   will NOT activate agent tracing.
-#
-# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for
-#   full details.
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
 #
 # (default: disabled)
 #enable_tracing = true
-#
-#trace_mode = "dynamic"
-#trace_type = "isolated"
 
 # Comma separated list of kernel modules and their parameters.
 # These modules will be loaded in the guest kernel using modprobe(8).

src/runtime/config/configuration-qemu.toml.in

@@ -422,24 +422,17 @@ valid_entropy_sources = @DEFVALIDENTROPYSOURCES@
 
 # Enable agent tracing.
 #
-# If enabled, the default trace mode is "dynamic" and the
-# default trace type is "isolated". The trace mode and type are set
-# explicity with the `trace_type=` and `trace_mode=` options.
+# If enabled, the agent will generate OpenTelemetry trace spans.
 #
 # Notes:
 #
-# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly
-#   setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing`
-#   will NOT activate agent tracing.
-#
-# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for
-#   full details.
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
 #
 # (default: disabled)
 #enable_tracing = true
-#
-#trace_mode = "dynamic"
-#trace_type = "isolated"
 
 # Comma separated list of kernel modules and their parameters.
 # These modules will be loaded in the guest kernel using modprobe(8).
@@ -550,6 +543,27 @@ sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
 # These will not be exposed to the container workloads, and are only provided for potential guest services.
 sandbox_bind_mounts=@DEFBINDMOUNTS@
 
+# VFIO Mode
+# Determines how VFIO devices should be be presented to the container.
+# Options:
+#
+#  - vfio
+#    Matches behaviour of OCI runtimes (e.g. runc) as much as
+#    possible.  VFIO devices will appear in the container as VFIO
+#    character devices under /dev/vfio.  The exact names may differ
+#    from the host (they need to match the VM's IOMMU group numbers
+#    rather than the host's)
+#
+#  - guest-kernel
+#    This is a Kata-specific behaviour that's useful in certain cases.
+#    The VFIO device is managed by whatever driver in the VM kernel
+#    claims it.  This means it will appear as one or more device nodes
+#    or network interfaces depending on the nature of the device.
+#    Using this mode requires specially built workloads that know how
+#    to locate the relevant device interfaces within the VM.
+#
+vfio_mode="@DEFVFIOMODE@"
+
 # Enabled experimental feature list, format: ["a", "b"].
 # Experimental features are features not stable enough for production,
 # they may break compatibility, and are prepared for a big version bump.

src/runtime/data/completions/bash/kata-runtime

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Copyright (c) 2018 Intel Corporation
 #
@@ -53,39 +53,6 @@ _kata_subcmd_seen()
     echo ""
 }
 
-# Return 0 if the specified sub-command requires the name of an
-# *existing* container, else 1.
-_kata_subcmd_needs_existing_container()
-{
-    local subcmd="$1"
-    local cmd
-
-    for cmd in \
-	    'kata-check' \
-	    'kata-env' \
-	    'check' \
-	    'env' \
-	    'create' \
-	    'help' \
-	    'list' \
-	    'version'; do
-        [ "$cmd" = "$subcmd" ] && return 1
-    done
-
-    return 0
-}
-
-# Returns a list of container names
-_kata_get_containers()
-{
-    # Commands that manipulate containers need root privileges.
-    # If the user isn't running as root, don't attempt to obtain a list
-    # as it will result in an error.
-    [ $(id -u) -eq 0 ] || return
-
-    "$_kataruntime" list --quiet
-}
-
 _kata_bash_autocomplete() {
     COMPREPLY=()
 
@@ -104,32 +71,22 @@ _kata_bash_autocomplete() {
     local subcmd_seen=$(_kata_subcmd_seen)
 
     if [ -n "$subcmd_seen" ]; then
-	    _kata_subcmd_needs_existing_container "$subcmd_seen"
-	    local container_cmd=$?
-
-            if [ -n "$cur" ]; then
-                    # Complete with local options and maybe container names
-                    opts=$(_kata_get_subcmd_options "$subcmd_seen")
-                    [ $container_cmd -eq 0 ] && opts="$opts $(_kata_get_containers)"
-            elif [[ "${cur}" == -* ]]; then
-                    # Complete with local options
-                    opts=$(_kata_get_subcmd_options "$subcmd_seen")
-            else
-                    # Potentially complete with container names
-                    [ $container_cmd -eq 0 ] && opts="$(_kata_get_containers)"
-            fi
+        if [ -n "$cur" ]; then
+                # Complete with local options
+                opts=$(_kata_get_subcmd_options "$subcmd_seen")
+        fi
     else
-            if [ -n "$cur" ]; then
-                    # Complete with global options and subcmds
-                    opts="$opts $(_kata_get_global_options)"
-                    opts="$opts $(_kata_get_subcmds)"
-            elif [[ "${cur}" == -* ]]; then
-                    # Complete with global options
-                    opts=$(_kata_get_global_options)
-            else
-                    # Complete with subcmds
-                    opts=$(_kata_get_subcmds)
-            fi
+        if [ -n "$cur" ]; then
+                # Complete with global options and subcmds
+                opts="$opts $(_kata_get_global_options)"
+                opts="$opts $(_kata_get_subcmds)"
+        elif [[ "${cur}" == -* ]]; then
+                # Complete with global options
+                opts=$(_kata_get_global_options)
+        else
+                # Complete with subcmds
+                opts=$(_kata_get_subcmds)
+        fi
     fi
 
     [ -n "$opts" ] && COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )

src/runtime/go.mod

@@ -9,7 +9,6 @@ require (
 	github.com/containerd/cgroups v1.0.1
 	github.com/containerd/console v1.0.2
 	github.com/containerd/containerd v1.5.7
-	github.com/containerd/cri-containerd v1.11.1-0.20190125013620-4dd6735020f5
 	github.com/containerd/fifo v1.0.0
 	github.com/containerd/ttrpc v1.0.2
 	github.com/containerd/typeurl v1.0.2
@@ -38,17 +37,22 @@ require (
 	github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8
 	github.com/sirupsen/logrus v1.8.1
 	github.com/smartystreets/goconvey v1.6.4 // indirect
-	github.com/stretchr/testify v1.6.1
+	github.com/stretchr/testify v1.7.0
 	github.com/urfave/cli v1.22.2
 	github.com/vishvananda/netlink v1.1.1-0.20210924202909-187053b97868
 	github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae
-	go.opentelemetry.io/otel v0.15.0
-	go.opentelemetry.io/otel/exporters/trace/jaeger v0.15.0
-	go.opentelemetry.io/otel/sdk v0.15.0
+	go.opencensus.io v0.23.0 // indirect
+	go.opentelemetry.io/otel v1.0.0
+	go.opentelemetry.io/otel/exporters/jaeger v1.0.0
+	go.opentelemetry.io/otel/sdk v1.0.0
+	go.opentelemetry.io/otel/trace v1.0.0
 	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110
-	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43
+	golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93
 	golang.org/x/sys v0.0.0-20210426230700-d19ff857e887
-	google.golang.org/grpc v1.33.2
+	golang.org/x/text v0.3.5 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb // indirect
+	google.golang.org/grpc v1.36.0
 	k8s.io/apimachinery v0.20.6
 	k8s.io/cri-api v0.20.6
 )

src/runtime/go.sum

@@ -45,7 +45,6 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/DataDog/sketches-go v0.0.1/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60=
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
@@ -67,13 +66,10 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
-github.com/apache/thrift v0.13.0 h1:5hryIiq9gtn+MiLVn0wP37kb/uTeRZgN08WoCsAhIhI=
-github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -100,6 +96,7 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ
 github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=
 github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss=
@@ -114,8 +111,6 @@ github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0
 github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ=
 github.com/containerd/continuity v0.1.0 h1:UFRRY5JemiAhPZrr/uE0n8fMTLcZsUvySPr1+D7pgr8=
 github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM=
-github.com/containerd/cri-containerd v1.11.1-0.20190125013620-4dd6735020f5 h1:/srF029I+oDfm/qeltxCGJyJ8urmlqWGOQmQ7HvwrRc=
-github.com/containerd/cri-containerd v1.11.1-0.20190125013620-4dd6735020f5/go.mod h1:wxbGdReWGCalzGOEpifoHeYCK4xAgnj4o/4bVB+9voU=
 github.com/containerd/fifo v1.0.0 h1:6PirWBr9/L7GDamKr+XM0IeUFXu5mf3M/BPpH9gaLBU=
 github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4=
 github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk=
@@ -181,6 +176,7 @@ github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@@ -297,9 +293,11 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -519,8 +517,10 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -552,14 +552,17 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opentelemetry.io/otel v0.15.0 h1:CZFy2lPhxd4HlhZnYK8gRyDotksO3Ip9rBweY1vVYJw=
-go.opentelemetry.io/otel v0.15.0/go.mod h1:e4GKElweB8W2gWUqbghw0B8t5MCTccc9212eNHnOHwA=
-go.opentelemetry.io/otel/exporters/trace/jaeger v0.15.0 h1:OZY+lMaUJiJ6ls1dDtqKhSPJWEVNytLuRUrzuR852jc=
-go.opentelemetry.io/otel/exporters/trace/jaeger v0.15.0/go.mod h1:4DeFMzRzr2l5o/Lzh4GfOYEH+mnf7ZrEGDzzJEP6ta8=
-go.opentelemetry.io/otel/sdk v0.15.0 h1:Hf2dl1Ad9Hn03qjcAuAq51GP5Pv1SV5puIkS2nRhdd8=
-go.opentelemetry.io/otel/sdk v0.15.0/go.mod h1:Qudkwgq81OcA9GYVlbyZ62wkLieeS1eWxIL0ufxgwoc=
+go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.0.0 h1:qTTn6x71GVBvoafHK/yaRUmFzI4LcONZD0/kXxl5PHI=
+go.opentelemetry.io/otel v1.0.0/go.mod h1:AjRVh9A5/5DE7S+mZtTR6t8vpKKryam+0lREnfmS4cg=
+go.opentelemetry.io/otel/exporters/jaeger v1.0.0 h1:cLhx8llHw02h5JTqGqaRbYn+QVKHmrzD9vEbKnSPk5U=
+go.opentelemetry.io/otel/exporters/jaeger v1.0.0/go.mod h1:q10N1AolE1JjqKrFJK2tYw0iZpmX+HBaXBtuCzRnBGQ=
+go.opentelemetry.io/otel/sdk v1.0.0 h1:BNPMYUONPNbLneMttKSjQhOTlFLOD9U22HNG1KrIN2Y=
+go.opentelemetry.io/otel/sdk v1.0.0/go.mod h1:PCrDHlSy5x1kjezSdL37PhbFUMjrsLRshJ2zCzeXwbM=
+go.opentelemetry.io/otel/trace v1.0.0 h1:TSBr8GTEtKevYMG/2d21M989r5WJYVimhTHBKVEZuh4=
+go.opentelemetry.io/otel/trace v1.0.0/go.mod h1:PXTWqayeFUlJV1YDNhsJYB184+IvAH814St6o6ajzIs=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
@@ -648,8 +651,8 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 h1:ld7aEMNHoBnnDAX15v1T6z31v8HwR2A9FYOuAhWqkwc=
-golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93 h1:alLDrZkL34Y2bnGHfvC1CYBRBXCXgx8AC2vY4MRtYX4=
+golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -659,7 +662,6 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -707,7 +709,6 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -718,6 +719,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 h1:dXfMednGJh/SUUFjTLsWJz3P+TQt9qnR11GgeI3vWKs=
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -726,8 +728,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -778,7 +781,6 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -801,15 +803,14 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
 google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
 google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/api v0.32.0 h1:Le77IccnTqEa8ryp9wIpX5W3zYm7Gf9LhOp9PHcwFts=
-google.golang.org/api v0.32.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
@@ -825,9 +826,9 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa
 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
 google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.33.2 h1:EQyQC3sa8M+p6Ulc8yy9SWSS2GVwyRc83gAbG8lrl4o=
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.36.0 h1:o1bcQ6imQMIOpdrO3SWf2z5RV72WbDwdXuK0MDlc8As=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

src/runtime/pkg/containerd-shim-v2/create.go

@@ -27,10 +27,11 @@ import (
 	"github.com/pkg/errors"
 
 	// only register the proto type
-	crioption "github.com/containerd/containerd/pkg/runtimeoptions/v1"
 	_ "github.com/containerd/containerd/runtime/linux/runctypes"
 	_ "github.com/containerd/containerd/runtime/v2/runc/options"
-	oldcrioption "github.com/containerd/cri-containerd/pkg/api/runtimeoptions/v1"
+
+	crioption "github.com/containerd/containerd/pkg/runtimeoptions/v1"
+	oldcrioption "github.com/containerd/containerd/pkg/runtimeoptions/v1"
 
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
@@ -311,24 +312,26 @@ func configureNonRootHypervisor(runtimeConfig *oci.RuntimeConfig) error {
 	runtimeConfig.HypervisorConfig.Gid = uint32(gid)
 
 	userTmpDir := path.Join("/run/user/", fmt.Sprint(uid))
-	dir, err := os.Stat(userTmpDir)
-	if os.IsNotExist(err) {
-		if err = os.Mkdir(userTmpDir, vc.DirMode); err != nil {
-			return err
-		}
-		defer func() {
-			if err != nil {
-				if err = os.RemoveAll(userTmpDir); err != nil {
-					shimLog.WithField("userTmpDir", userTmpDir).WithError(err).Warn("failed to remove userTmpDir")
-				}
-			}
-		}()
-		if err = syscall.Chown(userTmpDir, uid, gid); err != nil {
+	_, err = os.Stat(userTmpDir)
+	// Clean up the directory created by the previous run
+	if !os.IsNotExist(err) {
+		if err = os.RemoveAll(userTmpDir); err != nil {
 			return err
 		}
 	}
-	if dir != nil && !dir.IsDir() {
-		return fmt.Errorf("%s is expected to be a directory", userTmpDir)
+
+	if err = os.Mkdir(userTmpDir, vc.DirMode); err != nil {
+		return err
+	}
+	defer func() {
+		if err != nil {
+			if err = os.RemoveAll(userTmpDir); err != nil {
+				shimLog.WithField("userTmpDir", userTmpDir).WithError(err).Warn("failed to remove userTmpDir")
+			}
+		}
+	}()
+	if err = syscall.Chown(userTmpDir, uid, gid); err != nil {
+		return err
 	}
 
 	if err := os.Setenv("XDG_RUNTIME_DIR", userTmpDir); err != nil {

src/runtime/pkg/containerd-shim-v2/create_test.go

@@ -12,18 +12,16 @@ import (
 	"io/ioutil"
 	"os"
 	"path"
-	"path/filepath"
 	"testing"
 
 	"github.com/containerd/containerd/namespaces"
+	crioption "github.com/containerd/containerd/pkg/runtimeoptions/v1"
 	taskAPI "github.com/containerd/containerd/runtime/v2/task"
-	crioption "github.com/containerd/cri-containerd/pkg/api/runtimeoptions/v1"
 	"github.com/containerd/typeurl"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/stretchr/testify/assert"
 
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
@@ -52,21 +50,12 @@ func TestCreateSandboxSuccess(t *testing.T) {
 		testingImpl.CreateSandboxFunc = nil
 	}()
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
-	defer os.RemoveAll(tmpdir)
+	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
+	// defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(katautils.FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -82,7 +71,7 @@ func TestCreateSandboxSuccess(t *testing.T) {
 	}
 
 	// Rewrite the file
-	err = writeOCIConfigFile(spec, ociConfigFile)
+	err = ktu.WriteOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
 	s := &service{
@@ -110,25 +99,16 @@ func TestCreateSandboxFail(t *testing.T) {
 
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(katautils.FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
-	err = writeOCIConfigFile(spec, ociConfigFile)
+	err = ktu.WriteOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
 	s := &service{
@@ -157,21 +137,12 @@ func TestCreateSandboxConfigFail(t *testing.T) {
 
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, _ := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(katautils.FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -216,21 +187,12 @@ func TestCreateContainerSuccess(t *testing.T) {
 		},
 	}
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(katautils.FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -240,7 +202,7 @@ func TestCreateContainerSuccess(t *testing.T) {
 	spec.Annotations[testSandboxIDAnnotation] = testSandboxID
 
 	// rewrite file
-	err = writeOCIConfigFile(spec, ociConfigFile)
+	err = ktu.WriteOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
 	s := &service{
@@ -265,21 +227,12 @@ func TestCreateContainerSuccess(t *testing.T) {
 func TestCreateContainerFail(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(katautils.FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -287,7 +240,7 @@ func TestCreateContainerFail(t *testing.T) {
 	spec.Annotations[testContainerTypeAnnotation] = testContainerTypeContainer
 	spec.Annotations[testSandboxIDAnnotation] = testSandboxID
 
-	err = writeOCIConfigFile(spec, ociConfigFile)
+	err = ktu.WriteOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
 	// doesn't create sandbox first
@@ -325,21 +278,12 @@ func TestCreateContainerConfigFail(t *testing.T) {
 		sandbox.CreateContainerFunc = nil
 	}()
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(katautils.FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -348,7 +292,7 @@ func TestCreateContainerConfigFail(t *testing.T) {
 	spec.Annotations[testContainerTypeAnnotation] = "errorType"
 	spec.Annotations[testSandboxIDAnnotation] = testSandboxID
 
-	err = writeOCIConfigFile(spec, ociConfigFile)
+	err = ktu.WriteOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
 	s := &service{

src/runtime/pkg/containerd-shim-v2/delete_test.go

@@ -7,14 +7,13 @@
 package containerdshim
 
 import (
-	"io/ioutil"
 	"os"
-	"path/filepath"
 	"testing"
 
 	taskAPI "github.com/containerd/containerd/runtime/v2/task"
 	"github.com/stretchr/testify/assert"
 
+	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/vcmock"
 )
@@ -26,7 +25,7 @@ func TestDeleteContainerSuccessAndFail(t *testing.T) {
 		MockID: testSandboxID,
 	}
 
-	rootPath, bundlePath := testConfigSetup(t)
+	rootPath, bundlePath, _ := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(rootPath)
 	_, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
@@ -43,19 +42,3 @@ func TestDeleteContainerSuccessAndFail(t *testing.T) {
 	s.containers[testContainerID], err = newContainer(s, reqCreate, "", nil, true)
 	assert.NoError(err)
 }
-
-func testConfigSetup(t *testing.T) (rootPath string, bundlePath string) {
-	assert := assert.New(t)
-
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
-
-	bundlePath = filepath.Join(tmpdir, "bundle")
-	err = os.MkdirAll(bundlePath, testDirMode)
-	assert.NoError(err)
-
-	err = createOCIConfig(bundlePath)
-	assert.NoError(err)
-
-	return tmpdir, bundlePath
-}

src/runtime/pkg/containerd-shim-v2/service.go

@@ -935,6 +935,10 @@ func (s *service) Shutdown(ctx context.Context, r *taskAPI.ShutdownRequest) (_ *
 	s.mu.Lock()
 	if len(s.containers) != 0 {
 		s.mu.Unlock()
+
+		span.End()
+		katatrace.StopTracing(s.rootCtx)
+
 		return empty, nil
 	}
 	s.mu.Unlock()

src/runtime/pkg/containerd-shim-v2/service_test.go

@@ -8,9 +8,7 @@ package containerdshim
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"os"
-	"path/filepath"
 	"strings"
 	"testing"
 
@@ -43,13 +41,9 @@ func TestServiceCreate(t *testing.T) {
 
 	assert := assert.New(t)
 
-	tmpdir, _ := ioutil.TempDir("", "")
+	tmpdir, bundleDir, _ := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
-	bundleDir := filepath.Join(tmpdir, "bundle")
-	err := makeOCIBundle(bundleDir)
-	assert.NoError(err)
-
 	ctx := context.Background()
 
 	s, err := newService("foo")

src/runtime/pkg/containerd-shim-v2/stream_test.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"io"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 	"syscall"
 	"testing"
@@ -24,6 +25,11 @@ func TestNewTtyIOFifoReopen(t *testing.T) {
 	var tty *ttyIO
 	assert := assert.New(t)
 	ctx := context.TODO()
+
+	testDir, err := ioutil.TempDir("", "kata-")
+	assert.NoError(err)
+	defer os.RemoveAll(testDir)
+
 	fifoPath, err := ioutil.TempDir(testDir, "fifo-path-")
 	assert.NoError(err)
 	stdout := filepath.Join(fifoPath, "stdout")
@@ -91,8 +97,6 @@ func TestNewTtyIOFifoReopen(t *testing.T) {
 }
 
 func TestIoCopy(t *testing.T) {
-	t.Skip("TestIoCopy is failing randonly, see https://github.com/kata-containers/kata-containers/issues/2042")
-
 	assert := assert.New(t)
 	ctx := context.TODO()
 
@@ -100,6 +104,10 @@ func TestIoCopy(t *testing.T) {
 	testBytes2 := []byte("Test2")
 	testBytes3 := []byte("Test3")
 
+	testDir, err := ioutil.TempDir("", "kata-")
+	assert.NoError(err)
+	defer os.RemoveAll(testDir)
+
 	fifoPath, err := ioutil.TempDir(testDir, "fifo-path-")
 	assert.NoError(err)
 	dstStdoutPath := filepath.Join(fifoPath, "dststdout")
@@ -229,6 +237,18 @@ func TestIoCopy(t *testing.T) {
 			}
 		}
 
+		// check everything works without closed pipes
+		checkFifoWrite(firstW, testBytes1, first)
+		checkFifoRead(firstR, testBytes1, first)
+
+		checkFifoWrite(secondW, testBytes2, second)
+		checkFifoRead(secondR, testBytes2, second)
+
+		if thirdW != nil {
+			checkFifoWrite(thirdW, testBytes3, third)
+			checkFifoRead(thirdR, testBytes3, third)
+		}
+
 		// write to each pipe, and close them immediately
 		// the ioCopy function should copy the data, then stop the corresponding thread
 		checkFifoWrite(firstW, testBytes1, first)

src/runtime/pkg/containerd-shim-v2/utils_test.go

@@ -7,46 +7,28 @@
 package containerdshim
 
 import (
-	"encoding/json"
-	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
-	sysExec "os/exec"
 	"path"
-	"path/filepath"
-	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 
-	"github.com/opencontainers/runtime-spec/specs-go"
-
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/vcmock"
+	"github.com/pkg/errors"
 )
 
 const (
-	// specConf is the name of the file holding the containers configuration
-	specConf = "config.json"
-
 	TestID = "container_test"
 
-	testDirMode  = os.FileMode(0750)
 	testFileMode = os.FileMode(0640)
-	// testExeFileMode         = os.FileMode(0750)
-
-	// small docker image used to create root filesystems from
-	testDockerImage = "busybox"
 
 	testSandboxID   = "777-77-77777777"
 	testContainerID = "42"
-	testBundle      = "bundle"
 	testConsole     = "/dev/pts/888"
 
 	testContainerTypeAnnotation = "io.kubernetes.cri.container-type"
@@ -57,10 +39,7 @@ const (
 
 var (
 	// package variables set by calling TestMain()
-	testDir       = ""
-	testBundleDir = ""
-	tc            ktu.TestConstraint
-	ctrEngine     = katautils.CtrEngine{}
+	tc ktu.TestConstraint
 )
 
 // testingImpl is a concrete mock RVC implementation used for testing
@@ -73,57 +52,6 @@ func init() {
 	fmt.Printf("INFO: switching to fake virtcontainers implementation for testing\n")
 	vci = testingImpl
 
-	var err error
-
-	fmt.Printf("INFO: creating test directory\n")
-	testDir, err = ioutil.TempDir("", "shimV2-")
-	if err != nil {
-		panic(fmt.Sprintf("ERROR: failed to create test directory: %v", err))
-	}
-	fmt.Printf("INFO: test directory is %v\n", testDir)
-
-	var output string
-	for _, name := range katautils.DockerLikeCtrEngines {
-		fmt.Printf("INFO: checking for container engine: %s\n", name)
-
-		output, err = ctrEngine.Init(name)
-		if err == nil {
-			break
-		}
-	}
-
-	if ctrEngine.Name == "" {
-		panic(fmt.Sprintf("ERROR: Docker-like container engine not accessible to current user: %v (error %v)",
-			output, err))
-	}
-
-	// Do this now to avoid hitting the test timeout value due to
-	// slow network response.
-	fmt.Printf("INFO: ensuring required container image (%v) is available\n", testDockerImage)
-	// Only hit the network if the image doesn't exist locally
-	_, err = ctrEngine.Inspect(testDockerImage)
-	if err == nil {
-		fmt.Printf("INFO: container image %v already exists locally\n", testDockerImage)
-	} else {
-		fmt.Printf("INFO: pulling container image %v\n", testDockerImage)
-		_, err = ctrEngine.Pull(testDockerImage)
-		if err != nil {
-			panic(err)
-		}
-	}
-
-	testBundleDir = filepath.Join(testDir, testBundle)
-	err = os.MkdirAll(testBundleDir, testDirMode)
-	if err != nil {
-		panic(fmt.Sprintf("ERROR: failed to create bundle directory %v: %v", testBundleDir, err))
-	}
-
-	fmt.Printf("INFO: creating OCI bundle in %v for tests to use\n", testBundleDir)
-	err = realMakeOCIBundle(testBundleDir)
-	if err != nil {
-		panic(fmt.Sprintf("ERROR: failed to create OCI bundle: %v", err))
-	}
-
 	tc = ktu.NewTestConstraint(false)
 
 	// disable shim management server.
@@ -131,46 +59,6 @@ func init() {
 	defaultStartManagementServerFunc = nil
 }
 
-// createOCIConfig creates an OCI configuration (spec) file in
-// the bundle directory specified (which must exist).
-func createOCIConfig(bundleDir string) error {
-	if bundleDir == "" {
-		return errors.New("BUG: Need bundle directory")
-	}
-
-	if !katautils.FileExists(bundleDir) {
-		return fmt.Errorf("BUG: Bundle directory %s does not exist", bundleDir)
-	}
-
-	var configCmd string
-
-	// Search for a suitable version of runc to use to generate
-	// the OCI config file.
-	for _, cmd := range []string{"docker-runc", "runc"} {
-		fullPath, err := sysExec.LookPath(cmd)
-		if err == nil {
-			configCmd = fullPath
-			break
-		}
-	}
-
-	if configCmd == "" {
-		return errors.New("Cannot find command to generate OCI config file")
-	}
-
-	_, err := utils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
-	if err != nil {
-		return err
-	}
-
-	specFile := filepath.Join(bundleDir, specConf)
-	if !katautils.FileExists(specFile) {
-		return fmt.Errorf("generated OCI config file does not exist: %v", specFile)
-	}
-
-	return nil
-}
-
 func createEmptyFile(path string) (err error) {
 	return ioutil.WriteFile(path, []byte(""), testFileMode)
 }
@@ -221,119 +109,6 @@ func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConf
 	}, nil
 }
 
-// realMakeOCIBundle will create an OCI bundle (including the "config.json"
-// config file) in the directory specified (which must already exist).
-//
-// XXX: Note that tests should *NOT* call this function - they should
-// XXX: instead call makeOCIBundle().
-func realMakeOCIBundle(bundleDir string) error {
-	if bundleDir == "" {
-		return errors.New("BUG: Need bundle directory")
-	}
-
-	if !katautils.FileExists(bundleDir) {
-		return fmt.Errorf("BUG: Bundle directory %v does not exist", bundleDir)
-	}
-
-	err := createOCIConfig(bundleDir)
-	if err != nil {
-		return err
-	}
-
-	// Note the unusual parameter (a directory, not the config
-	// file to parse!)
-	spec, err := compatoci.ParseConfigJSON(bundleDir)
-	if err != nil {
-		return err
-	}
-
-	// Determine the rootfs directory name the OCI config refers to
-	ociRootPath := spec.Root.Path
-
-	rootfsDir := filepath.Join(bundleDir, ociRootPath)
-
-	if strings.HasPrefix(ociRootPath, "/") {
-		return fmt.Errorf("Cannot handle absolute rootfs as bundle must be unique to each test")
-	}
-
-	err = createRootfs(rootfsDir)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// Create an OCI bundle in the specified directory.
-//
-// Note that the directory will be created, but it's parent is expected to exist.
-//
-// This function works by copying the already-created test bundle. Ideally,
-// the bundle would be recreated for each test, but createRootfs() uses
-// docker which on some systems is too slow, resulting in the tests timing
-// out.
-func makeOCIBundle(bundleDir string) error {
-	from := testBundleDir
-	to := bundleDir
-
-	// only the basename of bundleDir needs to exist as bundleDir
-	// will get created by cp(1).
-	base := filepath.Dir(bundleDir)
-
-	for _, dir := range []string{from, base} {
-		if !katautils.FileExists(dir) {
-			return fmt.Errorf("BUG: directory %v should exist", dir)
-		}
-	}
-
-	output, err := utils.RunCommandFull([]string{"cp", "-a", from, to}, true)
-	if err != nil {
-		return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output)
-	}
-
-	return nil
-}
-
-// createRootfs creates a minimal root filesystem below the specified
-// directory.
-func createRootfs(dir string) error {
-	err := os.MkdirAll(dir, testDirMode)
-	if err != nil {
-		return err
-	}
-
-	container, err := ctrEngine.Create(testDockerImage)
-	if err != nil {
-		return err
-	}
-
-	err = ctrEngine.GetRootfs(container, dir)
-	if err != nil {
-		return err
-	}
-
-	// Clean up
-	_, err = ctrEngine.Rm(container)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func writeOCIConfigFile(spec specs.Spec, configPath string) error {
-	if configPath == "" {
-		return errors.New("BUG: need config file path")
-	}
-
-	bytes, err := json.MarshalIndent(spec, "", "\t")
-	if err != nil {
-		return err
-	}
-
-	return ioutil.WriteFile(configPath, bytes, testFileMode)
-}
-
 func TestNoNeedForOutput(t *testing.T) {
 	assert := assert.New(t)
 

src/runtime/pkg/katatestutils/constraints.go

@@ -147,8 +147,15 @@ func getDistroDetails() (name, version string, err error) {
 //   centos: 3.10.0-957.12.1.el7.x86_64
 //   fedora: 5.0.9-200.fc29.x86_64
 //
+// For some self compiled kernel, the kernel version will be with "+" as its suffix
+// For example:
+//   5.12.0-rc4+
+// These kernel version can't be parsed by the current lib and lead to panic
+// therefore the '+' should be removed.
+//
 func fixKernelVersion(version string) string {
-	return strings.Replace(version, "_", "-", -1)
+	version = strings.Replace(version, "_", "-", -1)
+	return strings.Replace(version, "+", "", -1)
 }
 
 // handleDistroName checks that the current distro is compatible with

src/runtime/pkg/katatestutils/constraints_test.go

@@ -21,7 +21,6 @@ import (
 )
 
 const (
-	testFileMode    = os.FileMode(0640)
 	invalidOperator = 1234
 
 	skipUnknownDistroName = "skipping test as cannot determine distro name"

src/runtime/pkg/katatestutils/utils.go

@@ -7,8 +7,203 @@
 package katatestutils
 
 import (
+	"encoding/json"
+	"errors"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strconv"
+	"testing"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	testDirMode  = os.FileMode(0750)
+	testFileMode = os.FileMode(0640)
+
+	busyboxConfigJson = `
+{
+	"ociVersion": "1.0.1-dev",
+	"process": {
+		"terminal": true,
+		"user": {
+			"uid": 0,
+			"gid": 0
+		},
+		"args": [
+			"sh"
+		],
+		"env": [
+			"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+			"TERM=xterm"
+		],
+		"cwd": "/",
+		"capabilities": {
+			"bounding": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			],
+			"effective": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			],
+			"inheritable": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			],
+			"permitted": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			],
+			"ambient": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			]
+		},
+		"rlimits": [
+			{
+				"type": "RLIMIT_NOFILE",
+				"hard": 1024,
+				"soft": 1024
+			}
+		],
+		"noNewPrivileges": true
+	},
+	"root": {
+		"path": "rootfs",
+		"readonly": true
+	},
+	"hostname": "runc",
+	"mounts": [
+		{
+			"destination": "/proc",
+			"type": "proc",
+			"source": "proc"
+		},
+		{
+			"destination": "/dev",
+			"type": "tmpfs",
+			"source": "tmpfs",
+			"options": [
+				"nosuid",
+				"strictatime",
+				"mode=755",
+				"size=65536k"
+			]
+		},
+		{
+			"destination": "/dev/pts",
+			"type": "devpts",
+			"source": "devpts",
+			"options": [
+				"nosuid",
+				"noexec",
+				"newinstance",
+				"ptmxmode=0666",
+				"mode=0620",
+				"gid=5"
+			]
+		},
+		{
+			"destination": "/dev/shm",
+			"type": "tmpfs",
+			"source": "shm",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev",
+				"mode=1777",
+				"size=65536k"
+			]
+		},
+		{
+			"destination": "/dev/mqueue",
+			"type": "mqueue",
+			"source": "mqueue",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev"
+			]
+		},
+		{
+			"destination": "/sys",
+			"type": "sysfs",
+			"source": "sysfs",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev",
+				"ro"
+			]
+		},
+		{
+			"destination": "/sys/fs/cgroup",
+			"type": "cgroup",
+			"source": "cgroup",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev",
+				"relatime",
+				"ro"
+			]
+		}
+	],
+	"linux": {
+		"resources": {
+			"devices": [
+				{
+					"allow": false,
+					"access": "rwm"
+				}
+			]
+		},
+		"namespaces": [
+			{
+				"type": "pid"
+			},
+			{
+				"type": "network"
+			},
+			{
+				"type": "ipc"
+			},
+			{
+				"type": "uts"
+			},
+			{
+				"type": "mount"
+			}
+		],
+		"maskedPaths": [
+			"/proc/acpi",
+			"/proc/asound",
+			"/proc/kcore",
+			"/proc/keys",
+			"/proc/latency_stats",
+			"/proc/timer_list",
+			"/proc/timer_stats",
+			"/proc/sched_debug",
+			"/sys/firmware",
+			"/proc/scsi"
+		],
+		"readonlyPaths": [
+			"/proc/bus",
+			"/proc/fs",
+			"/proc/irq",
+			"/proc/sys",
+			"/proc/sysrq-trigger"
+		]
+	}
+}`
 )
 
 type RuntimeConfigOptions struct {
@@ -23,8 +218,6 @@ type RuntimeConfigOptions struct {
 	NetmonPath           string
 	LogPath              string
 	BlockDeviceDriver    string
-	AgentTraceMode       string
-	AgentTraceType       string
 	SharedFS             string
 	VirtioFSDaemon       string
 	JaegerEndpoint       string
@@ -137,8 +330,6 @@ func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string {
 	[agent.kata]
 	enable_debug = ` + strconv.FormatBool(config.AgentDebug) + `
 	enable_tracing = ` + strconv.FormatBool(config.AgentTrace) + `
-	trace_mode = "` + config.AgentTraceMode + `"` + `
-	trace_type = "` + config.AgentTraceType + `"` + `
 
 	[netmon]
 	path = "` + config.NetmonPath + `"
@@ -158,3 +349,34 @@ func IsInGitHubActions() bool {
 	// https://docs.github.com/en/actions/reference/environment-variables#default-environment-variables
 	return os.Getenv("GITHUB_ACTIONS") == "true"
 }
+
+func SetupOCIConfigFile(t *testing.T) (rootPath string, bundlePath, ociConfigFile string) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "katatest-")
+	assert.NoError(err)
+
+	bundlePath = filepath.Join(tmpdir, "bundle")
+	err = os.MkdirAll(bundlePath, testDirMode)
+	assert.NoError(err)
+
+	ociConfigFile = filepath.Join(bundlePath, "config.json")
+	err = ioutil.WriteFile(ociConfigFile, []byte(busyboxConfigJson), testFileMode)
+	assert.NoError(err)
+
+	return tmpdir, bundlePath, ociConfigFile
+}
+
+// WriteOCIConfigFile using spec to update OCI config file by path configPath
+func WriteOCIConfigFile(spec specs.Spec, configPath string) error {
+	if configPath == "" {
+		return errors.New("BUG: need config file path")
+	}
+
+	bytes, err := json.MarshalIndent(spec, "", "\t")
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(configPath, bytes, testFileMode)
+}

src/runtime/pkg/katautils/config-settings.go.in

@@ -87,6 +87,8 @@ const defaultTxRateLimiterMaxRate = uint64(0)
 const defaultConfidentialGuest = false
 const defaultGuestSwap = false
 const defaultRootlessHypervisor = false
+const defaultDisableSeccomp = false
+const defaultVfioMode = "guest-kernel"
 
 var defaultSGXEPCSize = int64(0)
 

src/runtime/pkg/katautils/config.go

@@ -135,6 +135,7 @@ type hypervisor struct {
 	ConfidentialGuest       bool     `toml:"confidential_guest"`
 	GuestSwap               bool     `toml:"enable_guest_swap"`
 	Rootless                bool     `toml:"rootless"`
+	DisableSeccomp          bool     `toml:"disable_seccomp"`
 }
 
 type runtime struct {
@@ -142,6 +143,7 @@ type runtime struct {
 	JaegerEndpoint      string   `toml:"jaeger_endpoint"`
 	JaegerUser          string   `toml:"jaeger_user"`
 	JaegerPassword      string   `toml:"jaeger_password"`
+	VfioMode            string   `toml:"vfio_mode"`
 	SandboxBindMounts   []string `toml:"sandbox_bind_mounts"`
 	Experimental        []string `toml:"experimental"`
 	Debug               bool     `toml:"enable_debug"`
@@ -153,8 +155,6 @@ type runtime struct {
 }
 
 type agent struct {
-	TraceMode           string   `toml:"trace_mode"`
-	TraceType           string   `toml:"trace_type"`
 	KernelModules       []string `toml:"kernel_modules"`
 	Debug               bool     `toml:"enable_debug"`
 	Tracing             bool     `toml:"enable_tracing"`
@@ -308,11 +308,24 @@ func (h hypervisor) GetEntropySource() string {
 	return h.EntropySource
 }
 
+// Current cpu number should not larger than defaultMaxVCPUs()
+func getCurrentCpuNum() uint32 {
+	var cpu uint32
+	h := hypervisor{}
+
+	cpu = uint32(goruntime.NumCPU())
+	if cpu > h.defaultMaxVCPUs() {
+		cpu = h.defaultMaxVCPUs()
+	}
+
+	return cpu
+}
+
 func (h hypervisor) defaultVCPUs() uint32 {
-	numCPUs := goruntime.NumCPU()
+	numCPUs := getCurrentCpuNum()
 
 	if h.NumVCPUs < 0 || h.NumVCPUs > int32(numCPUs) {
-		return uint32(numCPUs)
+		return numCPUs
 	}
 	if h.NumVCPUs == 0 { // or unspecified
 		return defaultVCPUCount
@@ -489,14 +502,6 @@ func (a agent) trace() bool {
 	return a.Tracing
 }
 
-func (a agent) traceMode() string {
-	return a.TraceMode
-}
-
-func (a agent) traceType() string {
-	return a.TraceType
-}
-
 func (a agent) kernelModules() []string {
 	return a.KernelModules
 }
@@ -875,6 +880,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		VirtioFSExtraArgs:       h.VirtioFSExtraArgs,
 		SGXEPCSize:              defaultSGXEPCSize,
 		EnableAnnotations:       h.EnableAnnotations,
+		DisableSeccomp:          h.DisableSeccomp,
 	}, nil
 }
 
@@ -929,8 +935,6 @@ func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oc
 			LongLiveConn:       true,
 			Debug:              agent.debug(),
 			Trace:              agent.trace(),
-			TraceMode:          agent.traceMode(),
-			TraceType:          agent.traceType(),
 			KernelModules:      agent.kernelModules(),
 			EnableDebugConsole: agent.debugConsoleEnabled(),
 			DialTimeout:        agent.dialTimout(),
@@ -976,10 +980,6 @@ func SetKernelParams(runtimeConfig *oci.RuntimeConfig) error {
 	}
 
 	// next, check for agent specific kernel params
-	err := vc.KataAgentSetDefaultTraceConfigOptions(&runtimeConfig.AgentConfig)
-	if err != nil {
-		return err
-	}
 
 	params := vc.KataAgentKernelParams(runtimeConfig.AgentConfig)
 
@@ -1072,6 +1072,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
 		ConfidentialGuest:       defaultConfidentialGuest,
 		GuestSwap:               defaultGuestSwap,
 		Rootless:                defaultRootlessHypervisor,
+		DisableSeccomp:          defaultDisableSeccomp,
 	}
 }
 
@@ -1081,6 +1082,11 @@ func initConfig() (config oci.RuntimeConfig, err error) {
 		return oci.RuntimeConfig{}, err
 	}
 
+	err = config.VfioMode.VFIOSetMode(defaultVfioMode)
+	if err != nil {
+		return oci.RuntimeConfig{}, err
+	}
+
 	config = oci.RuntimeConfig{
 		HypervisorType:   defaultHypervisor,
 		HypervisorConfig: GetDefaultHypervisorConfig(),
@@ -1127,6 +1133,14 @@ func LoadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat
 		}
 	}
 
+	if tomlConf.Runtime.VfioMode != "" {
+		err = config.VfioMode.VFIOSetMode(tomlConf.Runtime.VfioMode)
+
+		if err != nil {
+			return "", config, err
+		}
+	}
+
 	if !ignoreLogging {
 		err := handleSystemLog("", "")
 		if err != nil {

src/runtime/pkg/katautils/config_test.go

@@ -14,7 +14,6 @@ import (
 	"path"
 	"path/filepath"
 	"reflect"
-	goruntime "runtime"
 	"strings"
 	"syscall"
 	"testing"
@@ -156,7 +155,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		KernelParams:          vc.DeserializeParams(strings.Fields(kernelParams)),
 		HypervisorMachineType: machineType,
 		NumVCPUs:              defaultVCPUCount,
-		DefaultMaxVCPUs:       uint32(goruntime.NumCPU()),
+		DefaultMaxVCPUs:       getCurrentCpuNum(),
 		MemorySize:            defaultMemSize,
 		DisableBlockDeviceUse: disableBlockDevice,
 		BlockDeviceDriver:     defaultBlockDeviceDriver,
@@ -919,13 +918,13 @@ func TestNewClhHypervisorConfig(t *testing.T) {
 func TestHypervisorDefaults(t *testing.T) {
 	assert := assert.New(t)
 
-	numCPUs := goruntime.NumCPU()
+	numCPUs := getCurrentCpuNum()
 
 	h := hypervisor{}
 
 	assert.Equal(h.machineType(), defaultMachineType, "default hypervisor machine type wrong")
 	assert.Equal(h.defaultVCPUs(), defaultVCPUCount, "default vCPU number is wrong")
-	assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong")
+	assert.Equal(h.defaultMaxVCPUs(), numCPUs, "default max vCPU number is wrong")
 	assert.Equal(h.defaultMemSz(), defaultMemSize, "default memory size is wrong")
 
 	machineType := "foo"
@@ -934,23 +933,23 @@ func TestHypervisorDefaults(t *testing.T) {
 
 	// auto inferring
 	h.NumVCPUs = -1
-	assert.Equal(h.defaultVCPUs(), uint32(numCPUs), "default vCPU number is wrong")
+	assert.Equal(h.defaultVCPUs(), numCPUs, "default vCPU number is wrong")
 
 	h.NumVCPUs = 2
 	assert.Equal(h.defaultVCPUs(), uint32(2), "default vCPU number is wrong")
 
 	h.NumVCPUs = int32(numCPUs) + 1
-	assert.Equal(h.defaultVCPUs(), uint32(numCPUs), "default vCPU number is wrong")
+	assert.Equal(h.defaultVCPUs(), numCPUs, "default vCPU number is wrong")
 
 	h.DefaultMaxVCPUs = 2
 	assert.Equal(h.defaultMaxVCPUs(), uint32(2), "default max vCPU number is wrong")
 
-	h.DefaultMaxVCPUs = uint32(numCPUs) + 1
-	assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong")
+	h.DefaultMaxVCPUs = numCPUs + 1
+	assert.Equal(h.defaultMaxVCPUs(), numCPUs, "default max vCPU number is wrong")
 
 	maxvcpus := vc.MaxQemuVCPUs()
 	h.DefaultMaxVCPUs = maxvcpus + 1
-	assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong")
+	assert.Equal(h.defaultMaxVCPUs(), numCPUs, "default max vCPU number is wrong")
 
 	h.MemorySize = 1024
 	assert.Equal(h.defaultMemSz(), uint32(1024), "default memory size is wrong")
@@ -1152,9 +1151,6 @@ func TestAgentDefaults(t *testing.T) {
 
 	a.Tracing = true
 	assert.Equal(a.trace(), a.Tracing)
-
-	assert.Equal(a.traceMode(), a.TraceMode)
-	assert.Equal(a.traceType(), a.TraceType)
 }
 
 func TestGetDefaultConfigFilePaths(t *testing.T) {

src/runtime/pkg/katautils/container_engine.go

@@ -1,77 +0,0 @@
-// Copyright (c) 2019 SUSE LLC
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-package katautils
-
-import (
-	"os/exec"
-
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
-)
-
-type CtrEngine struct {
-	Name string
-}
-
-var (
-	DockerLikeCtrEngines = []string{"docker", "podman"}
-)
-
-func (e *CtrEngine) Init(name string) (string, error) {
-	var out string
-	out, err := utils.RunCommandFull([]string{name, "version"}, true)
-	if err != nil {
-		return out, err
-	}
-
-	e.Name = name
-	return out, nil
-}
-
-func (e *CtrEngine) Inspect(image string) (string, error) {
-	// Only hit the network if the image doesn't exist locally
-	return utils.RunCommand([]string{e.Name, "inspect", "--type=image", image})
-}
-
-func (e *CtrEngine) Pull(image string) (string, error) {
-	return utils.RunCommand([]string{e.Name, "pull", image})
-}
-
-func (e *CtrEngine) Create(image string) (string, error) {
-	return utils.RunCommand([]string{e.Name, "create", image})
-}
-
-func (e *CtrEngine) Rm(ctrID string) (string, error) {
-	return utils.RunCommand([]string{e.Name, "rm", ctrID})
-}
-
-func (e *CtrEngine) GetRootfs(ctrID string, dir string) error {
-	cmd1 := exec.Command(e.Name, "export", ctrID)
-	cmd2 := exec.Command("tar", "-C", dir, "-xvf", "-")
-
-	cmd1Stdout, err := cmd1.StdoutPipe()
-	if err != nil {
-		return err
-	}
-
-	cmd2.Stdin = cmd1Stdout
-
-	err = cmd2.Start()
-	if err != nil {
-		return err
-	}
-
-	err = cmd1.Run()
-	if err != nil {
-		return err
-	}
-
-	err = cmd2.Wait()
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

src/runtime/pkg/katautils/create.go

@@ -112,7 +112,7 @@ func SetEphemeralStorageType(ociSpec specs.Spec) specs.Spec {
 func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec specs.Spec, runtimeConfig oci.RuntimeConfig, rootFs vc.RootFs,
 	containerID, bundlePath, console string, disableOutput, systemdCgroup bool) (_ vc.VCSandbox, _ vc.Process, err error) {
 	span, ctx := katatrace.Trace(ctx, nil, "CreateSandbox", createTracingTags)
-	katatrace.AddTag(span, "container_id", containerID)
+	katatrace.AddTags(span, "container_id", containerID)
 	defer span.End()
 
 	sandboxConfig, err := oci.SandboxConfig(ociSpec, runtimeConfig, bundlePath, containerID, console, disableOutput, systemdCgroup)
@@ -167,7 +167,7 @@ func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec specs.Spec, runtimeCo
 
 	sid := sandbox.ID()
 	kataUtilsLogger = kataUtilsLogger.WithField("sandbox", sid)
-	katatrace.AddTag(span, "sandbox_id", sid)
+	katatrace.AddTags(span, "sandbox_id", sid)
 
 	containers := sandbox.GetAllContainers()
 	if len(containers) != 1 {
@@ -211,7 +211,7 @@ func CreateContainer(ctx context.Context, sandbox vc.VCSandbox, ociSpec specs.Sp
 	var c vc.VCContainer
 
 	span, ctx := katatrace.Trace(ctx, nil, "CreateContainer", createTracingTags)
-	katatrace.AddTag(span, "container_id", containerID)
+	katatrace.AddTags(span, "container_id", containerID)
 	defer span.End()
 
 	ociSpec = SetEphemeralStorageType(ociSpec)
@@ -237,7 +237,7 @@ func CreateContainer(ctx context.Context, sandbox vc.VCSandbox, ociSpec specs.Sp
 		return vc.Process{}, err
 	}
 
-	katatrace.AddTag(span, "sandbox_id", sandboxID)
+	katatrace.AddTags(span, "sandbox_id", sandboxID)
 
 	c, err = sandbox.CreateContainer(ctx, contConfig)
 	if err != nil {

src/runtime/pkg/katautils/create_test.go

@@ -8,7 +8,6 @@ package katautils
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -20,7 +19,6 @@ import (
 	"testing"
 
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
@@ -37,8 +35,6 @@ const (
 )
 
 var (
-	testBundleDir = ""
-
 	// testingImpl is a concrete mock RVC implementation used for testing
 	testingImpl = &vcmock.VCMock{}
 	// mock sandbox
@@ -53,49 +49,6 @@ func init() {
 	tc = ktu.NewTestConstraint(false)
 }
 
-func writeOCIConfigFile(spec specs.Spec, configPath string) error {
-	if configPath == "" {
-		return errors.New("BUG: need config file path")
-	}
-
-	bytes, err := json.MarshalIndent(spec, "", "\t")
-	if err != nil {
-		return err
-	}
-
-	return ioutil.WriteFile(configPath, bytes, testFileMode)
-}
-
-// Create an OCI bundle in the specified directory.
-//
-// Note that the directory will be created, but it's parent is expected to exist.
-//
-// This function works by copying the already-created test bundle. Ideally,
-// the bundle would be recreated for each test, but createRootfs() uses
-// docker which on some systems is too slow, resulting in the tests timing
-// out.
-func makeOCIBundle(bundleDir string) error {
-	from := testBundleDir
-	to := bundleDir
-
-	// only the basename of bundleDir needs to exist as bundleDir
-	// will get created by cp(1).
-	base := filepath.Dir(bundleDir)
-
-	for _, dir := range []string{from, base} {
-		if !FileExists(dir) {
-			return fmt.Errorf("BUG: directory %v should exist", dir)
-		}
-	}
-
-	output, err := utils.RunCommandFull([]string{"cp", "-a", from, to}, true)
-	if err != nil {
-		return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output)
-	}
-
-	return nil
-}
-
 // newTestRuntimeConfig creates a new RuntimeConfig
 func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConfig, error) {
 	if dir == "" {
@@ -262,21 +215,12 @@ func TestSetKernelParamsUserOptionTakesPriority(t *testing.T) {
 func TestCreateSandboxConfigFail(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, _ := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -305,21 +249,12 @@ func TestCreateSandboxFail(t *testing.T) {
 
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, _ := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -375,18 +310,9 @@ func TestCheckForFips(t *testing.T) {
 func TestCreateContainerContainerConfigFail(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -396,7 +322,7 @@ func TestCreateContainerContainerConfigFail(t *testing.T) {
 	spec.Annotations[testContainerTypeAnnotation] = containerType
 
 	// rewrite file
-	err = writeOCIConfigFile(spec, ociConfigFile)
+	err = ktu.WriteOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
 	rootFs := vc.RootFs{Mounted: true}
@@ -412,18 +338,9 @@ func TestCreateContainerContainerConfigFail(t *testing.T) {
 func TestCreateContainerFail(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -433,7 +350,7 @@ func TestCreateContainerFail(t *testing.T) {
 	spec.Annotations[testSandboxIDAnnotation] = testSandboxID
 
 	// rewrite file
-	err = writeOCIConfigFile(spec, ociConfigFile)
+	err = ktu.WriteOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
 	rootFs := vc.RootFs{Mounted: true}
@@ -456,18 +373,9 @@ func TestCreateContainer(t *testing.T) {
 		mockSandbox.CreateContainerFunc = nil
 	}()
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -477,7 +385,7 @@ func TestCreateContainer(t *testing.T) {
 	spec.Annotations[testSandboxIDAnnotation] = testSandboxID
 
 	// rewrite file
-	err = writeOCIConfigFile(spec, ociConfigFile)
+	err = ktu.WriteOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
 	rootFs := vc.RootFs{Mounted: true}

src/runtime/pkg/katautils/hook.go

@@ -35,11 +35,7 @@ func hookLogger() *logrus.Entry {
 func runHook(ctx context.Context, hook specs.Hook, cid, bundlePath string) error {
 	span, _ := katatrace.Trace(ctx, hookLogger(), "runHook", hookTracingTags)
 	defer span.End()
-
-	// FIXME
-	// span.LogFields(
-	// 	log.String("hook-name", hook.Path),
-	// 	log.String("hook-args", strings.Join(hook.Args, " ")))
+	katatrace.AddTags(span, "path", hook.Path, "args", hook.Args)
 
 	state := specs.State{
 		Pid:    syscall.Gettid(),
@@ -96,7 +92,7 @@ func runHook(ctx context.Context, hook specs.Hook, cid, bundlePath string) error
 
 func runHooks(ctx context.Context, hooks []specs.Hook, cid, bundlePath, hookType string) error {
 	span, ctx := katatrace.Trace(ctx, hookLogger(), "runHooks", hookTracingTags)
-	katatrace.AddTag(span, "type", hookType)
+	katatrace.AddTags(span, "type", hookType)
 	defer span.End()
 
 	for _, hook := range hooks {

src/runtime/pkg/katautils/katatrace/tracing.go

@@ -7,14 +7,16 @@ package katatrace
 
 import (
 	"context"
+	"encoding/json"
 
 	"github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel"
-	"go.opentelemetry.io/otel/exporters/trace/jaeger"
-	"go.opentelemetry.io/otel/label"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/jaeger"
 	"go.opentelemetry.io/otel/propagation"
-	export "go.opentelemetry.io/otel/sdk/export/trace"
+	"go.opentelemetry.io/otel/sdk/resource"
 	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
 	"go.opentelemetry.io/otel/trace"
 	otelTrace "go.opentelemetry.io/otel/trace"
 )
@@ -25,10 +27,10 @@ import (
 // https: //github.com/kata-containers/tests/blob/master/tracing/tracing-test.sh
 type kataSpanExporter struct{}
 
-var _ export.SpanExporter = (*kataSpanExporter)(nil)
+var _ sdktrace.SpanExporter = (*kataSpanExporter)(nil)
 
 // ExportSpans exports SpanData to Jaeger.
-func (e *kataSpanExporter) ExportSpans(ctx context.Context, spans []*export.SpanData) error {
+func (e *kataSpanExporter) ExportSpans(ctx context.Context, spans []sdktrace.ReadOnlySpan) error {
 	for _, span := range spans {
 		kataTraceLogger.Tracef("Reporting span %+v", span)
 	}
@@ -39,9 +41,9 @@ func (e *kataSpanExporter) Shutdown(ctx context.Context) error {
 	return nil
 }
 
-// tracerCloser contains a copy of the closer returned by createTracer() which
-// is used by stopTracing().
-var tracerCloser func()
+// tp is the trace provider created in CreateTracer() and used in StopTracing()
+// to flush and shutdown all spans.
+var tp *sdktrace.TracerProvider
 
 var kataTraceLogger = logrus.NewEntry(logrus.New())
 
@@ -61,10 +63,10 @@ type JaegerConfig struct {
 }
 
 // CreateTracer create a tracer
-func CreateTracer(name string, config *JaegerConfig) (func(), error) {
+func CreateTracer(name string, config *JaegerConfig) (*sdktrace.TracerProvider, error) {
 	if !tracing {
 		otel.SetTracerProvider(trace.NewNoopTracerProvider())
-		return func() {}, nil
+		return nil, nil
 	}
 
 	// build kata exporter to log reporting span records
@@ -76,37 +78,32 @@ func CreateTracer(name string, config *JaegerConfig) (func(), error) {
 		collectorEndpoint = "http://localhost:14268/api/traces"
 	}
 
-	jaegerExporter, err := jaeger.NewRawExporter(
-		jaeger.WithCollectorEndpoint(collectorEndpoint,
+	jaegerExporter, err := jaeger.New(
+		jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(collectorEndpoint),
 			jaeger.WithUsername(config.JaegerUser),
 			jaeger.WithPassword(config.JaegerPassword),
-		), jaeger.WithProcess(jaeger.Process{
-			ServiceName: name,
-			Tags: []label.KeyValue{
-				label.String("exporter", "jaeger"),
-				label.String("lib", "opentelemetry"),
-			},
-		}))
+		),
+	)
+
 	if err != nil {
 		return nil, err
 	}
 
 	// build tracer provider, that combining both jaeger exporter and kata exporter.
 	tp := sdktrace.NewTracerProvider(
-		sdktrace.WithConfig(
-			sdktrace.Config{
-				DefaultSampler: sdktrace.AlwaysSample(),
-			},
-		),
+		sdktrace.WithSampler(sdktrace.AlwaysSample()),
 		sdktrace.WithSyncer(kataExporter),
 		sdktrace.WithSyncer(jaegerExporter),
+		sdktrace.WithResource(resource.NewSchemaless(
+			semconv.ServiceNameKey.String(name),
+			attribute.String("exporter", "jaeger"),
+			attribute.String("lib", "opentelemetry"),
+		)),
 	)
 
-	tracerCloser = jaegerExporter.Flush
-
 	otel.SetTracerProvider(tp)
 	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))
-	return tracerCloser, nil
+	return tp, nil
 }
 
 // StopTracing ends all tracing, reporting the spans to the collector.
@@ -121,9 +118,8 @@ func StopTracing(ctx context.Context) {
 	}
 
 	// report all possible spans to the collector
-	if tracerCloser != nil {
-		tracerCloser()
-	}
+	tp.ForceFlush(ctx)
+	tp.Shutdown(ctx)
 }
 
 // Trace creates a new tracing span based on the specified name and parent context.
@@ -138,12 +134,12 @@ func Trace(parent context.Context, logger *logrus.Entry, name string, tags ...ma
 		parent = context.Background()
 	}
 
-	var otelTags []label.KeyValue
+	var otelTags []attribute.KeyValue
 	// do not append tags if tracing is disabled
 	if tracing {
 		for _, tagSet := range tags {
 			for k, v := range tagSet {
-				otelTags = append(otelTags, label.Key(k).String(v))
+				otelTags = append(otelTags, attribute.Key(k).String(v))
 			}
 		}
 	}
@@ -163,8 +159,63 @@ func Trace(parent context.Context, logger *logrus.Entry, name string, tags ...ma
 	return span, ctx
 }
 
-// AddTag adds an additional key-value pair to a tracing span. This can be used to
-// provide dynamic tags that are determined at runtime.
-func AddTag(span otelTrace.Span, key string, value interface{}) {
-	span.SetAttributes(label.Any(key, value))
+func addTag(span otelTrace.Span, key string, value interface{}) {
+	// do not append tags if tracing is disabled
+	if !tracing {
+		return
+	}
+	if value == nil {
+		span.SetAttributes(attribute.String(key, "nil"))
+		return
+	}
+
+	switch value := value.(type) {
+	case string:
+		span.SetAttributes(attribute.String(key, value))
+	case bool:
+		span.SetAttributes(attribute.Bool(key, value))
+	case int:
+		span.SetAttributes(attribute.Int(key, value))
+	case int8:
+		span.SetAttributes(attribute.Int(key, int(value)))
+	case int16:
+		span.SetAttributes(attribute.Int(key, int(value)))
+	case int64:
+		span.SetAttributes(attribute.Int64(key, value))
+	case float64:
+		span.SetAttributes(attribute.Float64(key, value))
+	default:
+		content, err := json.Marshal(value)
+		if content == nil && err == nil {
+			span.SetAttributes(attribute.String(key, "nil"))
+		} else if content != nil && err == nil {
+			span.SetAttributes(attribute.String(key, string(content)))
+		} else {
+			kataTraceLogger.WithField("type", "bug").Error("span attribute value error")
+		}
+	}
+}
+
+// AddTag adds additional key-value pairs to a tracing span. This can be used to provide
+// dynamic tags that are determined at runtime and tags with a non-string value.
+// Must have an even number of keyValues with keys being strings.
+func AddTags(span otelTrace.Span, keyValues ...interface{}) {
+	if !tracing {
+		return
+	}
+	if len(keyValues) < 2 {
+		kataTraceLogger.WithField("type", "bug").Error("not enough inputs for attributes")
+		return
+	} else if len(keyValues)%2 != 0 {
+		kataTraceLogger.WithField("type", "bug").Error("number of attribute keyValues is not even")
+		return
+	}
+	for i := 0; i < len(keyValues); i++ {
+		if key, ok := keyValues[i].(string); ok {
+			addTag(span, key, keyValues[i+1])
+		} else {
+			kataTraceLogger.WithField("type", "bug").Error("key in attributes is not a string")
+		}
+		i++
+	}
 }

src/runtime/pkg/katautils/logger.go

@@ -20,6 +20,9 @@ import (
 var originalLoggerLevel = logrus.WarnLevel
 var kataUtilsLogger = logrus.NewEntry(logrus.New())
 
+// SYSLOGTAG is for a consistently named syslog identifier
+const SYSLOGTAG = "kata"
+
 // SetLogger sets the logger for the factory.
 func SetLogger(ctx context.Context, logger *logrus.Entry, level logrus.Level) {
 	fields := logrus.Fields{
@@ -61,7 +64,7 @@ func (h *sysLogHook) Fire(e *logrus.Entry) (err error) {
 }
 
 func newSystemLogHook(network, raddr string) (*sysLogHook, error) {
-	hook, err := lSyslog.NewSyslogHook(network, raddr, syslog.LOG_INFO, NAME)
+	hook, err := lSyslog.NewSyslogHook(network, raddr, syslog.LOG_INFO, SYSLOGTAG)
 	if err != nil {
 		return nil, err
 	}

src/runtime/pkg/katautils/utils_test.go

@@ -7,20 +7,15 @@
 package katautils
 
 import (
-	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path"
 	"path/filepath"
-	"strings"
 	"syscall"
 	"testing"
 
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -28,173 +23,14 @@ const (
 	testDirMode  = os.FileMode(0750)
 	testFileMode = os.FileMode(0640)
 
-	// small docker image used to create root filesystems from
-	testDockerImage = "busybox"
-
 	testSandboxID   = "99999999-9999-9999-99999999999999999"
 	testContainerID = "1"
-	testBundle      = "bundle"
-	specConfig      = "config.json"
 )
 
 var (
 	testDir = ""
 )
 
-func init() {
-	var err error
-
-	fmt.Printf("INFO: creating test directory\n")
-	testDir, err = ioutil.TempDir("", fmt.Sprintf("%s-", NAME))
-	if err != nil {
-		panic(fmt.Sprintf("ERROR: failed to create test directory: %v", err))
-	}
-
-	fmt.Printf("INFO: test directory is %v\n", testDir)
-
-	testBundleDir = filepath.Join(testDir, testBundle)
-	err = os.MkdirAll(testBundleDir, testDirMode)
-	if err != nil {
-		panic(fmt.Sprintf("ERROR: failed to create bundle directory %v: %v", testBundleDir, err))
-	}
-
-	fmt.Printf("INFO: creating OCI bundle in %v for tests to use\n", testBundleDir)
-	err = realMakeOCIBundle(testBundleDir)
-	if err != nil {
-		panic(fmt.Sprintf("ERROR: failed to create OCI bundle: %v", err))
-	}
-}
-
-// createOCIConfig creates an OCI configuration (spec) file in
-// the bundle directory specified (which must exist).
-func createOCIConfig(bundleDir string) error {
-	if bundleDir == "" {
-		return errors.New("BUG: Need bundle directory")
-	}
-
-	if !FileExists(bundleDir) {
-		return fmt.Errorf("BUG: Bundle directory %s does not exist", bundleDir)
-	}
-
-	var configCmd string
-
-	// Search for a suitable version of runc to use to generate
-	// the OCI config file.
-	for _, cmd := range []string{"docker-runc", "runc"} {
-		fullPath, err := exec.LookPath(cmd)
-		if err == nil {
-			configCmd = fullPath
-			break
-		}
-	}
-
-	if configCmd == "" {
-		return errors.New("Cannot find command to generate OCI config file")
-	}
-
-	_, err := utils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
-	if err != nil {
-		return err
-	}
-
-	specFile := filepath.Join(bundleDir, specConfig)
-	if !FileExists(specFile) {
-		return fmt.Errorf("generated OCI config file does not exist: %v", specFile)
-	}
-
-	return nil
-}
-
-// realMakeOCIBundle will create an OCI bundle (including the "config.json"
-// config file) in the directory specified (which must already exist).
-//
-// XXX: Note that tests should *NOT* call this function - they should
-// XXX: instead call makeOCIBundle().
-func realMakeOCIBundle(bundleDir string) error {
-	if bundleDir == "" {
-		return errors.New("BUG: Need bundle directory")
-	}
-
-	if !FileExists(bundleDir) {
-		return fmt.Errorf("BUG: Bundle directory %v does not exist", bundleDir)
-	}
-
-	err := createOCIConfig(bundleDir)
-	if err != nil {
-		return err
-	}
-
-	// Note the unusual parameter (a directory, not the config
-	// file to parse!)
-	spec, err := compatoci.ParseConfigJSON(bundleDir)
-	if err != nil {
-		return err
-	}
-
-	// Determine the rootfs directory name the OCI config refers to
-	ociRootPath := spec.Root.Path
-
-	rootfsDir := filepath.Join(bundleDir, ociRootPath)
-
-	if strings.HasPrefix(ociRootPath, "/") {
-		return fmt.Errorf("Cannot handle absolute rootfs as bundle must be unique to each test")
-	}
-
-	err = createRootfs(rootfsDir)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// createRootfs creates a minimal root filesystem below the specified
-// directory.
-func createRootfs(dir string) error {
-	var (
-		output string
-		err    error
-	)
-
-	ctrEngine := CtrEngine{}
-	for _, name := range DockerLikeCtrEngines {
-		fmt.Printf("INFO: checking for container engine: %s\n", name)
-
-		output, err = ctrEngine.Init(name)
-		if err == nil {
-			break
-		}
-	}
-
-	if ctrEngine.Name == "" {
-		panic(fmt.Sprintf("ERROR: Docker-like container engine not accessible to current user: %v (error %v)",
-			output, err))
-	}
-
-	err = os.MkdirAll(dir, testDirMode)
-	if err != nil {
-		return err
-	}
-
-	container, err := ctrEngine.Create(testDockerImage)
-	if err != nil {
-		return err
-	}
-
-	err = ctrEngine.GetRootfs(container, dir)
-	if err != nil {
-		return err
-	}
-
-	// Clean up
-	_, err = ctrEngine.Rm(container)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 func createFile(file, contents string) error {
 	return ioutil.WriteFile(file, []byte(contents), testFileMode)
 }

src/runtime/pkg/utils/utils.go

@@ -13,6 +13,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/sirupsen/logrus"
 )
@@ -124,6 +125,7 @@ func CreateVmmUser() (string, error) {
 	// Add retries to mitigate temporary errors and race conditions. For example, the user already exists
 	// or another instance of the runtime is also creating a user.
 	maxAttempt := 5
+	rand.Seed(time.Now().UnixNano())
 	for i := 0; i < maxAttempt; i++ {
 		userName = fmt.Sprintf("kata-%v", rand.Intn(100000))
 		_, err = RunCommand([]string{useraddPath, "-M", "-s", nologinPath, userName, "-c", "\"Kata Containers temporary hypervisor user\""})

src/runtime/vendor/github.com/apache/thrift/lib/go/thrift/binary_protocol.go

@@ -1,505 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package thrift
-
-import (
-	"bytes"
-	"context"
-	"encoding/binary"
-	"errors"
-	"fmt"
-	"io"
-	"math"
-)
-
-type TBinaryProtocol struct {
-	trans         TRichTransport
-	origTransport TTransport
-	strictRead    bool
-	strictWrite   bool
-	buffer        [64]byte
-}
-
-type TBinaryProtocolFactory struct {
-	strictRead  bool
-	strictWrite bool
-}
-
-func NewTBinaryProtocolTransport(t TTransport) *TBinaryProtocol {
-	return NewTBinaryProtocol(t, false, true)
-}
-
-func NewTBinaryProtocol(t TTransport, strictRead, strictWrite bool) *TBinaryProtocol {
-	p := &TBinaryProtocol{origTransport: t, strictRead: strictRead, strictWrite: strictWrite}
-	if et, ok := t.(TRichTransport); ok {
-		p.trans = et
-	} else {
-		p.trans = NewTRichTransport(t)
-	}
-	return p
-}
-
-func NewTBinaryProtocolFactoryDefault() *TBinaryProtocolFactory {
-	return NewTBinaryProtocolFactory(false, true)
-}
-
-func NewTBinaryProtocolFactory(strictRead, strictWrite bool) *TBinaryProtocolFactory {
-	return &TBinaryProtocolFactory{strictRead: strictRead, strictWrite: strictWrite}
-}
-
-func (p *TBinaryProtocolFactory) GetProtocol(t TTransport) TProtocol {
-	return NewTBinaryProtocol(t, p.strictRead, p.strictWrite)
-}
-
-/**
- * Writing Methods
- */
-
-func (p *TBinaryProtocol) WriteMessageBegin(name string, typeId TMessageType, seqId int32) error {
-	if p.strictWrite {
-		version := uint32(VERSION_1) | uint32(typeId)
-		e := p.WriteI32(int32(version))
-		if e != nil {
-			return e
-		}
-		e = p.WriteString(name)
-		if e != nil {
-			return e
-		}
-		e = p.WriteI32(seqId)
-		return e
-	} else {
-		e := p.WriteString(name)
-		if e != nil {
-			return e
-		}
-		e = p.WriteByte(int8(typeId))
-		if e != nil {
-			return e
-		}
-		e = p.WriteI32(seqId)
-		return e
-	}
-	return nil
-}
-
-func (p *TBinaryProtocol) WriteMessageEnd() error {
-	return nil
-}
-
-func (p *TBinaryProtocol) WriteStructBegin(name string) error {
-	return nil
-}
-
-func (p *TBinaryProtocol) WriteStructEnd() error {
-	return nil
-}
-
-func (p *TBinaryProtocol) WriteFieldBegin(name string, typeId TType, id int16) error {
-	e := p.WriteByte(int8(typeId))
-	if e != nil {
-		return e
-	}
-	e = p.WriteI16(id)
-	return e
-}
-
-func (p *TBinaryProtocol) WriteFieldEnd() error {
-	return nil
-}
-
-func (p *TBinaryProtocol) WriteFieldStop() error {
-	e := p.WriteByte(STOP)
-	return e
-}
-
-func (p *TBinaryProtocol) WriteMapBegin(keyType TType, valueType TType, size int) error {
-	e := p.WriteByte(int8(keyType))
-	if e != nil {
-		return e
-	}
-	e = p.WriteByte(int8(valueType))
-	if e != nil {
-		return e
-	}
-	e = p.WriteI32(int32(size))
-	return e
-}
-
-func (p *TBinaryProtocol) WriteMapEnd() error {
-	return nil
-}
-
-func (p *TBinaryProtocol) WriteListBegin(elemType TType, size int) error {
-	e := p.WriteByte(int8(elemType))
-	if e != nil {
-		return e
-	}
-	e = p.WriteI32(int32(size))
-	return e
-}
-
-func (p *TBinaryProtocol) WriteListEnd() error {
-	return nil
-}
-
-func (p *TBinaryProtocol) WriteSetBegin(elemType TType, size int) error {
-	e := p.WriteByte(int8(elemType))
-	if e != nil {
-		return e
-	}
-	e = p.WriteI32(int32(size))
-	return e
-}
-
-func (p *TBinaryProtocol) WriteSetEnd() error {
-	return nil
-}
-
-func (p *TBinaryProtocol) WriteBool(value bool) error {
-	if value {
-		return p.WriteByte(1)
-	}
-	return p.WriteByte(0)
-}
-
-func (p *TBinaryProtocol) WriteByte(value int8) error {
-	e := p.trans.WriteByte(byte(value))
-	return NewTProtocolException(e)
-}
-
-func (p *TBinaryProtocol) WriteI16(value int16) error {
-	v := p.buffer[0:2]
-	binary.BigEndian.PutUint16(v, uint16(value))
-	_, e := p.trans.Write(v)
-	return NewTProtocolException(e)
-}
-
-func (p *TBinaryProtocol) WriteI32(value int32) error {
-	v := p.buffer[0:4]
-	binary.BigEndian.PutUint32(v, uint32(value))
-	_, e := p.trans.Write(v)
-	return NewTProtocolException(e)
-}
-
-func (p *TBinaryProtocol) WriteI64(value int64) error {
-	v := p.buffer[0:8]
-	binary.BigEndian.PutUint64(v, uint64(value))
-	_, err := p.trans.Write(v)
-	return NewTProtocolException(err)
-}
-
-func (p *TBinaryProtocol) WriteDouble(value float64) error {
-	return p.WriteI64(int64(math.Float64bits(value)))
-}
-
-func (p *TBinaryProtocol) WriteString(value string) error {
-	e := p.WriteI32(int32(len(value)))
-	if e != nil {
-		return e
-	}
-	_, err := p.trans.WriteString(value)
-	return NewTProtocolException(err)
-}
-
-func (p *TBinaryProtocol) WriteBinary(value []byte) error {
-	e := p.WriteI32(int32(len(value)))
-	if e != nil {
-		return e
-	}
-	_, err := p.trans.Write(value)
-	return NewTProtocolException(err)
-}
-
-/**
- * Reading methods
- */
-
-func (p *TBinaryProtocol) ReadMessageBegin() (name string, typeId TMessageType, seqId int32, err error) {
-	size, e := p.ReadI32()
-	if e != nil {
-		return "", typeId, 0, NewTProtocolException(e)
-	}
-	if size < 0 {
-		typeId = TMessageType(size & 0x0ff)
-		version := int64(int64(size) & VERSION_MASK)
-		if version != VERSION_1 {
-			return name, typeId, seqId, NewTProtocolExceptionWithType(BAD_VERSION, fmt.Errorf("Bad version in ReadMessageBegin"))
-		}
-		name, e = p.ReadString()
-		if e != nil {
-			return name, typeId, seqId, NewTProtocolException(e)
-		}
-		seqId, e = p.ReadI32()
-		if e != nil {
-			return name, typeId, seqId, NewTProtocolException(e)
-		}
-		return name, typeId, seqId, nil
-	}
-	if p.strictRead {
-		return name, typeId, seqId, NewTProtocolExceptionWithType(BAD_VERSION, fmt.Errorf("Missing version in ReadMessageBegin"))
-	}
-	name, e2 := p.readStringBody(size)
-	if e2 != nil {
-		return name, typeId, seqId, e2
-	}
-	b, e3 := p.ReadByte()
-	if e3 != nil {
-		return name, typeId, seqId, e3
-	}
-	typeId = TMessageType(b)
-	seqId, e4 := p.ReadI32()
-	if e4 != nil {
-		return name, typeId, seqId, e4
-	}
-	return name, typeId, seqId, nil
-}
-
-func (p *TBinaryProtocol) ReadMessageEnd() error {
-	return nil
-}
-
-func (p *TBinaryProtocol) ReadStructBegin() (name string, err error) {
-	return
-}
-
-func (p *TBinaryProtocol) ReadStructEnd() error {
-	return nil
-}
-
-func (p *TBinaryProtocol) ReadFieldBegin() (name string, typeId TType, seqId int16, err error) {
-	t, err := p.ReadByte()
-	typeId = TType(t)
-	if err != nil {
-		return name, typeId, seqId, err
-	}
-	if t != STOP {
-		seqId, err = p.ReadI16()
-	}
-	return name, typeId, seqId, err
-}
-
-func (p *TBinaryProtocol) ReadFieldEnd() error {
-	return nil
-}
-
-var invalidDataLength = NewTProtocolExceptionWithType(INVALID_DATA, errors.New("Invalid data length"))
-
-func (p *TBinaryProtocol) ReadMapBegin() (kType, vType TType, size int, err error) {
-	k, e := p.ReadByte()
-	if e != nil {
-		err = NewTProtocolException(e)
-		return
-	}
-	kType = TType(k)
-	v, e := p.ReadByte()
-	if e != nil {
-		err = NewTProtocolException(e)
-		return
-	}
-	vType = TType(v)
-	size32, e := p.ReadI32()
-	if e != nil {
-		err = NewTProtocolException(e)
-		return
-	}
-	if size32 < 0 {
-		err = invalidDataLength
-		return
-	}
-	size = int(size32)
-	return kType, vType, size, nil
-}
-
-func (p *TBinaryProtocol) ReadMapEnd() error {
-	return nil
-}
-
-func (p *TBinaryProtocol) ReadListBegin() (elemType TType, size int, err error) {
-	b, e := p.ReadByte()
-	if e != nil {
-		err = NewTProtocolException(e)
-		return
-	}
-	elemType = TType(b)
-	size32, e := p.ReadI32()
-	if e != nil {
-		err = NewTProtocolException(e)
-		return
-	}
-	if size32 < 0 {
-		err = invalidDataLength
-		return
-	}
-	size = int(size32)
-
-	return
-}
-
-func (p *TBinaryProtocol) ReadListEnd() error {
-	return nil
-}
-
-func (p *TBinaryProtocol) ReadSetBegin() (elemType TType, size int, err error) {
-	b, e := p.ReadByte()
-	if e != nil {
-		err = NewTProtocolException(e)
-		return
-	}
-	elemType = TType(b)
-	size32, e := p.ReadI32()
-	if e != nil {
-		err = NewTProtocolException(e)
-		return
-	}
-	if size32 < 0 {
-		err = invalidDataLength
-		return
-	}
-	size = int(size32)
-	return elemType, size, nil
-}
-
-func (p *TBinaryProtocol) ReadSetEnd() error {
-	return nil
-}
-
-func (p *TBinaryProtocol) ReadBool() (bool, error) {
-	b, e := p.ReadByte()
-	v := true
-	if b != 1 {
-		v = false
-	}
-	return v, e
-}
-
-func (p *TBinaryProtocol) ReadByte() (int8, error) {
-	v, err := p.trans.ReadByte()
-	return int8(v), err
-}
-
-func (p *TBinaryProtocol) ReadI16() (value int16, err error) {
-	buf := p.buffer[0:2]
-	err = p.readAll(buf)
-	value = int16(binary.BigEndian.Uint16(buf))
-	return value, err
-}
-
-func (p *TBinaryProtocol) ReadI32() (value int32, err error) {
-	buf := p.buffer[0:4]
-	err = p.readAll(buf)
-	value = int32(binary.BigEndian.Uint32(buf))
-	return value, err
-}
-
-func (p *TBinaryProtocol) ReadI64() (value int64, err error) {
-	buf := p.buffer[0:8]
-	err = p.readAll(buf)
-	value = int64(binary.BigEndian.Uint64(buf))
-	return value, err
-}
-
-func (p *TBinaryProtocol) ReadDouble() (value float64, err error) {
-	buf := p.buffer[0:8]
-	err = p.readAll(buf)
-	value = math.Float64frombits(binary.BigEndian.Uint64(buf))
-	return value, err
-}
-
-func (p *TBinaryProtocol) ReadString() (value string, err error) {
-	size, e := p.ReadI32()
-	if e != nil {
-		return "", e
-	}
-	if size < 0 {
-		err = invalidDataLength
-		return
-	}
-
-	return p.readStringBody(size)
-}
-
-func (p *TBinaryProtocol) ReadBinary() ([]byte, error) {
-	size, e := p.ReadI32()
-	if e != nil {
-		return nil, e
-	}
-	if size < 0 {
-		return nil, invalidDataLength
-	}
-
-	isize := int(size)
-	buf := make([]byte, isize)
-	_, err := io.ReadFull(p.trans, buf)
-	return buf, NewTProtocolException(err)
-}
-
-func (p *TBinaryProtocol) Flush(ctx context.Context) (err error) {
-	return NewTProtocolException(p.trans.Flush(ctx))
-}
-
-func (p *TBinaryProtocol) Skip(fieldType TType) (err error) {
-	return SkipDefaultDepth(p, fieldType)
-}
-
-func (p *TBinaryProtocol) Transport() TTransport {
-	return p.origTransport
-}
-
-func (p *TBinaryProtocol) readAll(buf []byte) error {
-	_, err := io.ReadFull(p.trans, buf)
-	return NewTProtocolException(err)
-}
-
-const readLimit = 32768
-
-func (p *TBinaryProtocol) readStringBody(size int32) (value string, err error) {
-	if size < 0 {
-		return "", nil
-	}
-
-	var (
-		buf bytes.Buffer
-		e   error
-		b   []byte
-	)
-
-	switch {
-	case int(size) <= len(p.buffer):
-		b = p.buffer[:size] // avoids allocation for small reads
-	case int(size) < readLimit:
-		b = make([]byte, size)
-	default:
-		b = make([]byte, readLimit)
-	}
-
-	for size > 0 {
-		_, e = io.ReadFull(p.trans, b)
-		buf.Write(b)
-		if e != nil {
-			break
-		}
-		size -= readLimit
-		if size < readLimit && size > 0 {
-			b = b[:size]
-		}
-	}
-	return buf.String(), NewTProtocolException(e)
-}

src/runtime/vendor/github.com/apache/thrift/lib/go/thrift/debug_protocol.go

@@ -1,270 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package thrift
-
-import (
-	"context"
-	"log"
-)
-
-type TDebugProtocol struct {
-	Delegate  TProtocol
-	LogPrefix string
-}
-
-type TDebugProtocolFactory struct {
-	Underlying TProtocolFactory
-	LogPrefix  string
-}
-
-func NewTDebugProtocolFactory(underlying TProtocolFactory, logPrefix string) *TDebugProtocolFactory {
-	return &TDebugProtocolFactory{
-		Underlying: underlying,
-		LogPrefix:  logPrefix,
-	}
-}
-
-func (t *TDebugProtocolFactory) GetProtocol(trans TTransport) TProtocol {
-	return &TDebugProtocol{
-		Delegate:  t.Underlying.GetProtocol(trans),
-		LogPrefix: t.LogPrefix,
-	}
-}
-
-func (tdp *TDebugProtocol) WriteMessageBegin(name string, typeId TMessageType, seqid int32) error {
-	err := tdp.Delegate.WriteMessageBegin(name, typeId, seqid)
-	log.Printf("%sWriteMessageBegin(name=%#v, typeId=%#v, seqid=%#v) => %#v", tdp.LogPrefix, name, typeId, seqid, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteMessageEnd() error {
-	err := tdp.Delegate.WriteMessageEnd()
-	log.Printf("%sWriteMessageEnd() => %#v", tdp.LogPrefix, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteStructBegin(name string) error {
-	err := tdp.Delegate.WriteStructBegin(name)
-	log.Printf("%sWriteStructBegin(name=%#v) => %#v", tdp.LogPrefix, name, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteStructEnd() error {
-	err := tdp.Delegate.WriteStructEnd()
-	log.Printf("%sWriteStructEnd() => %#v", tdp.LogPrefix, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteFieldBegin(name string, typeId TType, id int16) error {
-	err := tdp.Delegate.WriteFieldBegin(name, typeId, id)
-	log.Printf("%sWriteFieldBegin(name=%#v, typeId=%#v, id%#v) => %#v", tdp.LogPrefix, name, typeId, id, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteFieldEnd() error {
-	err := tdp.Delegate.WriteFieldEnd()
-	log.Printf("%sWriteFieldEnd() => %#v", tdp.LogPrefix, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteFieldStop() error {
-	err := tdp.Delegate.WriteFieldStop()
-	log.Printf("%sWriteFieldStop() => %#v", tdp.LogPrefix, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteMapBegin(keyType TType, valueType TType, size int) error {
-	err := tdp.Delegate.WriteMapBegin(keyType, valueType, size)
-	log.Printf("%sWriteMapBegin(keyType=%#v, valueType=%#v, size=%#v) => %#v", tdp.LogPrefix, keyType, valueType, size, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteMapEnd() error {
-	err := tdp.Delegate.WriteMapEnd()
-	log.Printf("%sWriteMapEnd() => %#v", tdp.LogPrefix, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteListBegin(elemType TType, size int) error {
-	err := tdp.Delegate.WriteListBegin(elemType, size)
-	log.Printf("%sWriteListBegin(elemType=%#v, size=%#v) => %#v", tdp.LogPrefix, elemType, size, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteListEnd() error {
-	err := tdp.Delegate.WriteListEnd()
-	log.Printf("%sWriteListEnd() => %#v", tdp.LogPrefix, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteSetBegin(elemType TType, size int) error {
-	err := tdp.Delegate.WriteSetBegin(elemType, size)
-	log.Printf("%sWriteSetBegin(elemType=%#v, size=%#v) => %#v", tdp.LogPrefix, elemType, size, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteSetEnd() error {
-	err := tdp.Delegate.WriteSetEnd()
-	log.Printf("%sWriteSetEnd() => %#v", tdp.LogPrefix, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteBool(value bool) error {
-	err := tdp.Delegate.WriteBool(value)
-	log.Printf("%sWriteBool(value=%#v) => %#v", tdp.LogPrefix, value, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteByte(value int8) error {
-	err := tdp.Delegate.WriteByte(value)
-	log.Printf("%sWriteByte(value=%#v) => %#v", tdp.LogPrefix, value, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteI16(value int16) error {
-	err := tdp.Delegate.WriteI16(value)
-	log.Printf("%sWriteI16(value=%#v) => %#v", tdp.LogPrefix, value, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteI32(value int32) error {
-	err := tdp.Delegate.WriteI32(value)
-	log.Printf("%sWriteI32(value=%#v) => %#v", tdp.LogPrefix, value, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteI64(value int64) error {
-	err := tdp.Delegate.WriteI64(value)
-	log.Printf("%sWriteI64(value=%#v) => %#v", tdp.LogPrefix, value, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteDouble(value float64) error {
-	err := tdp.Delegate.WriteDouble(value)
-	log.Printf("%sWriteDouble(value=%#v) => %#v", tdp.LogPrefix, value, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteString(value string) error {
-	err := tdp.Delegate.WriteString(value)
-	log.Printf("%sWriteString(value=%#v) => %#v", tdp.LogPrefix, value, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteBinary(value []byte) error {
-	err := tdp.Delegate.WriteBinary(value)
-	log.Printf("%sWriteBinary(value=%#v) => %#v", tdp.LogPrefix, value, err)
-	return err
-}
-
-func (tdp *TDebugProtocol) ReadMessageBegin() (name string, typeId TMessageType, seqid int32, err error) {
-	name, typeId, seqid, err = tdp.Delegate.ReadMessageBegin()
-	log.Printf("%sReadMessageBegin() (name=%#v, typeId=%#v, seqid=%#v, err=%#v)", tdp.LogPrefix, name, typeId, seqid, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadMessageEnd() (err error) {
-	err = tdp.Delegate.ReadMessageEnd()
-	log.Printf("%sReadMessageEnd() err=%#v", tdp.LogPrefix, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadStructBegin() (name string, err error) {
-	name, err = tdp.Delegate.ReadStructBegin()
-	log.Printf("%sReadStructBegin() (name%#v, err=%#v)", tdp.LogPrefix, name, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadStructEnd() (err error) {
-	err = tdp.Delegate.ReadStructEnd()
-	log.Printf("%sReadStructEnd() err=%#v", tdp.LogPrefix, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadFieldBegin() (name string, typeId TType, id int16, err error) {
-	name, typeId, id, err = tdp.Delegate.ReadFieldBegin()
-	log.Printf("%sReadFieldBegin() (name=%#v, typeId=%#v, id=%#v, err=%#v)", tdp.LogPrefix, name, typeId, id, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadFieldEnd() (err error) {
-	err = tdp.Delegate.ReadFieldEnd()
-	log.Printf("%sReadFieldEnd() err=%#v", tdp.LogPrefix, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadMapBegin() (keyType TType, valueType TType, size int, err error) {
-	keyType, valueType, size, err = tdp.Delegate.ReadMapBegin()
-	log.Printf("%sReadMapBegin() (keyType=%#v, valueType=%#v, size=%#v, err=%#v)", tdp.LogPrefix, keyType, valueType, size, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadMapEnd() (err error) {
-	err = tdp.Delegate.ReadMapEnd()
-	log.Printf("%sReadMapEnd() err=%#v", tdp.LogPrefix, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadListBegin() (elemType TType, size int, err error) {
-	elemType, size, err = tdp.Delegate.ReadListBegin()
-	log.Printf("%sReadListBegin() (elemType=%#v, size=%#v, err=%#v)", tdp.LogPrefix, elemType, size, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadListEnd() (err error) {
-	err = tdp.Delegate.ReadListEnd()
-	log.Printf("%sReadListEnd() err=%#v", tdp.LogPrefix, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadSetBegin() (elemType TType, size int, err error) {
-	elemType, size, err = tdp.Delegate.ReadSetBegin()
-	log.Printf("%sReadSetBegin() (elemType=%#v, size=%#v, err=%#v)", tdp.LogPrefix, elemType, size, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadSetEnd() (err error) {
-	err = tdp.Delegate.ReadSetEnd()
-	log.Printf("%sReadSetEnd() err=%#v", tdp.LogPrefix, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadBool() (value bool, err error) {
-	value, err = tdp.Delegate.ReadBool()
-	log.Printf("%sReadBool() (value=%#v, err=%#v)", tdp.LogPrefix, value, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadByte() (value int8, err error) {
-	value, err = tdp.Delegate.ReadByte()
-	log.Printf("%sReadByte() (value=%#v, err=%#v)", tdp.LogPrefix, value, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadI16() (value int16, err error) {
-	value, err = tdp.Delegate.ReadI16()
-	log.Printf("%sReadI16() (value=%#v, err=%#v)", tdp.LogPrefix, value, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadI32() (value int32, err error) {
-	value, err = tdp.Delegate.ReadI32()
-	log.Printf("%sReadI32() (value=%#v, err=%#v)", tdp.LogPrefix, value, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadI64() (value int64, err error) {
-	value, err = tdp.Delegate.ReadI64()
-	log.Printf("%sReadI64() (value=%#v, err=%#v)", tdp.LogPrefix, value, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadDouble() (value float64, err error) {
-	value, err = tdp.Delegate.ReadDouble()
-	log.Printf("%sReadDouble() (value=%#v, err=%#v)", tdp.LogPrefix, value, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadString() (value string, err error) {
-	value, err = tdp.Delegate.ReadString()
-	log.Printf("%sReadString() (value=%#v, err=%#v)", tdp.LogPrefix, value, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadBinary() (value []byte, err error) {
-	value, err = tdp.Delegate.ReadBinary()
-	log.Printf("%sReadBinary() (value=%#v, err=%#v)", tdp.LogPrefix, value, err)
-	return
-}
-func (tdp *TDebugProtocol) Skip(fieldType TType) (err error) {
-	err = tdp.Delegate.Skip(fieldType)
-	log.Printf("%sSkip(fieldType=%#v) (err=%#v)", tdp.LogPrefix, fieldType, err)
-	return
-}
-func (tdp *TDebugProtocol) Flush(ctx context.Context) (err error) {
-	err = tdp.Delegate.Flush(ctx)
-	log.Printf("%sFlush() (err=%#v)", tdp.LogPrefix, err)
-	return
-}
-
-func (tdp *TDebugProtocol) Transport() TTransport {
-	return tdp.Delegate.Transport()
-}

src/runtime/vendor/github.com/apache/thrift/lib/go/thrift/deserializer.go

@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package thrift
-
-type TDeserializer struct {
-	Transport TTransport
-	Protocol  TProtocol
-}
-
-func NewTDeserializer() *TDeserializer {
-	var transport TTransport
-	transport = NewTMemoryBufferLen(1024)
-
-	protocol := NewTBinaryProtocolFactoryDefault().GetProtocol(transport)
-
-	return &TDeserializer{
-		transport,
-		protocol}
-}
-
-func (t *TDeserializer) ReadString(msg TStruct, s string) (err error) {
-	err = nil
-	if _, err = t.Transport.Write([]byte(s)); err != nil {
-		return
-	}
-	if err = msg.Read(t.Protocol); err != nil {
-		return
-	}
-	return
-}
-
-func (t *TDeserializer) Read(msg TStruct, b []byte) (err error) {
-	err = nil
-	if _, err = t.Transport.Write(b); err != nil {
-		return
-	}
-	if err = msg.Read(t.Protocol); err != nil {
-		return
-	}
-	return
-}

src/runtime/vendor/github.com/apache/thrift/lib/go/thrift/field.go

@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package thrift
-
-// Helper class that encapsulates field metadata.
-type field struct {
-	name   string
-	typeId TType
-	id     int
-}
-
-func newField(n string, t TType, i int) *field {
-	return &field{name: n, typeId: t, id: i}
-}
-
-func (p *field) Name() string {
-	if p == nil {
-		return ""
-	}
-	return p.name
-}
-
-func (p *field) TypeId() TType {
-	if p == nil {
-		return TType(VOID)
-	}
-	return p.typeId
-}
-
-func (p *field) Id() int {
-	if p == nil {
-		return -1
-	}
-	return p.id
-}
-
-func (p *field) String() string {
-	if p == nil {
-		return "<nil>"
-	}
-	return "<TField name:'" + p.name + "' type:" + string(p.typeId) + " field-id:" + string(p.id) + ">"
-}
-
-var ANONYMOUS_FIELD *field
-
-type fieldSlice []field
-
-func (p fieldSlice) Len() int {
-	return len(p)
-}
-
-func (p fieldSlice) Less(i, j int) bool {
-	return p[i].Id() < p[j].Id()
-}
-
-func (p fieldSlice) Swap(i, j int) {
-	p[i], p[j] = p[j], p[i]
-}
-
-func init() {
-	ANONYMOUS_FIELD = newField("", STOP, 0)
-}

src/runtime/vendor/github.com/apache/thrift/lib/go/thrift/framed_transport.go

@@ -1,187 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package thrift
-
-import (
-	"bufio"
-	"bytes"
-	"context"
-	"encoding/binary"
-	"fmt"
-	"io"
-)
-
-const DEFAULT_MAX_LENGTH = 16384000
-
-type TFramedTransport struct {
-	transport TTransport
-	buf       bytes.Buffer
-	reader    *bufio.Reader
-	frameSize uint32 //Current remaining size of the frame. if ==0 read next frame header
-	buffer    [4]byte
-	maxLength uint32
-}
-
-type tFramedTransportFactory struct {
-	factory   TTransportFactory
-	maxLength uint32
-}
-
-func NewTFramedTransportFactory(factory TTransportFactory) TTransportFactory {
-	return &tFramedTransportFactory{factory: factory, maxLength: DEFAULT_MAX_LENGTH}
-}
-
-func NewTFramedTransportFactoryMaxLength(factory TTransportFactory, maxLength uint32) TTransportFactory {
-	return &tFramedTransportFactory{factory: factory, maxLength: maxLength}
-}
-
-func (p *tFramedTransportFactory) GetTransport(base TTransport) (TTransport, error) {
-	tt, err := p.factory.GetTransport(base)
-	if err != nil {
-		return nil, err
-	}
-	return NewTFramedTransportMaxLength(tt, p.maxLength), nil
-}
-
-func NewTFramedTransport(transport TTransport) *TFramedTransport {
-	return &TFramedTransport{transport: transport, reader: bufio.NewReader(transport), maxLength: DEFAULT_MAX_LENGTH}
-}
-
-func NewTFramedTransportMaxLength(transport TTransport, maxLength uint32) *TFramedTransport {
-	return &TFramedTransport{transport: transport, reader: bufio.NewReader(transport), maxLength: maxLength}
-}
-
-func (p *TFramedTransport) Open() error {
-	return p.transport.Open()
-}
-
-func (p *TFramedTransport) IsOpen() bool {
-	return p.transport.IsOpen()
-}
-
-func (p *TFramedTransport) Close() error {
-	return p.transport.Close()
-}
-
-func (p *TFramedTransport) Read(buf []byte) (l int, err error) {
-	if p.frameSize == 0 {
-		p.frameSize, err = p.readFrameHeader()
-		if err != nil {
-			return
-		}
-	}
-	if p.frameSize < uint32(len(buf)) {
-		frameSize := p.frameSize
-		tmp := make([]byte, p.frameSize)
-		l, err = p.Read(tmp)
-		copy(buf, tmp)
-		if err == nil {
-			// Note: It's important to only return an error when l
-			// is zero.
-			// In io.Reader.Read interface, it's perfectly fine to
-			// return partial data and nil error, which means
-			// "This is all the data we have right now without
-			// blocking. If you need the full data, call Read again
-			// or use io.ReadFull instead".
-			// Returning partial data with an error actually means
-			// there's no more data after the partial data just
-			// returned, which is not true in this case
-			// (it might be that the other end just haven't written
-			// them yet).
-			if l == 0 {
-				err = NewTTransportExceptionFromError(fmt.Errorf("Not enough frame size %d to read %d bytes", frameSize, len(buf)))
-			}
-			return
-		}
-	}
-	got, err := p.reader.Read(buf)
-	p.frameSize = p.frameSize - uint32(got)
-	//sanity check
-	if p.frameSize < 0 {
-		return 0, NewTTransportException(UNKNOWN_TRANSPORT_EXCEPTION, "Negative frame size")
-	}
-	return got, NewTTransportExceptionFromError(err)
-}
-
-func (p *TFramedTransport) ReadByte() (c byte, err error) {
-	if p.frameSize == 0 {
-		p.frameSize, err = p.readFrameHeader()
-		if err != nil {
-			return
-		}
-	}
-	if p.frameSize < 1 {
-		return 0, NewTTransportExceptionFromError(fmt.Errorf("Not enough frame size %d to read %d bytes", p.frameSize, 1))
-	}
-	c, err = p.reader.ReadByte()
-	if err == nil {
-		p.frameSize--
-	}
-	return
-}
-
-func (p *TFramedTransport) Write(buf []byte) (int, error) {
-	n, err := p.buf.Write(buf)
-	return n, NewTTransportExceptionFromError(err)
-}
-
-func (p *TFramedTransport) WriteByte(c byte) error {
-	return p.buf.WriteByte(c)
-}
-
-func (p *TFramedTransport) WriteString(s string) (n int, err error) {
-	return p.buf.WriteString(s)
-}
-
-func (p *TFramedTransport) Flush(ctx context.Context) error {
-	size := p.buf.Len()
-	buf := p.buffer[:4]
-	binary.BigEndian.PutUint32(buf, uint32(size))
-	_, err := p.transport.Write(buf)
-	if err != nil {
-		p.buf.Truncate(0)
-		return NewTTransportExceptionFromError(err)
-	}
-	if size > 0 {
-		if n, err := p.buf.WriteTo(p.transport); err != nil {
-			print("Error while flushing write buffer of size ", size, " to transport, only wrote ", n, " bytes: ", err.Error(), "\n")
-			p.buf.Truncate(0)
-			return NewTTransportExceptionFromError(err)
-		}
-	}
-	err = p.transport.Flush(ctx)
-	return NewTTransportExceptionFromError(err)
-}
-
-func (p *TFramedTransport) readFrameHeader() (uint32, error) {
-	buf := p.buffer[:4]
-	if _, err := io.ReadFull(p.reader, buf); err != nil {
-		return 0, err
-	}
-	size := binary.BigEndian.Uint32(buf)
-	if size < 0 || size > p.maxLength {
-		return 0, NewTTransportException(UNKNOWN_TRANSPORT_EXCEPTION, fmt.Sprintf("Incorrect frame size (%d)", size))
-	}
-	return size, nil
-}
-
-func (p *TFramedTransport) RemainingBytes() (num_bytes uint64) {
-	return uint64(p.frameSize)
-}

src/runtime/vendor/github.com/apache/thrift/lib/go/thrift/header_protocol.go

@@ -1,305 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package thrift
-
-import (
-	"context"
-)
-
-// THeaderProtocol is a thrift protocol that implements THeader:
-// https://github.com/apache/thrift/blob/master/doc/specs/HeaderFormat.md
-//
-// It supports either binary or compact protocol as the wrapped protocol.
-//
-// Most of the THeader handlings are happening inside THeaderTransport.
-type THeaderProtocol struct {
-	transport *THeaderTransport
-
-	// Will be initialized on first read/write.
-	protocol TProtocol
-}
-
-// NewTHeaderProtocol creates a new THeaderProtocol from the underlying
-// transport. The passed in transport will be wrapped with THeaderTransport.
-//
-// Note that THeaderTransport handles frame and zlib by itself,
-// so the underlying transport should be a raw socket transports (TSocket or TSSLSocket),
-// instead of rich transports like TZlibTransport or TFramedTransport.
-func NewTHeaderProtocol(trans TTransport) *THeaderProtocol {
-	t := NewTHeaderTransport(trans)
-	p, _ := THeaderProtocolDefault.GetProtocol(t)
-	return &THeaderProtocol{
-		transport: t,
-		protocol:  p,
-	}
-}
-
-type tHeaderProtocolFactory struct{}
-
-func (tHeaderProtocolFactory) GetProtocol(trans TTransport) TProtocol {
-	return NewTHeaderProtocol(trans)
-}
-
-// NewTHeaderProtocolFactory creates a factory for THeader.
-//
-// It's a wrapper for NewTHeaderProtocol
-func NewTHeaderProtocolFactory() TProtocolFactory {
-	return tHeaderProtocolFactory{}
-}
-
-// Transport returns the underlying transport.
-//
-// It's guaranteed to be of type *THeaderTransport.
-func (p *THeaderProtocol) Transport() TTransport {
-	return p.transport
-}
-
-// GetReadHeaders returns the THeaderMap read from transport.
-func (p *THeaderProtocol) GetReadHeaders() THeaderMap {
-	return p.transport.GetReadHeaders()
-}
-
-// SetWriteHeader sets a header for write.
-func (p *THeaderProtocol) SetWriteHeader(key, value string) {
-	p.transport.SetWriteHeader(key, value)
-}
-
-// ClearWriteHeaders clears all write headers previously set.
-func (p *THeaderProtocol) ClearWriteHeaders() {
-	p.transport.ClearWriteHeaders()
-}
-
-// AddTransform add a transform for writing.
-func (p *THeaderProtocol) AddTransform(transform THeaderTransformID) error {
-	return p.transport.AddTransform(transform)
-}
-
-func (p *THeaderProtocol) Flush(ctx context.Context) error {
-	return p.transport.Flush(ctx)
-}
-
-func (p *THeaderProtocol) WriteMessageBegin(name string, typeID TMessageType, seqID int32) error {
-	newProto, err := p.transport.Protocol().GetProtocol(p.transport)
-	if err != nil {
-		return err
-	}
-	p.protocol = newProto
-	p.transport.SequenceID = seqID
-	return p.protocol.WriteMessageBegin(name, typeID, seqID)
-}
-
-func (p *THeaderProtocol) WriteMessageEnd() error {
-	if err := p.protocol.WriteMessageEnd(); err != nil {
-		return err
-	}
-	return p.transport.Flush(context.Background())
-}
-
-func (p *THeaderProtocol) WriteStructBegin(name string) error {
-	return p.protocol.WriteStructBegin(name)
-}
-
-func (p *THeaderProtocol) WriteStructEnd() error {
-	return p.protocol.WriteStructEnd()
-}
-
-func (p *THeaderProtocol) WriteFieldBegin(name string, typeID TType, id int16) error {
-	return p.protocol.WriteFieldBegin(name, typeID, id)
-}
-
-func (p *THeaderProtocol) WriteFieldEnd() error {
-	return p.protocol.WriteFieldEnd()
-}
-
-func (p *THeaderProtocol) WriteFieldStop() error {
-	return p.protocol.WriteFieldStop()
-}
-
-func (p *THeaderProtocol) WriteMapBegin(keyType TType, valueType TType, size int) error {
-	return p.protocol.WriteMapBegin(keyType, valueType, size)
-}
-
-func (p *THeaderProtocol) WriteMapEnd() error {
-	return p.protocol.WriteMapEnd()
-}
-
-func (p *THeaderProtocol) WriteListBegin(elemType TType, size int) error {
-	return p.protocol.WriteListBegin(elemType, size)
-}
-
-func (p *THeaderProtocol) WriteListEnd() error {
-	return p.protocol.WriteListEnd()
-}
-
-func (p *THeaderProtocol) WriteSetBegin(elemType TType, size int) error {
-	return p.protocol.WriteSetBegin(elemType, size)
-}
-
-func (p *THeaderProtocol) WriteSetEnd() error {
-	return p.protocol.WriteSetEnd()
-}
-
-func (p *THeaderProtocol) WriteBool(value bool) error {
-	return p.protocol.WriteBool(value)
-}
-
-func (p *THeaderProtocol) WriteByte(value int8) error {
-	return p.protocol.WriteByte(value)
-}
-
-func (p *THeaderProtocol) WriteI16(value int16) error {
-	return p.protocol.WriteI16(value)
-}
-
-func (p *THeaderProtocol) WriteI32(value int32) error {
-	return p.protocol.WriteI32(value)
-}
-
-func (p *THeaderProtocol) WriteI64(value int64) error {
-	return p.protocol.WriteI64(value)
-}
-
-func (p *THeaderProtocol) WriteDouble(value float64) error {
-	return p.protocol.WriteDouble(value)
-}
-
-func (p *THeaderProtocol) WriteString(value string) error {
-	return p.protocol.WriteString(value)
-}
-
-func (p *THeaderProtocol) WriteBinary(value []byte) error {
-	return p.protocol.WriteBinary(value)
-}
-
-// ReadFrame calls underlying THeaderTransport's ReadFrame function.
-func (p *THeaderProtocol) ReadFrame() error {
-	return p.transport.ReadFrame()
-}
-
-func (p *THeaderProtocol) ReadMessageBegin() (name string, typeID TMessageType, seqID int32, err error) {
-	if err = p.transport.ReadFrame(); err != nil {
-		return
-	}
-
-	var newProto TProtocol
-	newProto, err = p.transport.Protocol().GetProtocol(p.transport)
-	if err != nil {
-		tAppExc, ok := err.(TApplicationException)
-		if !ok {
-			return
-		}
-		if e := p.protocol.WriteMessageBegin("", EXCEPTION, seqID); e != nil {
-			return
-		}
-		if e := tAppExc.Write(p.protocol); e != nil {
-			return
-		}
-		if e := p.protocol.WriteMessageEnd(); e != nil {
-			return
-		}
-		if e := p.transport.Flush(context.Background()); e != nil {
-			return
-		}
-		return
-	}
-	p.protocol = newProto
-
-	return p.protocol.ReadMessageBegin()
-}
-
-func (p *THeaderProtocol) ReadMessageEnd() error {
-	return p.protocol.ReadMessageEnd()
-}
-
-func (p *THeaderProtocol) ReadStructBegin() (name string, err error) {
-	return p.protocol.ReadStructBegin()
-}
-
-func (p *THeaderProtocol) ReadStructEnd() error {
-	return p.protocol.ReadStructEnd()
-}
-
-func (p *THeaderProtocol) ReadFieldBegin() (name string, typeID TType, id int16, err error) {
-	return p.protocol.ReadFieldBegin()
-}
-
-func (p *THeaderProtocol) ReadFieldEnd() error {
-	return p.protocol.ReadFieldEnd()
-}
-
-func (p *THeaderProtocol) ReadMapBegin() (keyType TType, valueType TType, size int, err error) {
-	return p.protocol.ReadMapBegin()
-}
-
-func (p *THeaderProtocol) ReadMapEnd() error {
-	return p.protocol.ReadMapEnd()
-}
-
-func (p *THeaderProtocol) ReadListBegin() (elemType TType, size int, err error) {
-	return p.protocol.ReadListBegin()
-}
-
-func (p *THeaderProtocol) ReadListEnd() error {
-	return p.protocol.ReadListEnd()
-}
-
-func (p *THeaderProtocol) ReadSetBegin() (elemType TType, size int, err error) {
-	return p.protocol.ReadSetBegin()
-}
-
-func (p *THeaderProtocol) ReadSetEnd() error {
-	return p.protocol.ReadSetEnd()
-}
-
-func (p *THeaderProtocol) ReadBool() (value bool, err error) {
-	return p.protocol.ReadBool()
-}
-
-func (p *THeaderProtocol) ReadByte() (value int8, err error) {
-	return p.protocol.ReadByte()
-}
-
-func (p *THeaderProtocol) ReadI16() (value int16, err error) {
-	return p.protocol.ReadI16()
-}
-
-func (p *THeaderProtocol) ReadI32() (value int32, err error) {
-	return p.protocol.ReadI32()
-}
-
-func (p *THeaderProtocol) ReadI64() (value int64, err error) {
-	return p.protocol.ReadI64()
-}
-
-func (p *THeaderProtocol) ReadDouble() (value float64, err error) {
-	return p.protocol.ReadDouble()
-}
-
-func (p *THeaderProtocol) ReadString() (value string, err error) {
-	return p.protocol.ReadString()
-}
-
-func (p *THeaderProtocol) ReadBinary() (value []byte, err error) {
-	return p.protocol.ReadBinary()
-}
-
-func (p *THeaderProtocol) Skip(fieldType TType) error {
-	return p.protocol.Skip(fieldType)
-}

src/runtime/vendor/github.com/apache/thrift/lib/go/thrift/protocol.go

@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package thrift
-
-import (
-	"context"
-	"errors"
-	"fmt"
-)
-
-const (
-	VERSION_MASK = 0xffff0000
-	VERSION_1    = 0x80010000
-)
-
-type TProtocol interface {
-	WriteMessageBegin(name string, typeId TMessageType, seqid int32) error
-	WriteMessageEnd() error
-	WriteStructBegin(name string) error
-	WriteStructEnd() error
-	WriteFieldBegin(name string, typeId TType, id int16) error
-	WriteFieldEnd() error
-	WriteFieldStop() error
-	WriteMapBegin(keyType TType, valueType TType, size int) error
-	WriteMapEnd() error
-	WriteListBegin(elemType TType, size int) error
-	WriteListEnd() error
-	WriteSetBegin(elemType TType, size int) error
-	WriteSetEnd() error
-	WriteBool(value bool) error
-	WriteByte(value int8) error
-	WriteI16(value int16) error
-	WriteI32(value int32) error
-	WriteI64(value int64) error
-	WriteDouble(value float64) error
-	WriteString(value string) error
-	WriteBinary(value []byte) error
-
-	ReadMessageBegin() (name string, typeId TMessageType, seqid int32, err error)
-	ReadMessageEnd() error
-	ReadStructBegin() (name string, err error)
-	ReadStructEnd() error
-	ReadFieldBegin() (name string, typeId TType, id int16, err error)
-	ReadFieldEnd() error
-	ReadMapBegin() (keyType TType, valueType TType, size int, err error)
-	ReadMapEnd() error
-	ReadListBegin() (elemType TType, size int, err error)
-	ReadListEnd() error
-	ReadSetBegin() (elemType TType, size int, err error)
-	ReadSetEnd() error
-	ReadBool() (value bool, err error)
-	ReadByte() (value int8, err error)
-	ReadI16() (value int16, err error)
-	ReadI32() (value int32, err error)
-	ReadI64() (value int64, err error)
-	ReadDouble() (value float64, err error)
-	ReadString() (value string, err error)
-	ReadBinary() (value []byte, err error)
-
-	Skip(fieldType TType) (err error)
-	Flush(ctx context.Context) (err error)
-
-	Transport() TTransport
-}
-
-// The maximum recursive depth the skip() function will traverse
-const DEFAULT_RECURSION_DEPTH = 64
-
-// Skips over the next data element from the provided input TProtocol object.
-func SkipDefaultDepth(prot TProtocol, typeId TType) (err error) {
-	return Skip(prot, typeId, DEFAULT_RECURSION_DEPTH)
-}
-
-// Skips over the next data element from the provided input TProtocol object.
-func Skip(self TProtocol, fieldType TType, maxDepth int) (err error) {
-
-	if maxDepth <= 0 {
-		return NewTProtocolExceptionWithType(DEPTH_LIMIT, errors.New("Depth limit exceeded"))
-	}
-
-	switch fieldType {
-	case BOOL:
-		_, err = self.ReadBool()
-		return
-	case BYTE:
-		_, err = self.ReadByte()
-		return
-	case I16:
-		_, err = self.ReadI16()
-		return
-	case I32:
-		_, err = self.ReadI32()
-		return
-	case I64:
-		_, err = self.ReadI64()
-		return
-	case DOUBLE:
-		_, err = self.ReadDouble()
-		return
-	case STRING:
-		_, err = self.ReadString()
-		return
-	case STRUCT:
-		if _, err = self.ReadStructBegin(); err != nil {
-			return err
-		}
-		for {
-			_, typeId, _, _ := self.ReadFieldBegin()
-			if typeId == STOP {
-				break
-			}
-			err := Skip(self, typeId, maxDepth-1)
-			if err != nil {
-				return err
-			}
-			self.ReadFieldEnd()
-		}
-		return self.ReadStructEnd()
-	case MAP:
-		keyType, valueType, size, err := self.ReadMapBegin()
-		if err != nil {
-			return err
-		}
-		for i := 0; i < size; i++ {
-			err := Skip(self, keyType, maxDepth-1)
-			if err != nil {
-				return err
-			}
-			self.Skip(valueType)
-		}
-		return self.ReadMapEnd()
-	case SET:
-		elemType, size, err := self.ReadSetBegin()
-		if err != nil {
-			return err
-		}
-		for i := 0; i < size; i++ {
-			err := Skip(self, elemType, maxDepth-1)
-			if err != nil {
-				return err
-			}
-		}
-		return self.ReadSetEnd()
-	case LIST:
-		elemType, size, err := self.ReadListBegin()
-		if err != nil {
-			return err
-		}
-		for i := 0; i < size; i++ {
-			err := Skip(self, elemType, maxDepth-1)
-			if err != nil {
-				return err
-			}
-		}
-		return self.ReadListEnd()
-	default:
-		return NewTProtocolExceptionWithType(INVALID_DATA, errors.New(fmt.Sprintf("Unknown data type %d", fieldType)))
-	}
-	return nil
-}

src/runtime/vendor/github.com/apache/thrift/lib/go/thrift/serializer.go

@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package thrift
-
-import (
-	"context"
-)
-
-type TSerializer struct {
-	Transport *TMemoryBuffer
-	Protocol  TProtocol
-}
-
-type TStruct interface {
-	Write(p TProtocol) error
-	Read(p TProtocol) error
-}
-
-func NewTSerializer() *TSerializer {
-	transport := NewTMemoryBufferLen(1024)
-	protocol := NewTBinaryProtocolFactoryDefault().GetProtocol(transport)
-
-	return &TSerializer{
-		transport,
-		protocol}
-}
-
-func (t *TSerializer) WriteString(ctx context.Context, msg TStruct) (s string, err error) {
-	t.Transport.Reset()
-
-	if err = msg.Write(t.Protocol); err != nil {
-		return
-	}
-
-	if err = t.Protocol.Flush(ctx); err != nil {
-		return
-	}
-	if err = t.Transport.Flush(ctx); err != nil {
-		return
-	}
-
-	return t.Transport.String(), nil
-}
-
-func (t *TSerializer) Write(ctx context.Context, msg TStruct) (b []byte, err error) {
-	t.Transport.Reset()
-
-	if err = msg.Write(t.Protocol); err != nil {
-		return
-	}
-
-	if err = t.Protocol.Flush(ctx); err != nil {
-		return
-	}
-
-	if err = t.Transport.Flush(ctx); err != nil {
-		return
-	}
-
-	b = append(b, t.Transport.Bytes()...)
-	return
-}

src/runtime/vendor/github.com/apache/thrift/lib/go/thrift/socket.go

@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package thrift
-
-import (
-	"context"
-	"net"
-	"time"
-)
-
-type TSocket struct {
-	conn    net.Conn
-	addr    net.Addr
-	timeout time.Duration
-}
-
-// NewTSocket creates a net.Conn-backed TTransport, given a host and port
-//
-// Example:
-// 	trans, err := thrift.NewTSocket("localhost:9090")
-func NewTSocket(hostPort string) (*TSocket, error) {
-	return NewTSocketTimeout(hostPort, 0)
-}
-
-// NewTSocketTimeout creates a net.Conn-backed TTransport, given a host and port
-// it also accepts a timeout as a time.Duration
-func NewTSocketTimeout(hostPort string, timeout time.Duration) (*TSocket, error) {
-	//conn, err := net.DialTimeout(network, address, timeout)
-	addr, err := net.ResolveTCPAddr("tcp", hostPort)
-	if err != nil {
-		return nil, err
-	}
-	return NewTSocketFromAddrTimeout(addr, timeout), nil
-}
-
-// Creates a TSocket from a net.Addr
-func NewTSocketFromAddrTimeout(addr net.Addr, timeout time.Duration) *TSocket {
-	return &TSocket{addr: addr, timeout: timeout}
-}
-
-// Creates a TSocket from an existing net.Conn
-func NewTSocketFromConnTimeout(conn net.Conn, timeout time.Duration) *TSocket {
-	return &TSocket{conn: conn, addr: conn.RemoteAddr(), timeout: timeout}
-}
-
-// Sets the socket timeout
-func (p *TSocket) SetTimeout(timeout time.Duration) error {
-	p.timeout = timeout
-	return nil
-}
-
-func (p *TSocket) pushDeadline(read, write bool) {
-	var t time.Time
-	if p.timeout > 0 {
-		t = time.Now().Add(time.Duration(p.timeout))
-	}
-	if read && write {
-		p.conn.SetDeadline(t)
-	} else if read {
-		p.conn.SetReadDeadline(t)
-	} else if write {
-		p.conn.SetWriteDeadline(t)
-	}
-}
-
-// Connects the socket, creating a new socket object if necessary.
-func (p *TSocket) Open() error {
-	if p.IsOpen() {
-		return NewTTransportException(ALREADY_OPEN, "Socket already connected.")
-	}
-	if p.addr == nil {
-		return NewTTransportException(NOT_OPEN, "Cannot open nil address.")
-	}
-	if len(p.addr.Network()) == 0 {
-		return NewTTransportException(NOT_OPEN, "Cannot open bad network name.")
-	}
-	if len(p.addr.String()) == 0 {
-		return NewTTransportException(NOT_OPEN, "Cannot open bad address.")
-	}
-	var err error
-	if p.conn, err = net.DialTimeout(p.addr.Network(), p.addr.String(), p.timeout); err != nil {
-		return NewTTransportException(NOT_OPEN, err.Error())
-	}
-	return nil
-}
-
-// Retrieve the underlying net.Conn
-func (p *TSocket) Conn() net.Conn {
-	return p.conn
-}
-
-// Returns true if the connection is open
-func (p *TSocket) IsOpen() bool {
-	if p.conn == nil {
-		return false
-	}
-	return true
-}
-
-// Closes the socket.
-func (p *TSocket) Close() error {
-	// Close the socket
-	if p.conn != nil {
-		err := p.conn.Close()
-		if err != nil {
-			return err
-		}
-		p.conn = nil
-	}
-	return nil
-}
-
-//Returns the remote address of the socket.
-func (p *TSocket) Addr() net.Addr {
-	return p.addr
-}
-
-func (p *TSocket) Read(buf []byte) (int, error) {
-	if !p.IsOpen() {
-		return 0, NewTTransportException(NOT_OPEN, "Connection not open")
-	}
-	p.pushDeadline(true, false)
-	n, err := p.conn.Read(buf)
-	return n, NewTTransportExceptionFromError(err)
-}
-
-func (p *TSocket) Write(buf []byte) (int, error) {
-	if !p.IsOpen() {
-		return 0, NewTTransportException(NOT_OPEN, "Connection not open")
-	}
-	p.pushDeadline(false, true)
-	return p.conn.Write(buf)
-}
-
-func (p *TSocket) Flush(ctx context.Context) error {
-	return nil
-}
-
-func (p *TSocket) Interrupt() error {
-	if !p.IsOpen() {
-		return nil
-	}
-	return p.conn.Close()
-}
-
-func (p *TSocket) RemainingBytes() (num_bytes uint64) {
-	const maxSize = ^uint64(0)
-	return maxSize // the truth is, we just don't know unless framed is used
-}

src/runtime/vendor/github.com/apache/thrift/lib/go/thrift/ssl_socket.go

@@ -1,176 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package thrift
-
-import (
-	"context"
-	"crypto/tls"
-	"net"
-	"time"
-)
-
-type TSSLSocket struct {
-	conn net.Conn
-	// hostPort contains host:port (e.g. "asdf.com:12345"). The field is
-	// only valid if addr is nil.
-	hostPort string
-	// addr is nil when hostPort is not "", and is only used when the
-	// TSSLSocket is constructed from a net.Addr.
-	addr    net.Addr
-	timeout time.Duration
-	cfg     *tls.Config
-}
-
-// NewTSSLSocket creates a net.Conn-backed TTransport, given a host and port and tls Configuration
-//
-// Example:
-// 	trans, err := thrift.NewTSSLSocket("localhost:9090", nil)
-func NewTSSLSocket(hostPort string, cfg *tls.Config) (*TSSLSocket, error) {
-	return NewTSSLSocketTimeout(hostPort, cfg, 0)
-}
-
-// NewTSSLSocketTimeout creates a net.Conn-backed TTransport, given a host and port
-// it also accepts a tls Configuration and a timeout as a time.Duration
-func NewTSSLSocketTimeout(hostPort string, cfg *tls.Config, timeout time.Duration) (*TSSLSocket, error) {
-	if cfg.MinVersion == 0 {
-		cfg.MinVersion = tls.VersionTLS10
-	}
-	return &TSSLSocket{hostPort: hostPort, timeout: timeout, cfg: cfg}, nil
-}
-
-// Creates a TSSLSocket from a net.Addr
-func NewTSSLSocketFromAddrTimeout(addr net.Addr, cfg *tls.Config, timeout time.Duration) *TSSLSocket {
-	return &TSSLSocket{addr: addr, timeout: timeout, cfg: cfg}
-}
-
-// Creates a TSSLSocket from an existing net.Conn
-func NewTSSLSocketFromConnTimeout(conn net.Conn, cfg *tls.Config, timeout time.Duration) *TSSLSocket {
-	return &TSSLSocket{conn: conn, addr: conn.RemoteAddr(), timeout: timeout, cfg: cfg}
-}
-
-// Sets the socket timeout
-func (p *TSSLSocket) SetTimeout(timeout time.Duration) error {
-	p.timeout = timeout
-	return nil
-}
-
-func (p *TSSLSocket) pushDeadline(read, write bool) {
-	var t time.Time
-	if p.timeout > 0 {
-		t = time.Now().Add(time.Duration(p.timeout))
-	}
-	if read && write {
-		p.conn.SetDeadline(t)
-	} else if read {
-		p.conn.SetReadDeadline(t)
-	} else if write {
-		p.conn.SetWriteDeadline(t)
-	}
-}
-
-// Connects the socket, creating a new socket object if necessary.
-func (p *TSSLSocket) Open() error {
-	var err error
-	// If we have a hostname, we need to pass the hostname to tls.Dial for
-	// certificate hostname checks.
-	if p.hostPort != "" {
-		if p.conn, err = tls.DialWithDialer(&net.Dialer{
-			Timeout: p.timeout}, "tcp", p.hostPort, p.cfg); err != nil {
-			return NewTTransportException(NOT_OPEN, err.Error())
-		}
-	} else {
-		if p.IsOpen() {
-			return NewTTransportException(ALREADY_OPEN, "Socket already connected.")
-		}
-		if p.addr == nil {
-			return NewTTransportException(NOT_OPEN, "Cannot open nil address.")
-		}
-		if len(p.addr.Network()) == 0 {
-			return NewTTransportException(NOT_OPEN, "Cannot open bad network name.")
-		}
-		if len(p.addr.String()) == 0 {
-			return NewTTransportException(NOT_OPEN, "Cannot open bad address.")
-		}
-		if p.conn, err = tls.DialWithDialer(&net.Dialer{
-			Timeout: p.timeout}, p.addr.Network(), p.addr.String(), p.cfg); err != nil {
-			return NewTTransportException(NOT_OPEN, err.Error())
-		}
-	}
-	return nil
-}
-
-// Retrieve the underlying net.Conn
-func (p *TSSLSocket) Conn() net.Conn {
-	return p.conn
-}
-
-// Returns true if the connection is open
-func (p *TSSLSocket) IsOpen() bool {
-	if p.conn == nil {
-		return false
-	}
-	return true
-}
-
-// Closes the socket.
-func (p *TSSLSocket) Close() error {
-	// Close the socket
-	if p.conn != nil {
-		err := p.conn.Close()
-		if err != nil {
-			return err
-		}
-		p.conn = nil
-	}
-	return nil
-}
-
-func (p *TSSLSocket) Read(buf []byte) (int, error) {
-	if !p.IsOpen() {
-		return 0, NewTTransportException(NOT_OPEN, "Connection not open")
-	}
-	p.pushDeadline(true, false)
-	n, err := p.conn.Read(buf)
-	return n, NewTTransportExceptionFromError(err)
-}
-
-func (p *TSSLSocket) Write(buf []byte) (int, error) {
-	if !p.IsOpen() {
-		return 0, NewTTransportException(NOT_OPEN, "Connection not open")
-	}
-	p.pushDeadline(false, true)
-	return p.conn.Write(buf)
-}
-
-func (p *TSSLSocket) Flush(ctx context.Context) error {
-	return nil
-}
-
-func (p *TSSLSocket) Interrupt() error {
-	if !p.IsOpen() {
-		return nil
-	}
-	return p.conn.Close()
-}
-
-func (p *TSSLSocket) RemainingBytes() (num_bytes uint64) {
-	const maxSize = ^uint64(0)
-	return maxSize // the thruth is, we just don't know unless framed is used
-}

src/runtime/vendor/github.com/containerd/containerd/pkg/cri/annotations/annotations.go

@@ -0,0 +1,62 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package annotations
+
+// ContainerType values
+// Following OCI annotations are used by katacontainers now.
+// We'll switch to standard secure pod API after it is defined in CRI.
+const (
+	// ContainerTypeSandbox represents a pod sandbox container
+	ContainerTypeSandbox = "sandbox"
+
+	// ContainerTypeContainer represents a container running within a pod
+	ContainerTypeContainer = "container"
+
+	// ContainerType is the container type (sandbox or container) annotation
+	ContainerType = "io.kubernetes.cri.container-type"
+
+	// SandboxID is the sandbox ID annotation
+	SandboxID = "io.kubernetes.cri.sandbox-id"
+
+	// SandboxLogDir is the pod log directory annotation.
+	// If the sandbox needs to generate any log, it will put it into this directory.
+	// Kubelet will be responsible for:
+	// 1) Monitoring the disk usage of the log, and including it as part of the pod
+	// ephemeral storage usage.
+	// 2) Cleaning up the logs when the pod is deleted.
+	// NOTE: Kubelet is not responsible for rotating the logs.
+	SandboxLogDir = "io.kubernetes.cri.sandbox-log-directory"
+
+	// UntrustedWorkload is the sandbox annotation for untrusted workload. Untrusted
+	// workload can only run on dedicated runtime for untrusted workload.
+	UntrustedWorkload = "io.kubernetes.cri.untrusted-workload"
+
+	// SandboxNamespace is the name of the namespace of the sandbox (pod)
+	SandboxNamespace = "io.kubernetes.cri.sandbox-namespace"
+
+	// SandboxName is the name of the sandbox (pod)
+	SandboxName = "io.kubernetes.cri.sandbox-name"
+
+	// ContainerName is the name of the container in the pod
+	ContainerName = "io.kubernetes.cri.container-name"
+
+	// ImageName is the name of the image used to create the container
+	ImageName = "io.kubernetes.cri.image-name"
+
+	// PodAnnotations are the annotations of the pod
+	PodAnnotations = "io.kubernetes.cri.pod-annotations"
+)

src/runtime/vendor/github.com/containerd/cri-containerd/pkg/annotations/annotations.go

@@ -1,38 +0,0 @@
-/*
-Copyright 2018 The Containerd Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package annotations
-
-// ContainerType values
-// Following OCI annotations are used by katacontainers now.
-// We'll switch to standard secure pod API after it is defined in CRI.
-const (
-	// ContainerTypeSandbox represents a pod sandbox container
-	ContainerTypeSandbox = "sandbox"
-
-	// ContainerTypeContainer represents a container running within a pod
-	ContainerTypeContainer = "container"
-
-	// ContainerType is the container type (sandbox or container) annotation
-	ContainerType = "io.kubernetes.cri.container-type"
-
-	// SandboxID is the sandbox ID annotation
-	SandboxID = "io.kubernetes.cri.sandbox-id"
-
-	// UntrustedWorkload is the sandbox annotation for untrusted workload. Untrusted
-	// workload can only run on dedicated runtime for untrusted workload.
-	UntrustedWorkload = "io.kubernetes.cri.untrusted-workload"
-)

src/runtime/vendor/github.com/containerd/cri-containerd/pkg/api/runtimeoptions/v1/api.pb.go

@@ -1,394 +0,0 @@
-/*
-Copyright 2019 The containerd Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-// Code generated by protoc-gen-gogo. DO NOT EDIT.
-// source: api.proto
-
-/*
-	Package cri_runtimeoptions_v1 is a generated protocol buffer package.
-
-	It is generated from these files:
-		api.proto
-
-	It has these top-level messages:
-		Options
-*/
-package cri_runtimeoptions_v1
-
-import proto "github.com/gogo/protobuf/proto"
-import fmt "fmt"
-import math "math"
-import _ "github.com/gogo/protobuf/gogoproto"
-
-import strings "strings"
-import reflect "reflect"
-
-import io "io"
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
-
-type Options struct {
-	// TypeUrl specifies the type of the content inside the config file.
-	TypeUrl string `protobuf:"bytes,1,opt,name=type_url,json=typeUrl,proto3" json:"type_url,omitempty"`
-	// ConfigPath specifies the filesystem location of the config file
-	// used by the runtime.
-	ConfigPath string `protobuf:"bytes,2,opt,name=config_path,json=configPath,proto3" json:"config_path,omitempty"`
-}
-
-func (m *Options) Reset()                    { *m = Options{} }
-func (*Options) ProtoMessage()               {}
-func (*Options) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{0} }
-
-func (m *Options) GetTypeUrl() string {
-	if m != nil {
-		return m.TypeUrl
-	}
-	return ""
-}
-
-func (m *Options) GetConfigPath() string {
-	if m != nil {
-		return m.ConfigPath
-	}
-	return ""
-}
-
-func init() {
-	proto.RegisterType((*Options)(nil), "cri.runtimeoptions.v1.Options")
-}
-func (m *Options) Marshal() (dAtA []byte, err error) {
-	size := m.Size()
-	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(dAtA)
-	if err != nil {
-		return nil, err
-	}
-	return dAtA[:n], nil
-}
-
-func (m *Options) MarshalTo(dAtA []byte) (int, error) {
-	var i int
-	_ = i
-	var l int
-	_ = l
-	if len(m.TypeUrl) > 0 {
-		dAtA[i] = 0xa
-		i++
-		i = encodeVarintApi(dAtA, i, uint64(len(m.TypeUrl)))
-		i += copy(dAtA[i:], m.TypeUrl)
-	}
-	if len(m.ConfigPath) > 0 {
-		dAtA[i] = 0x12
-		i++
-		i = encodeVarintApi(dAtA, i, uint64(len(m.ConfigPath)))
-		i += copy(dAtA[i:], m.ConfigPath)
-	}
-	return i, nil
-}
-
-func encodeVarintApi(dAtA []byte, offset int, v uint64) int {
-	for v >= 1<<7 {
-		dAtA[offset] = uint8(v&0x7f | 0x80)
-		v >>= 7
-		offset++
-	}
-	dAtA[offset] = uint8(v)
-	return offset + 1
-}
-func (m *Options) Size() (n int) {
-	var l int
-	_ = l
-	l = len(m.TypeUrl)
-	if l > 0 {
-		n += 1 + l + sovApi(uint64(l))
-	}
-	l = len(m.ConfigPath)
-	if l > 0 {
-		n += 1 + l + sovApi(uint64(l))
-	}
-	return n
-}
-
-func sovApi(x uint64) (n int) {
-	for {
-		n++
-		x >>= 7
-		if x == 0 {
-			break
-		}
-	}
-	return n
-}
-func sozApi(x uint64) (n int) {
-	return sovApi(uint64((x << 1) ^ uint64((int64(x) >> 63))))
-}
-func (this *Options) String() string {
-	if this == nil {
-		return "nil"
-	}
-	s := strings.Join([]string{`&Options{`,
-		`TypeUrl:` + fmt.Sprintf("%v", this.TypeUrl) + `,`,
-		`ConfigPath:` + fmt.Sprintf("%v", this.ConfigPath) + `,`,
-		`}`,
-	}, "")
-	return s
-}
-func valueToStringApi(v interface{}) string {
-	rv := reflect.ValueOf(v)
-	if rv.IsNil() {
-		return "nil"
-	}
-	pv := reflect.Indirect(rv).Interface()
-	return fmt.Sprintf("*%v", pv)
-}
-func (m *Options) Unmarshal(dAtA []byte) error {
-	l := len(dAtA)
-	iNdEx := 0
-	for iNdEx < l {
-		preIndex := iNdEx
-		var wire uint64
-		for shift := uint(0); ; shift += 7 {
-			if shift >= 64 {
-				return ErrIntOverflowApi
-			}
-			if iNdEx >= l {
-				return io.ErrUnexpectedEOF
-			}
-			b := dAtA[iNdEx]
-			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
-			if b < 0x80 {
-				break
-			}
-		}
-		fieldNum := int32(wire >> 3)
-		wireType := int(wire & 0x7)
-		if wireType == 4 {
-			return fmt.Errorf("proto: Options: wiretype end group for non-group")
-		}
-		if fieldNum <= 0 {
-			return fmt.Errorf("proto: Options: illegal tag %d (wire type %d)", fieldNum, wire)
-		}
-		switch fieldNum {
-		case 1:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field TypeUrl", wireType)
-			}
-			var stringLen uint64
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowApi
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				stringLen |= (uint64(b) & 0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			intStringLen := int(stringLen)
-			if intStringLen < 0 {
-				return ErrInvalidLengthApi
-			}
-			postIndex := iNdEx + intStringLen
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.TypeUrl = string(dAtA[iNdEx:postIndex])
-			iNdEx = postIndex
-		case 2:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field ConfigPath", wireType)
-			}
-			var stringLen uint64
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowApi
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				stringLen |= (uint64(b) & 0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			intStringLen := int(stringLen)
-			if intStringLen < 0 {
-				return ErrInvalidLengthApi
-			}
-			postIndex := iNdEx + intStringLen
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.ConfigPath = string(dAtA[iNdEx:postIndex])
-			iNdEx = postIndex
-		default:
-			iNdEx = preIndex
-			skippy, err := skipApi(dAtA[iNdEx:])
-			if err != nil {
-				return err
-			}
-			if skippy < 0 {
-				return ErrInvalidLengthApi
-			}
-			if (iNdEx + skippy) > l {
-				return io.ErrUnexpectedEOF
-			}
-			iNdEx += skippy
-		}
-	}
-
-	if iNdEx > l {
-		return io.ErrUnexpectedEOF
-	}
-	return nil
-}
-func skipApi(dAtA []byte) (n int, err error) {
-	l := len(dAtA)
-	iNdEx := 0
-	for iNdEx < l {
-		var wire uint64
-		for shift := uint(0); ; shift += 7 {
-			if shift >= 64 {
-				return 0, ErrIntOverflowApi
-			}
-			if iNdEx >= l {
-				return 0, io.ErrUnexpectedEOF
-			}
-			b := dAtA[iNdEx]
-			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
-			if b < 0x80 {
-				break
-			}
-		}
-		wireType := int(wire & 0x7)
-		switch wireType {
-		case 0:
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return 0, ErrIntOverflowApi
-				}
-				if iNdEx >= l {
-					return 0, io.ErrUnexpectedEOF
-				}
-				iNdEx++
-				if dAtA[iNdEx-1] < 0x80 {
-					break
-				}
-			}
-			return iNdEx, nil
-		case 1:
-			iNdEx += 8
-			return iNdEx, nil
-		case 2:
-			var length int
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return 0, ErrIntOverflowApi
-				}
-				if iNdEx >= l {
-					return 0, io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				length |= (int(b) & 0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			iNdEx += length
-			if length < 0 {
-				return 0, ErrInvalidLengthApi
-			}
-			return iNdEx, nil
-		case 3:
-			for {
-				var innerWire uint64
-				var start int = iNdEx
-				for shift := uint(0); ; shift += 7 {
-					if shift >= 64 {
-						return 0, ErrIntOverflowApi
-					}
-					if iNdEx >= l {
-						return 0, io.ErrUnexpectedEOF
-					}
-					b := dAtA[iNdEx]
-					iNdEx++
-					innerWire |= (uint64(b) & 0x7F) << shift
-					if b < 0x80 {
-						break
-					}
-				}
-				innerWireType := int(innerWire & 0x7)
-				if innerWireType == 4 {
-					break
-				}
-				next, err := skipApi(dAtA[start:])
-				if err != nil {
-					return 0, err
-				}
-				iNdEx = start + next
-			}
-			return iNdEx, nil
-		case 4:
-			return iNdEx, nil
-		case 5:
-			iNdEx += 4
-			return iNdEx, nil
-		default:
-			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
-		}
-	}
-	panic("unreachable")
-}
-
-var (
-	ErrInvalidLengthApi = fmt.Errorf("proto: negative length found during unmarshaling")
-	ErrIntOverflowApi   = fmt.Errorf("proto: integer overflow")
-)
-
-func init() { proto.RegisterFile("api.proto", fileDescriptorApi) }
-
-var fileDescriptorApi = []byte{
-	// 183 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4c, 0x2c, 0xc8, 0xd4,
-	0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x12, 0x4d, 0x2e, 0xca, 0xd4, 0x2b, 0x2a, 0xcd, 0x2b, 0xc9,
-	0xcc, 0x4d, 0xcd, 0x2f, 0x28, 0xc9, 0xcc, 0xcf, 0x2b, 0xd6, 0x2b, 0x33, 0x94, 0xd2, 0x4d, 0xcf,
-	0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0xcf, 0x4f, 0xcf, 0xd7, 0x07, 0xab,
-	0x4e, 0x2a, 0x4d, 0x03, 0xf3, 0xc0, 0x1c, 0x30, 0x0b, 0x62, 0x8a, 0x92, 0x2b, 0x17, 0xbb, 0x3f,
-	0x44, 0xb3, 0x90, 0x24, 0x17, 0x47, 0x49, 0x65, 0x41, 0x6a, 0x7c, 0x69, 0x51, 0x8e, 0x04, 0xa3,
-	0x02, 0xa3, 0x06, 0x67, 0x10, 0x3b, 0x88, 0x1f, 0x5a, 0x94, 0x23, 0x24, 0xcf, 0xc5, 0x9d, 0x9c,
-	0x9f, 0x97, 0x96, 0x99, 0x1e, 0x5f, 0x90, 0x58, 0x92, 0x21, 0xc1, 0x04, 0x96, 0xe5, 0x82, 0x08,
-	0x05, 0x24, 0x96, 0x64, 0x38, 0xc9, 0x9c, 0x78, 0x28, 0xc7, 0x78, 0xe3, 0xa1, 0x1c, 0x43, 0xc3,
-	0x23, 0x39, 0xc6, 0x13, 0x8f, 0xe4, 0x18, 0x2f, 0x3c, 0x92, 0x63, 0x7c, 0xf0, 0x48, 0x8e, 0x71,
-	0xc2, 0x63, 0x39, 0x86, 0x24, 0x36, 0xb0, 0x5d, 0xc6, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0x07,
-	0x00, 0xf2, 0x18, 0xbe, 0x00, 0x00, 0x00,
-}

src/runtime/vendor/github.com/containerd/cri-containerd/pkg/api/runtimeoptions/v1/api.proto

@@ -1,22 +0,0 @@
-// To regenerate api.pb.go run `make proto`
-syntax = "proto3";
-
-package cri.runtimeoptions.v1;
-
-import "github.com/gogo/protobuf/gogoproto/gogo.proto";
-
-option (gogoproto.goproto_stringer_all) = false;
-option (gogoproto.stringer_all) =  true;
-option (gogoproto.goproto_getters_all) = true;
-option (gogoproto.marshaler_all) = true;
-option (gogoproto.sizer_all) = true;
-option (gogoproto.unmarshaler_all) = true;
-option (gogoproto.goproto_unrecognized_all) = false;
-
-message Options {
-  // TypeUrl specifies the type of the content inside the config file.
-  string type_url = 1;
-  // ConfigPath specifies the filesystem location of the config file
-  // used by the runtime.
-  string config_path = 2;
-}

src/runtime/vendor/github.com/stretchr/testify/assert/assertion_compare.go

@@ -13,12 +13,42 @@ const (
 	compareGreater
 )
 
+var (
+	intType   = reflect.TypeOf(int(1))
+	int8Type  = reflect.TypeOf(int8(1))
+	int16Type = reflect.TypeOf(int16(1))
+	int32Type = reflect.TypeOf(int32(1))
+	int64Type = reflect.TypeOf(int64(1))
+
+	uintType   = reflect.TypeOf(uint(1))
+	uint8Type  = reflect.TypeOf(uint8(1))
+	uint16Type = reflect.TypeOf(uint16(1))
+	uint32Type = reflect.TypeOf(uint32(1))
+	uint64Type = reflect.TypeOf(uint64(1))
+
+	float32Type = reflect.TypeOf(float32(1))
+	float64Type = reflect.TypeOf(float64(1))
+
+	stringType = reflect.TypeOf("")
+)
+
 func compare(obj1, obj2 interface{}, kind reflect.Kind) (CompareType, bool) {
+	obj1Value := reflect.ValueOf(obj1)
+	obj2Value := reflect.ValueOf(obj2)
+
+	// throughout this switch we try and avoid calling .Convert() if possible,
+	// as this has a pretty big performance impact
 	switch kind {
 	case reflect.Int:
 		{
-			intobj1 := obj1.(int)
-			intobj2 := obj2.(int)
+			intobj1, ok := obj1.(int)
+			if !ok {
+				intobj1 = obj1Value.Convert(intType).Interface().(int)
+			}
+			intobj2, ok := obj2.(int)
+			if !ok {
+				intobj2 = obj2Value.Convert(intType).Interface().(int)
+			}
 			if intobj1 > intobj2 {
 				return compareGreater, true
 			}
@@ -31,8 +61,14 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (CompareType, bool) {
 		}
 	case reflect.Int8:
 		{
-			int8obj1 := obj1.(int8)
-			int8obj2 := obj2.(int8)
+			int8obj1, ok := obj1.(int8)
+			if !ok {
+				int8obj1 = obj1Value.Convert(int8Type).Interface().(int8)
+			}
+			int8obj2, ok := obj2.(int8)
+			if !ok {
+				int8obj2 = obj2Value.Convert(int8Type).Interface().(int8)
+			}
 			if int8obj1 > int8obj2 {
 				return compareGreater, true
 			}
@@ -45,8 +81,14 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (CompareType, bool) {
 		}
 	case reflect.Int16:
 		{
-			int16obj1 := obj1.(int16)
-			int16obj2 := obj2.(int16)
+			int16obj1, ok := obj1.(int16)
+			if !ok {
+				int16obj1 = obj1Value.Convert(int16Type).Interface().(int16)
+			}
+			int16obj2, ok := obj2.(int16)
+			if !ok {
+				int16obj2 = obj2Value.Convert(int16Type).Interface().(int16)
+			}
 			if int16obj1 > int16obj2 {
 				return compareGreater, true
 			}
@@ -59,8 +101,14 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (CompareType, bool) {
 		}
 	case reflect.Int32:
 		{
-			int32obj1 := obj1.(int32)
-			int32obj2 := obj2.(int32)
+			int32obj1, ok := obj1.(int32)
+			if !ok {
+				int32obj1 = obj1Value.Convert(int32Type).Interface().(int32)
+			}
+			int32obj2, ok := obj2.(int32)
+			if !ok {
+				int32obj2 = obj2Value.Convert(int32Type).Interface().(int32)
+			}
 			if int32obj1 > int32obj2 {
 				return compareGreater, true
 			}
@@ -73,8 +121,14 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (CompareType, bool) {
 		}
 	case reflect.Int64:
 		{
-			int64obj1 := obj1.(int64)
-			int64obj2 := obj2.(int64)
+			int64obj1, ok := obj1.(int64)
+			if !ok {
+				int64obj1 = obj1Value.Convert(int64Type).Interface().(int64)
+			}
+			int64obj2, ok := obj2.(int64)
+			if !ok {
+				int64obj2 = obj2Value.Convert(int64Type).Interface().(int64)
+			}
 			if int64obj1 > int64obj2 {
 				return compareGreater, true
 			}
@@ -87,8 +141,14 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (CompareType, bool) {
 		}
 	case reflect.Uint:
 		{
-			uintobj1 := obj1.(uint)
-			uintobj2 := obj2.(uint)
+			uintobj1, ok := obj1.(uint)
+			if !ok {
+				uintobj1 = obj1Value.Convert(uintType).Interface().(uint)
+			}
+			uintobj2, ok := obj2.(uint)
+			if !ok {
+				uintobj2 = obj2Value.Convert(uintType).Interface().(uint)
+			}
 			if uintobj1 > uintobj2 {
 				return compareGreater, true
 			}
@@ -101,8 +161,14 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (CompareType, bool) {
 		}
 	case reflect.Uint8:
 		{
-			uint8obj1 := obj1.(uint8)
-			uint8obj2 := obj2.(uint8)
+			uint8obj1, ok := obj1.(uint8)
+			if !ok {
+				uint8obj1 = obj1Value.Convert(uint8Type).Interface().(uint8)
+			}
+			uint8obj2, ok := obj2.(uint8)
+			if !ok {
+				uint8obj2 = obj2Value.Convert(uint8Type).Interface().(uint8)
+			}
 			if uint8obj1 > uint8obj2 {
 				return compareGreater, true
 			}
@@ -115,8 +181,14 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (CompareType, bool) {
 		}
 	case reflect.Uint16:
 		{
-			uint16obj1 := obj1.(uint16)
-			uint16obj2 := obj2.(uint16)
+			uint16obj1, ok := obj1.(uint16)
+			if !ok {
+				uint16obj1 = obj1Value.Convert(uint16Type).Interface().(uint16)
+			}
+			uint16obj2, ok := obj2.(uint16)
+			if !ok {
+				uint16obj2 = obj2Value.Convert(uint16Type).Interface().(uint16)
+			}
 			if uint16obj1 > uint16obj2 {
 				return compareGreater, true
 			}
@@ -129,8 +201,14 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (CompareType, bool) {
 		}
 	case reflect.Uint32:
 		{
-			uint32obj1 := obj1.(uint32)
-			uint32obj2 := obj2.(uint32)
+			uint32obj1, ok := obj1.(uint32)
+			if !ok {
+				uint32obj1 = obj1Value.Convert(uint32Type).Interface().(uint32)
+			}
+			uint32obj2, ok := obj2.(uint32)
+			if !ok {
+				uint32obj2 = obj2Value.Convert(uint32Type).Interface().(uint32)
+			}
 			if uint32obj1 > uint32obj2 {
 				return compareGreater, true
 			}
@@ -143,8 +221,14 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (CompareType, bool) {
 		}
 	case reflect.Uint64:
 		{
-			uint64obj1 := obj1.(uint64)
-			uint64obj2 := obj2.(uint64)
+			uint64obj1, ok := obj1.(uint64)
+			if !ok {
+				uint64obj1 = obj1Value.Convert(uint64Type).Interface().(uint64)
+			}
+			uint64obj2, ok := obj2.(uint64)
+			if !ok {
+				uint64obj2 = obj2Value.Convert(uint64Type).Interface().(uint64)
+			}
 			if uint64obj1 > uint64obj2 {
 				return compareGreater, true
 			}
@@ -157,8 +241,14 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (CompareType, bool) {
 		}
 	case reflect.Float32:
 		{
-			float32obj1 := obj1.(float32)
-			float32obj2 := obj2.(float32)
+			float32obj1, ok := obj1.(float32)
+			if !ok {
+				float32obj1 = obj1Value.Convert(float32Type).Interface().(float32)
+			}
+			float32obj2, ok := obj2.(float32)
+			if !ok {
+				float32obj2 = obj2Value.Convert(float32Type).Interface().(float32)
+			}
 			if float32obj1 > float32obj2 {
 				return compareGreater, true
 			}
@@ -171,8 +261,14 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (CompareType, bool) {
 		}
 	case reflect.Float64:
 		{
-			float64obj1 := obj1.(float64)
-			float64obj2 := obj2.(float64)
+			float64obj1, ok := obj1.(float64)
+			if !ok {
+				float64obj1 = obj1Value.Convert(float64Type).Interface().(float64)
+			}
+			float64obj2, ok := obj2.(float64)
+			if !ok {
+				float64obj2 = obj2Value.Convert(float64Type).Interface().(float64)
+			}
 			if float64obj1 > float64obj2 {
 				return compareGreater, true
 			}
@@ -185,8 +281,14 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (CompareType, bool) {
 		}
 	case reflect.String:
 		{
-			stringobj1 := obj1.(string)
-			stringobj2 := obj2.(string)
+			stringobj1, ok := obj1.(string)
+			if !ok {
+				stringobj1 = obj1Value.Convert(stringType).Interface().(string)
+			}
+			stringobj2, ok := obj2.(string)
+			if !ok {
+				stringobj2 = obj2Value.Convert(stringType).Interface().(string)
+			}
 			if stringobj1 > stringobj2 {
 				return compareGreater, true
 			}
@@ -240,6 +342,24 @@ func LessOrEqual(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...inter
 	return compareTwoValues(t, e1, e2, []CompareType{compareLess, compareEqual}, "\"%v\" is not less than or equal to \"%v\"", msgAndArgs)
 }
 
+// Positive asserts that the specified element is positive
+//
+//    assert.Positive(t, 1)
+//    assert.Positive(t, 1.23)
+func Positive(t TestingT, e interface{}, msgAndArgs ...interface{}) bool {
+	zero := reflect.Zero(reflect.TypeOf(e))
+	return compareTwoValues(t, e, zero.Interface(), []CompareType{compareGreater}, "\"%v\" is not positive", msgAndArgs)
+}
+
+// Negative asserts that the specified element is negative
+//
+//    assert.Negative(t, -1)
+//    assert.Negative(t, -1.23)
+func Negative(t TestingT, e interface{}, msgAndArgs ...interface{}) bool {
+	zero := reflect.Zero(reflect.TypeOf(e))
+	return compareTwoValues(t, e, zero.Interface(), []CompareType{compareLess}, "\"%v\" is not negative", msgAndArgs)
+}
+
 func compareTwoValues(t TestingT, e1 interface{}, e2 interface{}, allowedComparesResults []CompareType, failMessage string, msgAndArgs ...interface{}) bool {
 	if h, ok := t.(tHelper); ok {
 		h.Helper()

src/runtime/vendor/github.com/stretchr/testify/assert/assertion_format.go

@@ -114,6 +114,24 @@ func Errorf(t TestingT, err error, msg string, args ...interface{}) bool {
 	return Error(t, err, append([]interface{}{msg}, args...)...)
 }
 
+// ErrorAsf asserts that at least one of the errors in err's chain matches target, and if so, sets target to that error value.
+// This is a wrapper for errors.As.
+func ErrorAsf(t TestingT, err error, target interface{}, msg string, args ...interface{}) bool {
+	if h, ok := t.(tHelper); ok {
+		h.Helper()
+	}
+	return ErrorAs(t, err, target, append([]interface{}{msg}, args...)...)
+}
+
+// ErrorIsf asserts that at least one of the errors in err's chain matches target.
+// This is a wrapper for errors.Is.
+func ErrorIsf(t TestingT, err error, target error, msg string, args ...interface{}) bool {
+	if h, ok := t.(tHelper); ok {
+		h.Helper()
+	}
+	return ErrorIs(t, err, target, append([]interface{}{msg}, args...)...)
+}
+
 // Eventuallyf asserts that given condition will be met in waitFor time,
 // periodically checking target function each tick.
 //
@@ -321,6 +339,54 @@ func InEpsilonSlicef(t TestingT, expected interface{}, actual interface{}, epsil
 	return InEpsilonSlice(t, expected, actual, epsilon, append([]interface{}{msg}, args...)...)
 }
 
+// IsDecreasingf asserts that the collection is decreasing
+//
+//    assert.IsDecreasingf(t, []int{2, 1, 0}, "error message %s", "formatted")
+//    assert.IsDecreasingf(t, []float{2, 1}, "error message %s", "formatted")
+//    assert.IsDecreasingf(t, []string{"b", "a"}, "error message %s", "formatted")
+func IsDecreasingf(t TestingT, object interface{}, msg string, args ...interface{}) bool {
+	if h, ok := t.(tHelper); ok {
+		h.Helper()
+	}
+	return IsDecreasing(t, object, append([]interface{}{msg}, args...)...)
+}
+
+// IsIncreasingf asserts that the collection is increasing
+//
+//    assert.IsIncreasingf(t, []int{1, 2, 3}, "error message %s", "formatted")
+//    assert.IsIncreasingf(t, []float{1, 2}, "error message %s", "formatted")
+//    assert.IsIncreasingf(t, []string{"a", "b"}, "error message %s", "formatted")
+func IsIncreasingf(t TestingT, object interface{}, msg string, args ...interface{}) bool {
+	if h, ok := t.(tHelper); ok {
+		h.Helper()
+	}
+	return IsIncreasing(t, object, append([]interface{}{msg}, args...)...)
+}
+
+// IsNonDecreasingf asserts that the collection is not decreasing
+//
+//    assert.IsNonDecreasingf(t, []int{1, 1, 2}, "error message %s", "formatted")
+//    assert.IsNonDecreasingf(t, []float{1, 2}, "error message %s", "formatted")
+//    assert.IsNonDecreasingf(t, []string{"a", "b"}, "error message %s", "formatted")
+func IsNonDecreasingf(t TestingT, object interface{}, msg string, args ...interface{}) bool {
+	if h, ok := t.(tHelper); ok {
+		h.Helper()
+	}
+	return IsNonDecreasing(t, object, append([]interface{}{msg}, args...)...)
+}
+
+// IsNonIncreasingf asserts that the collection is not increasing
+//
+//    assert.IsNonIncreasingf(t, []int{2, 1, 1}, "error message %s", "formatted")
+//    assert.IsNonIncreasingf(t, []float{2, 1}, "error message %s", "formatted")
+//    assert.IsNonIncreasingf(t, []string{"b", "a"}, "error message %s", "formatted")
+func IsNonIncreasingf(t TestingT, object interface{}, msg string, args ...interface{}) bool {
+	if h, ok := t.(tHelper); ok {
+		h.Helper()
+	}
+	return IsNonIncreasing(t, object, append([]interface{}{msg}, args...)...)
+}
+
 // IsTypef asserts that the specified objects are of the same type.
 func IsTypef(t TestingT, expectedType interface{}, object interface{}, msg string, args ...interface{}) bool {
 	if h, ok := t.(tHelper); ok {
@@ -375,6 +441,17 @@ func LessOrEqualf(t TestingT, e1 interface{}, e2 interface{}, msg string, args .
 	return LessOrEqual(t, e1, e2, append([]interface{}{msg}, args...)...)
 }
 
+// Negativef asserts that the specified element is negative
+//
+//    assert.Negativef(t, -1, "error message %s", "formatted")
+//    assert.Negativef(t, -1.23, "error message %s", "formatted")
+func Negativef(t TestingT, e interface{}, msg string, args ...interface{}) bool {
+	if h, ok := t.(tHelper); ok {
+		h.Helper()
+	}
+	return Negative(t, e, append([]interface{}{msg}, args...)...)
+}
+
 // Neverf asserts that the given condition doesn't satisfy in waitFor time,
 // periodically checking the target function each tick.
 //
@@ -476,6 +553,15 @@ func NotEqualValuesf(t TestingT, expected interface{}, actual interface{}, msg s
 	return NotEqualValues(t, expected, actual, append([]interface{}{msg}, args...)...)
 }
 
+// NotErrorIsf asserts that at none of the errors in err's chain matches target.
+// This is a wrapper for errors.Is.
+func NotErrorIsf(t TestingT, err error, target error, msg string, args ...interface{}) bool {
+	if h, ok := t.(tHelper); ok {
+		h.Helper()
+	}
+	return NotErrorIs(t, err, target, append([]interface{}{msg}, args...)...)
+}
+
 // NotNilf asserts that the specified object is not nil.
 //
 //    assert.NotNilf(t, err, "error message %s", "formatted")
@@ -572,6 +658,17 @@ func PanicsWithValuef(t TestingT, expected interface{}, f PanicTestFunc, msg str
 	return PanicsWithValue(t, expected, f, append([]interface{}{msg}, args...)...)
 }
 
+// Positivef asserts that the specified element is positive
+//
+//    assert.Positivef(t, 1, "error message %s", "formatted")
+//    assert.Positivef(t, 1.23, "error message %s", "formatted")
+func Positivef(t TestingT, e interface{}, msg string, args ...interface{}) bool {
+	if h, ok := t.(tHelper); ok {
+		h.Helper()
+	}
+	return Positive(t, e, append([]interface{}{msg}, args...)...)
+}
+
 // Regexpf asserts that a specified regexp matches a string.
 //
 //  assert.Regexpf(t, regexp.MustCompile("start"), "it's starting", "error message %s", "formatted")

src/runtime/vendor/github.com/stretchr/testify/assert/assertion_forward.go

@@ -204,6 +204,42 @@ func (a *Assertions) Error(err error, msgAndArgs ...interface{}) bool {
 	return Error(a.t, err, msgAndArgs...)
 }
 
+// ErrorAs asserts that at least one of the errors in err's chain matches target, and if so, sets target to that error value.
+// This is a wrapper for errors.As.
+func (a *Assertions) ErrorAs(err error, target interface{}, msgAndArgs ...interface{}) bool {
+	if h, ok := a.t.(tHelper); ok {
+		h.Helper()
+	}
+	return ErrorAs(a.t, err, target, msgAndArgs...)
+}
+
+// ErrorAsf asserts that at least one of the errors in err's chain matches target, and if so, sets target to that error value.
+// This is a wrapper for errors.As.
+func (a *Assertions) ErrorAsf(err error, target interface{}, msg string, args ...interface{}) bool {
+	if h, ok := a.t.(tHelper); ok {
+		h.Helper()
+	}
+	return ErrorAsf(a.t, err, target, msg, args...)
+}
+
+// ErrorIs asserts that at least one of the errors in err's chain matches target.
+// This is a wrapper for errors.Is.
+func (a *Assertions) ErrorIs(err error, target error, msgAndArgs ...interface{}) bool {
+	if h, ok := a.t.(tHelper); ok {
+		h.Helper()
+	}
+	return ErrorIs(a.t, err, target, msgAndArgs...)
+}
+
+// ErrorIsf asserts that at least one of the errors in err's chain matches target.
+// This is a wrapper for errors.Is.
+func (a *Assertions) ErrorIsf(err error, target error, msg string, args ...interface{}) bool {
+	if h, ok := a.t.(tHelper); ok {
+		h.Helper()
+	}
+	return ErrorIsf(a.t, err, target, msg, args...)
+}
+
 // Errorf asserts that a function returned an error (i.e. not `nil`).
 //
 //   actualObj, err := SomeFunction()
@@ -631,6 +667,102 @@ func (a *Assertions) InEpsilonf(expected interface{}, actual interface{}, epsilo
 	return InEpsilonf(a.t, expected, actual, epsilon, msg, args...)
 }
 
+// IsDecreasing asserts that the collection is decreasing
+//
+//    a.IsDecreasing([]int{2, 1, 0})
+//    a.IsDecreasing([]float{2, 1})
+//    a.IsDecreasing([]string{"b", "a"})
+func (a *Assertions) IsDecreasing(object interface{}, msgAndArgs ...interface{}) bool {
+	if h, ok := a.t.(tHelper); ok {
+		h.Helper()
+	}
+	return IsDecreasing(a.t, object, msgAndArgs...)
+}
+
+// IsDecreasingf asserts that the collection is decreasing
+//
+//    a.IsDecreasingf([]int{2, 1, 0}, "error message %s", "formatted")
+//    a.IsDecreasingf([]float{2, 1}, "error message %s", "formatted")
+//    a.IsDecreasingf([]string{"b", "a"}, "error message %s", "formatted")
+func (a *Assertions) IsDecreasingf(object interface{}, msg string, args ...interface{}) bool {
+	if h, ok := a.t.(tHelper); ok {
+		h.Helper()
+	}
+	return IsDecreasingf(a.t, object, msg, args...)
+}
+
+// IsIncreasing asserts that the collection is increasing
+//
+//    a.IsIncreasing([]int{1, 2, 3})
+//    a.IsIncreasing([]float{1, 2})
+//    a.IsIncreasing([]string{"a", "b"})
+func (a *Assertions) IsIncreasing(object interface{}, msgAndArgs ...interface{}) bool {
+	if h, ok := a.t.(tHelper); ok {
+		h.Helper()
+	}
+	return IsIncreasing(a.t, object, msgAndArgs...)
+}
+
+// IsIncreasingf asserts that the collection is increasing
+//
+//    a.IsIncreasingf([]int{1, 2, 3}, "error message %s", "formatted")
+//    a.IsIncreasingf([]float{1, 2}, "error message %s", "formatted")
+//    a.IsIncreasingf([]string{"a", "b"}, "error message %s", "formatted")
+func (a *Assertions) IsIncreasingf(object interface{}, msg string, args ...interface{}) bool {
+	if h, ok := a.t.(tHelper); ok {
+		h.Helper()
+	}
+	return IsIncreasingf(a.t, object, msg, args...)
+}
+
+// IsNonDecreasing asserts that the collection is not decreasing
+//
+//    a.IsNonDecreasing([]int{1, 1, 2})
+//    a.IsNonDecreasing([]float{1, 2})
+//    a.IsNonDecreasing([]string{"a", "b"})
+func (a *Assertions) IsNonDecreasing(object interface{}, msgAndArgs ...interface{}) bool {
+	if h, ok := a.t.(tHelper); ok {
+		h.Helper()
+	}
+	return IsNonDecreasing(a.t, object, msgAndArgs...)
+}
+
+// IsNonDecreasingf asserts that the collection is not decreasing
+//
+//    a.IsNonDecreasingf([]int{1, 1, 2}, "error message %s", "formatted")
+//    a.IsNonDecreasingf([]float{1, 2}, "error message %s", "formatted")
+//    a.IsNonDecreasingf([]string{"a", "b"}, "error message %s", "formatted")
+func (a *Assertions) IsNonDecreasingf(object interface{}, msg string, args ...interface{}) bool {
+	if h, ok := a.t.(tHelper); ok {
+		h.Helper()
+	}
+	return IsNonDecreasingf(a.t, object, msg, args...)
+}
+
+// IsNonIncreasing asserts that the collection is not increasing
+//
+//    a.IsNonIncreasing([]int{2, 1, 1})
+//    a.IsNonIncreasing([]float{2, 1})
+//    a.IsNonIncreasing([]string{"b", "a"})
+func (a *Assertions) IsNonIncreasing(object interface{}, msgAndArgs ...interface{}) bool {
+	if h, ok := a.t.(tHelper); ok {
+		h.Helper()
+	}
+	return IsNonIncreasing(a.t, object, msgAndArgs...)
+}
+
+// IsNonIncreasingf asserts that the collection is not increasing
+//
+//    a.IsNonIncreasingf([]int{2, 1, 1}, "error message %s", "formatted")
+//    a.IsNonIncreasingf([]float{2, 1}, "error message %s", "formatted")
+//    a.IsNonIncreasingf([]string{"b", "a"}, "error message %s", "formatted")
+func (a *Assertions) IsNonIncreasingf(object interface{}, msg string, args ...interface{}) bool {
+	if h, ok := a.t.(tHelper); ok {
+		h.Helper()
+	}
+	return IsNonIncreasingf(a.t, object, msg, args...)
+}
+
 // IsType asserts that the specified objects are of the same type.
 func (a *Assertions) IsType(expectedType interface{}, object interface{}, msgAndArgs ...interface{}) bool {
 	if h, ok := a.t.(tHelper); ok {
@@ -739,6 +871,28 @@ func (a *Assertions) Lessf(e1 interface{}, e2 interface{}, msg string, args ...i
 	return Lessf(a.t, e1, e2, msg, args...)
 }
 
+// Negative asserts that the specified element is negative
+//
+//    a.Negative(-1)
+//    a.Negative(-1.23)
+func (a *Assertions) Negative(e interface{}, msgAndArgs ...interface{}) bool {
+	if h, ok := a.t.(tHelper); ok {
+		h.Helper()
+	}
+	return Negative(a.t, e, msgAndArgs...)
+}
+
+// Negativef asserts that the specified element is negative
+//
+//    a.Negativef(-1, "error message %s", "formatted")
+//    a.Negativef(-1.23, "error message %s", "formatted")
+func (a *Assertions) Negativef(e interface{}, msg string, args ...interface{}) bool {
+	if h, ok := a.t.(tHelper); ok {
+		h.Helper()
+	}
+	return Negativef(a.t, e, msg, args...)
+}
+
 // Never asserts that the given condition doesn't satisfy in waitFor time,
 // periodically checking the target function each tick.
 //
@@ -941,6 +1095,24 @@ func (a *Assertions) NotEqualf(expected interface{}, actual interface{}, msg str
 	return NotEqualf(a.t, expected, actual, msg, args...)
 }
 
+// NotErrorIs asserts that at none of the errors in err's chain matches target.
+// This is a wrapper for errors.Is.
+func (a *Assertions) NotErrorIs(err error, target error, msgAndArgs ...interface{}) bool {
+	if h, ok := a.t.(tHelper); ok {
+		h.Helper()
+	}
+	return NotErrorIs(a.t, err, target, msgAndArgs...)
+}
+
+// NotErrorIsf asserts that at none of the errors in err's chain matches target.
+// This is a wrapper for errors.Is.
+func (a *Assertions) NotErrorIsf(err error, target error, msg string, args ...interface{}) bool {
+	if h, ok := a.t.(tHelper); ok {
+		h.Helper()
+	}
+	return NotErrorIsf(a.t, err, target, msg, args...)
+}
+
 // NotNil asserts that the specified object is not nil.
 //
 //    a.NotNil(err)
@@ -1133,6 +1305,28 @@ func (a *Assertions) Panicsf(f PanicTestFunc, msg string, args ...interface{}) b
 	return Panicsf(a.t, f, msg, args...)
 }
 
+// Positive asserts that the specified element is positive
+//
+//    a.Positive(1)
+//    a.Positive(1.23)
+func (a *Assertions) Positive(e interface{}, msgAndArgs ...interface{}) bool {
+	if h, ok := a.t.(tHelper); ok {
+		h.Helper()
+	}
+	return Positive(a.t, e, msgAndArgs...)
+}
+
+// Positivef asserts that the specified element is positive
+//
+//    a.Positivef(1, "error message %s", "formatted")
+//    a.Positivef(1.23, "error message %s", "formatted")
+func (a *Assertions) Positivef(e interface{}, msg string, args ...interface{}) bool {
+	if h, ok := a.t.(tHelper); ok {
+		h.Helper()
+	}
+	return Positivef(a.t, e, msg, args...)
+}
+
 // Regexp asserts that a specified regexp matches a string.
 //
 //  a.Regexp(regexp.MustCompile("start"), "it's starting")

src/runtime/vendor/github.com/stretchr/testify/assert/assertion_order.go

@@ -0,0 +1,81 @@
+package assert
+
+import (
+	"fmt"
+	"reflect"
+)
+
+// isOrdered checks that collection contains orderable elements.
+func isOrdered(t TestingT, object interface{}, allowedComparesResults []CompareType, failMessage string, msgAndArgs ...interface{}) bool {
+	objKind := reflect.TypeOf(object).Kind()
+	if objKind != reflect.Slice && objKind != reflect.Array {
+		return false
+	}
+
+	objValue := reflect.ValueOf(object)
+	objLen := objValue.Len()
+
+	if objLen <= 1 {
+		return true
+	}
+
+	value := objValue.Index(0)
+	valueInterface := value.Interface()
+	firstValueKind := value.Kind()
+
+	for i := 1; i < objLen; i++ {
+		prevValue := value
+		prevValueInterface := valueInterface
+
+		value = objValue.Index(i)
+		valueInterface = value.Interface()
+
+		compareResult, isComparable := compare(prevValueInterface, valueInterface, firstValueKind)
+
+		if !isComparable {
+			return Fail(t, fmt.Sprintf("Can not compare type \"%s\" and \"%s\"", reflect.TypeOf(value), reflect.TypeOf(prevValue)), msgAndArgs...)
+		}
+
+		if !containsValue(allowedComparesResults, compareResult) {
+			return Fail(t, fmt.Sprintf(failMessage, prevValue, value), msgAndArgs...)
+		}
+	}
+
+	return true
+}
+
+// IsIncreasing asserts that the collection is increasing
+//
+//    assert.IsIncreasing(t, []int{1, 2, 3})
+//    assert.IsIncreasing(t, []float{1, 2})
+//    assert.IsIncreasing(t, []string{"a", "b"})
+func IsIncreasing(t TestingT, object interface{}, msgAndArgs ...interface{}) bool {
+	return isOrdered(t, object, []CompareType{compareLess}, "\"%v\" is not less than \"%v\"", msgAndArgs)
+}
+
+// IsNonIncreasing asserts that the collection is not increasing
+//
+//    assert.IsNonIncreasing(t, []int{2, 1, 1})
+//    assert.IsNonIncreasing(t, []float{2, 1})
+//    assert.IsNonIncreasing(t, []string{"b", "a"})
+func IsNonIncreasing(t TestingT, object interface{}, msgAndArgs ...interface{}) bool {
+	return isOrdered(t, object, []CompareType{compareEqual, compareGreater}, "\"%v\" is not greater than or equal to \"%v\"", msgAndArgs)
+}
+
+// IsDecreasing asserts that the collection is decreasing
+//
+//    assert.IsDecreasing(t, []int{2, 1, 0})
+//    assert.IsDecreasing(t, []float{2, 1})
+//    assert.IsDecreasing(t, []string{"b", "a"})
+func IsDecreasing(t TestingT, object interface{}, msgAndArgs ...interface{}) bool {
+	return isOrdered(t, object, []CompareType{compareGreater}, "\"%v\" is not greater than \"%v\"", msgAndArgs)
+}
+
+// IsNonDecreasing asserts that the collection is not decreasing
+//
+//    assert.IsNonDecreasing(t, []int{1, 1, 2})
+//    assert.IsNonDecreasing(t, []float{1, 2})
+//    assert.IsNonDecreasing(t, []string{"a", "b"})
+func IsNonDecreasing(t TestingT, object interface{}, msgAndArgs ...interface{}) bool {
+	return isOrdered(t, object, []CompareType{compareLess, compareEqual}, "\"%v\" is not less than or equal to \"%v\"", msgAndArgs)
+}

src/runtime/vendor/github.com/stretchr/testify/assert/assertions.go

@@ -172,8 +172,8 @@ func isTest(name, prefix string) bool {
 	if len(name) == len(prefix) { // "Test" is ok
 		return true
 	}
-	rune, _ := utf8.DecodeRuneInString(name[len(prefix):])
-	return !unicode.IsLower(rune)
+	r, _ := utf8.DecodeRuneInString(name[len(prefix):])
+	return !unicode.IsLower(r)
 }
 
 func messageFromMsgAndArgs(msgAndArgs ...interface{}) string {
@@ -1622,6 +1622,7 @@ var spewConfig = spew.ConfigState{
 	DisableCapacities:       true,
 	SortKeys:                true,
 	DisableMethods:          true,
+	MaxDepth:                10,
 }
 
 type tHelper interface {
@@ -1693,3 +1694,81 @@ func Never(t TestingT, condition func() bool, waitFor time.Duration, tick time.D
 		}
 	}
 }
+
+// ErrorIs asserts that at least one of the errors in err's chain matches target.
+// This is a wrapper for errors.Is.
+func ErrorIs(t TestingT, err, target error, msgAndArgs ...interface{}) bool {
+	if h, ok := t.(tHelper); ok {
+		h.Helper()
+	}
+	if errors.Is(err, target) {
+		return true
+	}
+
+	var expectedText string
+	if target != nil {
+		expectedText = target.Error()
+	}
+
+	chain := buildErrorChainString(err)
+
+	return Fail(t, fmt.Sprintf("Target error should be in err chain:\n"+
+		"expected: %q\n"+
+		"in chain: %s", expectedText, chain,
+	), msgAndArgs...)
+}
+
+// NotErrorIs asserts that at none of the errors in err's chain matches target.
+// This is a wrapper for errors.Is.
+func NotErrorIs(t TestingT, err, target error, msgAndArgs ...interface{}) bool {
+	if h, ok := t.(tHelper); ok {
+		h.Helper()
+	}
+	if !errors.Is(err, target) {
+		return true
+	}
+
+	var expectedText string
+	if target != nil {
+		expectedText = target.Error()
+	}
+
+	chain := buildErrorChainString(err)
+
+	return Fail(t, fmt.Sprintf("Target error should not be in err chain:\n"+
+		"found: %q\n"+
+		"in chain: %s", expectedText, chain,
+	), msgAndArgs...)
+}
+
+// ErrorAs asserts that at least one of the errors in err's chain matches target, and if so, sets target to that error value.
+// This is a wrapper for errors.As.
+func ErrorAs(t TestingT, err error, target interface{}, msgAndArgs ...interface{}) bool {
+	if h, ok := t.(tHelper); ok {
+		h.Helper()
+	}
+	if errors.As(err, target) {
+		return true
+	}
+
+	chain := buildErrorChainString(err)
+
+	return Fail(t, fmt.Sprintf("Should be in error chain:\n"+
+		"expected: %q\n"+
+		"in chain: %s", target, chain,
+	), msgAndArgs...)
+}
+
+func buildErrorChainString(err error) string {
+	if err == nil {
+		return ""
+	}
+
+	e := errors.Unwrap(err)
+	chain := fmt.Sprintf("%q", err.Error())
+	for e != nil {
+		chain += fmt.Sprintf("\n\t%q", e.Error())
+		e = errors.Unwrap(e)
+	}
+	return chain
+}

src/runtime/vendor/go.opencensus.io/.travis.yml

@@ -1,17 +0,0 @@
-language: go
-
-go_import_path: go.opencensus.io
-
-go:
-  - 1.11.x
-
-env:
-  global:
-    GO111MODULE=on
-
-before_script:
-  - make install-tools
-
-script:
-  - make travis-ci
-  - go run internal/check/version.go # TODO move this to makefile

src/runtime/vendor/go.opencensus.io/go.mod

@@ -1,15 +1,12 @@
 module go.opencensus.io
 
 require (
-	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6
-	github.com/golang/protobuf v1.3.1
-	github.com/google/go-cmp v0.3.0
-	github.com/stretchr/testify v1.4.0
-	golang.org/x/net v0.0.0-20190620200207-3b0461eec859
-	golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd // indirect
-	golang.org/x/text v0.3.2 // indirect
-	google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb // indirect
-	google.golang.org/grpc v1.20.1
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e
+	github.com/golang/protobuf v1.4.3
+	github.com/google/go-cmp v0.5.3
+	github.com/stretchr/testify v1.6.1
+	golang.org/x/net v0.0.0-20201110031124-69a78807bb2b
+	google.golang.org/grpc v1.33.2
 )
 
 go 1.13