Merge pull request #3402 from snir911/2.3.1-branch-bump

# Kata Containers 2.3.1

.github/workflows/kata-deploy-test.yaml

@@ -5,60 +5,121 @@ on:
 name: test-kata-deploy
 
 jobs:
-  check_comments:
-    if: ${{ github.event.issue.pull_request }}
+  check-comment-and-membership:
     runs-on: ubuntu-latest
+    if: |
+      github.event.issue.pull_request
+      && github.event_name == 'issue_comment'
+      && github.event.action == 'created'
+      && startsWith(github.event.comment.body, '/test_kata_deploy')
     steps:
-      - name: Check for Command
-        id: command
-        uses: kata-containers/slash-command-action@v1
+      - name: Check membership
+        uses: kata-containers/is-organization-member@1.0.1
+        id: is_organization_member
         with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          command: "test_kata_deploy"
-          reaction: "true"
-          reaction-type: "eyes"
-          allow-edits: "false"
-          permission-level: admin
-      - name: verify command arg is kata-deploy
+          organization: kata-containers
+          username: ${{ github.event.comment.user.login }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Fail if not member
         run: |
-           echo "The command was '${{ steps.command.outputs.command-name }}' with arguments '${{ steps.command.outputs.command-arguments }}'"
+          result=${{ steps.is_organization_member.outputs.result }}
+          if [ $result == false ]; then
+              user=${{ github.event.comment.user.login }}
+              echo Either ${user} is not part of the kata-containers organization
+              echo or ${user} has its Organization Visibility set to Private at
+              echo https://github.com/orgs/kata-containers/people?query=${user}
+              echo 
+              echo Ensure you change your Organization Visibility to Public and
+              echo trigger the test again.
+              exit 1
+          fi
 
-  create-and-test-container:
-    needs: check_comments
+  build-asset:
     runs-on: ubuntu-latest
+    needs: check-comment-and-membership
+    strategy:
+      matrix:
+        asset:
+          - cloud-hypervisor
+          - firecracker
+          - kernel
+          - qemu
+          - rootfs-image
+          - rootfs-initrd
+          - shim-v2
     steps:
-      - name: get-PR-ref
-        id: get-PR-ref
+      - uses: actions/checkout@v2
+      - name: Install docker
         run: |
-            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
-            echo "reference for PR: " ${ref}
-            echo "##[set-output name=pr-ref;]${ref}"
+          curl -fsSL https://test.docker.com -o test-docker.sh
+          sh test-docker.sh
 
-      - name: check out
-        uses: actions/checkout@v2
-        with:
-           ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
-
-      - name: build-container-image
-        id: build-container-image
+      - name: Build ${{ matrix.asset }}
         run: |
-            PR_SHA=$(git log --format=format:%H -n1)
-            VERSION="2.0.0"
-            ARTIFACT_URL="https://github.com/kata-containers/kata-containers/releases/download/${VERSION}/kata-static-${VERSION}-x86_64.tar.xz"
-            wget "${ARTIFACT_URL}" -O tools/packaging/kata-deploy/kata-static.tar.xz
-            docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:${PR_SHA} -t quay.io/kata-containers/kata-deploy-ci:${PR_SHA} ./tools/packaging/kata-deploy
-            docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
-            docker push katadocker/kata-deploy-ci:$PR_SHA
-            docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
-            docker push quay.io/kata-containers/kata-deploy-ci:$PR_SHA
-            echo "##[set-output name=pr-sha;]${PR_SHA}"
-
-      - name: test-kata-deploy-ci-in-aks
-        uses: ./tools/packaging/kata-deploy/action
-        with:
-          packaging-sha: ${{ steps.build-container-image.outputs.pr-sha }}
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
         env:
-          PKG_SHA: ${{ steps.build-container-image.outputs.pr-sha }}
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: build-asset
+    steps:
+      - uses: actions/checkout@v2
+      - name: get-artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: kata-artifacts
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: kata-static-tarball
+          path: kata-static.tar.xz
+
+  kata-deploy:
+    needs: create-kata-tarball
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v2
+        with:
+          name: kata-static-tarball
+      - name: build-and-push-kata-deploy-ci
+        id: build-and-push-kata-deploy-ci
+        run: |
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          pushd $GITHUB_WORKSPACE
+          git checkout $tag
+          pkg_sha=$(git rev-parse HEAD)
+          popd
+          mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
+          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t quay.io/kata-containers/kata-deploy-ci:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
+          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
+          docker push quay.io/kata-containers/kata-deploy-ci:$pkg_sha
+          mkdir -p packaging/kata-deploy
+          ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
+          echo "::set-output name=PKG_SHA::${pkg_sha}"
+      - name: test-kata-deploy-ci-in-aks
+        uses: ./packaging/kata-deploy/action
+        with:
+          packaging-sha: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
+        env:
+          PKG_SHA: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
           AZ_APPID: ${{ secrets.AZ_APPID }}
           AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
           AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}

.github/workflows/main.yaml

@@ -1,295 +0,0 @@
-name: Publish release tarball
-on: 
-  push: 
-    tags:
-     - '1.*'
-
-jobs:
-  get-artifact-list:
-    runs-on: ubuntu-latest
-    steps:
-      - name: get the list
-        run: |
-         pushd $GITHUB_WORKSPACE
-         tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-         git checkout $tag
-         popd
-         $GITHUB_WORKSPACE/tools/packaging/artifact-list.sh > artifact-list.txt
-      - name: save-artifact-list
-        uses: actions/upload-artifact@master
-        with:
-          name: artifact-list
-          path: artifact-list.txt
-
-  build-kernel:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_kernel"
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifact-list
-        uses: actions/download-artifact@master
-        with:
-          name: artifact-list
-      - run: |
-         sudo apt-get update && sudo apt install -y flex bison libelf-dev bc iptables
-      - name: build-kernel
-        run: |
-         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@master
-        with:
-          name: kata-artifacts
-          path: kata-static-kernel.tar.gz
-
-  build-experimental-kernel:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_experimental_kernel"
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifact-list
-        uses: actions/download-artifact@master
-        with:
-          name: artifact-list
-      - run: |
-         sudo apt-get update && sudo apt install -y flex bison libelf-dev bc iptables
-      - name: build-experimental-kernel
-        run: |
-         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@master
-        with:
-          name: kata-artifacts
-          path: kata-static-experimental-kernel.tar.gz
-
-  build-qemu:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_qemu"
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifact-list
-        uses: actions/download-artifact@master
-        with:
-          name: artifact-list
-      - name: build-qemu
-        run: |
-         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@master
-        with:
-          name: kata-artifacts
-          path: kata-static-qemu.tar.gz
-
-  # Job for building the image
-  build-image:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_image"
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifact-list
-        uses: actions/download-artifact@master
-        with:
-          name: artifact-list
-      - name: build-image
-        run: |
-         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@master
-        with:
-          name: kata-artifacts
-          path: kata-static-image.tar.gz
-
-  # Job for building firecracker hypervisor
-  build-firecracker:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_firecracker"
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifact-list
-        uses: actions/download-artifact@master
-        with:
-          name: artifact-list
-      - name: build-firecracker
-        run: |
-         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@master
-        with:
-          name: kata-artifacts
-          path: kata-static-firecracker.tar.gz
-
-  # Job for building cloud-hypervisor
-  build-clh:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_clh"
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifact-list
-        uses: actions/download-artifact@master
-        with:
-          name: artifact-list
-      - name: build-clh
-        run: |
-         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@master
-        with:
-          name: kata-artifacts
-          path: kata-static-clh.tar.gz
-
-  # Job for building kata components
-  build-kata-components:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_kata_components"
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifact-list
-        uses: actions/download-artifact@master
-        with:
-          name: artifact-list
-      - name: build-kata-components
-        run: |
-         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@master
-        with:
-          name: kata-artifacts
-          path: kata-static-kata-components.tar.gz
-
-  gather-artifacts:
-    runs-on: ubuntu-16.04
-    needs: [build-experimental-kernel, build-kernel, build-qemu, build-image, build-firecracker, build-kata-components, build-clh]
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifacts
-        uses: actions/download-artifact@master
-        with:
-          name: kata-artifacts
-      - name: colate-artifacts
-        run: |
-          $GITHUB_WORKSPACE/.github/workflows/gather-artifacts.sh
-      - name: store-artifacts
-        uses: actions/upload-artifact@master
-        with:
-          name: release-candidate
-          path: kata-static.tar.xz
-
-  kata-deploy:
-    needs: gather-artifacts
-    runs-on: ubuntu-latest
-    steps:
-      - name: get-artifacts
-        uses: actions/download-artifact@master
-        with:
-          name: release-candidate
-      - name: build-and-push-kata-deploy-ci
-        id: build-and-push-kata-deploy-ci
-        run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          git clone https://github.com/kata-containers/packaging
-          pushd packaging
-          git checkout $tag
-          pkg_sha=$(git rev-parse HEAD)
-          popd
-          mv release-candidate/kata-static.tar.xz ./packaging/kata-deploy/kata-static.tar.xz
-          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:$pkg_sha -t quay.io/kata-containers/kata-deploy-ci:$pkg_sha ./packaging/kata-deploy
-          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
-          docker push katadocker/kata-deploy-ci:$pkg_sha
-          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
-          docker push quay.io/kata-containers/kata-deploy-ci:$pkg_sha
-          echo "::set-output name=PKG_SHA::${pkg_sha}"
-      - name: test-kata-deploy-ci-in-aks
-        uses: ./packaging/kata-deploy/action
-        with:
-          packaging-sha: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
-        env:
-          PKG_SHA: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
-          AZ_APPID: ${{ secrets.AZ_APPID }}
-          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
-          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
-          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
-      - name: push-tarball
-        run: |
-          # tag the container image we created and push to DockerHub
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          docker tag katadocker/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} katadocker/kata-deploy:${tag}
-          docker push katadocker/kata-deploy:${tag}
-
-  upload-static-tarball:
-    needs: kata-deploy
-    runs-on: ubuntu-latest
-    steps:
-      - name: download-artifacts
-        uses: actions/download-artifact@master
-        with:
-          name: release-candidate
-      - name: install hub
-        run: |
-          HUB_VER=$(curl -s "https://api.github.com/repos/github/hub/releases/latest" | jq -r .tag_name | sed 's/^v//')
-          wget -q -O- https://github.com/github/hub/releases/download/v$HUB_VER/hub-linux-amd64-$HUB_VER.tgz | \
-          tar xz --strip-components=2 --wildcards '*/bin/hub' && sudo mv hub /usr/local/bin/hub
-      - name: push static tarball to github
-        run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          tarball="kata-static-$tag-x86_64.tar.xz"
-          repo="https://github.com/kata-containers/runtime.git"
-          mv release-candidate/kata-static.tar.xz "release-candidate/${tarball}"
-          git clone "${repo}"
-          cd runtime
-          echo "uploading asset '${tarball}' to '${repo}' tag: ${tag}"
-          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "../release-candidate/${tarball}" "${tag}"

.github/workflows/release.yaml

@@ -149,3 +149,31 @@ jobs:
           tar -cvzf "${tarball}" src/agent/.cargo/config src/agent/vendor
           GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}" 
           popd
+
+  upload-libseccomp-tarball:
+    needs: upload-cargo-vendored-tarball
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: download-and-upload-tarball
+        env:
+          GITHUB_TOKEN: ${{ secrets.GIT_UPLOAD_TOKEN }}
+          GOPATH: ${HOME}/go
+        run: |
+          pushd $GITHUB_WORKSPACE
+          ./ci/install_yq.sh
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          versions_yaml="versions.yaml"
+          version=$(${GOPATH}/bin/yq read ${versions_yaml} "externals.libseccomp.version")
+          repo_url=$(${GOPATH}/bin/yq read ${versions_yaml} "externals.libseccomp.url")
+          download_url="${repo_url}/releases/download/v${version}"
+          tarball="libseccomp-${version}.tar.gz"
+          asc="${tarball}.asc"
+          curl -sSLO "${download_url}/${tarball}"
+          curl -sSLO "${download_url}/${asc}"
+          # "-m" option should be empty to re-use the existing release title
+          # without opening a text editor.
+          # For the details, check https://hub.github.com/hub-release.1.html.
+          hub release edit -m "" -a "${tarball}" "${tag}"
+          hub release edit -m "" -a "${asc}" "${tag}"
+          popd

.github/workflows/static-checks.yaml

@@ -13,7 +13,7 @@ jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.15.x, 1.16.x]
+        go-version: [1.16.x, 1.17.x]
         os: [ubuntu-20.04]
     runs-on: ${{ matrix.os }}
     env:
@@ -67,6 +67,14 @@ jobs:
         PATH=$PATH:"$HOME/.cargo/bin"
         rustup target add x86_64-unknown-linux-musl
         rustup component add rustfmt clippy
+    - name: Setup seccomp
+      run: |
+        libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
+        gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
+        echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
+        echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
+        echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
     # Check whether the vendored code is up-to-date & working as the first thing
     - name: Check vendored code
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}

VERSION

@@ -1 +1 @@
-2.3.0-alpha2
+2.3.1

ci/go-no-os-exit.sh

@@ -1,30 +0,0 @@
-#!/bin/bash
-# Copyright (c) 2018 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-# Check there are no os.Exit() calls creeping into the code
-# We don't use that exit path in the Kata codebase.
-
-# Allow the path to check to be over-ridden.
-# Default to the current directory.
-go_packages=${1:-.}
-
-echo "Checking for no os.Exit() calls for package [${go_packages}]"
-
-candidates=`go list -f '{{.Dir}}/*.go' $go_packages`
-for f in $candidates; do
-	filename=`basename $f`
-	# skip all go test files
-	[[ $filename == *_test.go ]] && continue
-	# skip exit.go where, the only file we should call os.Exit() from.
-	[[ $filename == "exit.go" ]] && continue
-	files="$f $files"
-done
-
-[ -z "$files" ] && echo "No files to check, skipping" && exit 0
-
-if egrep -n '\<os\.Exit\>' $files; then
-	echo "Direct calls to os.Exit() are forbidden, please use exit() so atexit() works"
-	exit 1
-fi

ci/install_libseccomp.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+#
+# Copyright 2021 Sony Group Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -o errexit
+
+cidir=$(dirname "$0")
+source "${cidir}/lib.sh"
+
+clone_tests_repo
+
+source "${tests_repo_dir}/.ci/lib.sh"
+
+# The following variables if set on the environment will change the behavior
+# of gperf and libseccomp configure scripts, that may lead this script to
+# fail. So let's ensure they are unset here.
+unset PREFIX DESTDIR
+
+arch=$(uname -m)
+workdir="$(mktemp -d --tmpdir build-libseccomp.XXXXX)"
+
+# Variables for libseccomp
+# Currently, specify the libseccomp version directly without using `versions.yaml`
+# because the current Snap workflow is incomplete.
+# After solving the issue, replace this code by using the `versions.yaml`.
+# libseccomp_version=$(get_version "externals.libseccomp.version")
+# libseccomp_url=$(get_version "externals.libseccomp.url")
+libseccomp_version="2.5.1"
+libseccomp_url="https://github.com/seccomp/libseccomp"
+libseccomp_tarball="libseccomp-${libseccomp_version}.tar.gz"
+libseccomp_tarball_url="${libseccomp_url}/releases/download/v${libseccomp_version}/${libseccomp_tarball}"
+cflags="-O2"
+
+# Variables for gperf
+# Currently, specify the gperf version directly without using `versions.yaml`
+# because the current Snap workflow is incomplete.
+# After solving the issue, replace this code by using the `versions.yaml`.
+# gperf_version=$(get_version "externals.gperf.version")
+# gperf_url=$(get_version "externals.gperf.url")
+gperf_version="3.1"
+gperf_url="https://ftp.gnu.org/gnu/gperf"
+gperf_tarball="gperf-${gperf_version}.tar.gz"
+gperf_tarball_url="${gperf_url}/${gperf_tarball}"
+
+# We need to build the libseccomp library from sources to create a static library for the musl libc.
+# However, ppc64le and s390x have no musl targets in Rust. Hence, we do not set cflags for the musl libc.
+if ([ "${arch}" != "ppc64le" ] && [ "${arch}" != "s390x" ]); then
+    # Set FORTIFY_SOURCE=1 because the musl-libc does not have some functions about FORTIFY_SOURCE=2
+    cflags="-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=1 -O2"
+fi
+
+die() {
+    msg="$*"
+    echo "[Error] ${msg}" >&2
+    exit 1
+}
+
+finish() {
+    rm -rf "${workdir}"
+}
+
+trap finish EXIT
+
+build_and_install_gperf() {
+    echo "Build and install gperf version ${gperf_version}"
+    mkdir -p "${gperf_install_dir}"
+    curl -sLO "${gperf_tarball_url}"
+    tar -xf "${gperf_tarball}"
+    pushd "gperf-${gperf_version}"
+    ./configure --prefix="${gperf_install_dir}"
+    make
+    make install
+    export PATH=$PATH:"${gperf_install_dir}"/bin
+    popd
+    echo "Gperf installed successfully"
+}
+
+build_and_install_libseccomp() {
+    echo "Build and install libseccomp version ${libseccomp_version}"
+    mkdir -p "${libseccomp_install_dir}"
+    curl -sLO "${libseccomp_tarball_url}"
+    tar -xf "${libseccomp_tarball}"
+    pushd "libseccomp-${libseccomp_version}"
+    ./configure --prefix="${libseccomp_install_dir}" CFLAGS="${cflags}" --enable-static
+    make
+    make install
+    popd
+    echo "Libseccomp installed successfully"
+}
+
+main() {
+    local libseccomp_install_dir="${1:-}"
+    local gperf_install_dir="${2:-}"
+
+    if [ -z "${libseccomp_install_dir}" ] || [ -z "${gperf_install_dir}" ]; then
+        die "Usage: ${0} <libseccomp-install-dir> <gperf-install-dir>"
+    fi
+
+    pushd "$workdir"
+    # gperf is required for building the libseccomp.
+    build_and_install_gperf
+    build_and_install_libseccomp
+    popd
+}
+
+main "$@"

ci/install_rust.sh

@@ -12,5 +12,5 @@ source "${cidir}/lib.sh"
 clone_tests_repo
 
 pushd ${tests_repo_dir}
-.ci/install_rust.sh
+.ci/install_rust.sh ${1:-}
 popd

ci/openshift-ci/images/Dockerfile.buildroot

@@ -6,4 +6,9 @@
 #
 FROM registry.centos.org/centos:8
 
-RUN yum -y update && yum -y install git sudo wget
+RUN yum -y update && \
+    yum -y install \
+    git \
+    sudo \
+    wget && \
+    yum clean all

docs/Developer-Guide.md

@@ -86,6 +86,16 @@ One of the `initrd` and `image` options in Kata runtime config file **MUST** be
 The main difference between the options is that the size of `initrd`(10MB+) is significantly smaller than
 rootfs `image`(100MB+).
 
+## Enable seccomp
+
+Enable seccomp as follows:
+
+```
+$ sudo sed -i '/^disable_guest_seccomp/ s/true/false/' /etc/kata-containers/configuration.toml
+```
+
+This will pass container seccomp profiles to the kata agent.
+
 ## Enable full debug
 
 Enable full debug as follows:
@@ -216,6 +226,18 @@ $ go get -d -u github.com/kata-containers/kata-containers
 $ cd $GOPATH/src/github.com/kata-containers/kata-containers/src/agent && make
 ```
 
+The agent is built with seccomp capability by default.
+If you want to build the agent without the seccomp capability, you need to run `make` with `SECCOMP=no` as follows.
+
+```
+$ make -C $GOPATH/src/github.com/kata-containers/kata-containers/src/agent SECCOMP=no
+```
+
+> **Note:**
+>
+> - If you enable seccomp in the main configuration file but build the agent without seccomp capability,
+>   the runtime exits conservatively with an error message.
+
 ## Get the osbuilder
 
 ```
@@ -234,9 +256,21 @@ the following example.
 $ export ROOTFS_DIR=${GOPATH}/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder/rootfs
 $ sudo rm -rf ${ROOTFS_DIR}
 $ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder
-$ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
+$ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true ./rootfs.sh ${distro}'
+```
+
+You MUST choose a distribution (e.g., `ubuntu`) for `${distro}`.
+You can get a supported distributions list in the Kata Containers by running the following.
+
+```
+$ ./rootfs.sh -l
+```
+
+If you want to build the agent without seccomp capability, you need to run the `rootfs.sh` script with `SECCOMP=no` as follows.
+
+```
+$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
 ```
-You MUST choose one of `alpine`, `centos`, `clearlinux`, `debian`, `euleros`, `fedora`, `suse`, and `ubuntu` for `${distro}`. By default `seccomp` packages are not included in the rootfs image. Set `SECCOMP` to `yes` to include them.
 
 > **Note:**
 >
@@ -272,6 +306,7 @@ $ script -fec 'sudo -E USE_DOCKER=true ./image_builder.sh ${ROOTFS_DIR}'
 > - If you do *not* wish to build under Docker, remove the `USE_DOCKER`
 >   variable in the previous command and ensure the `qemu-img` command is
 >   available on your system.
+>   - If `qemu-img` is not installed, you will likely see errors such as `ERROR: File /dev/loop19p1 is not a block device` and `losetup: /tmp/tmp.bHz11oY851: Warning: file is smaller than 512 bytes; the loop device may be useless or invisible for system tools`. These can be mitigated by installing the `qemu-img` command (available in the `qemu-img` package on Fedora or the `qemu-utils` package on Debian).
 
 
 ### Install the rootfs image
@@ -290,12 +325,23 @@ $ (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
 $ export ROOTFS_DIR="${GOPATH}/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder/rootfs"
 $ sudo rm -rf ${ROOTFS_DIR}
 $ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder
-$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
+$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true ./rootfs.sh ${distro}'
 ```
 `AGENT_INIT` controls if the guest image uses the Kata agent as the guest `init` process. When you create an initrd image,
-always set `AGENT_INIT` to `yes`. By default `seccomp` packages are not included in the initrd image. Set `SECCOMP` to `yes` to include them.
+always set `AGENT_INIT` to `yes`.
 
-You MUST choose one of `alpine`, `centos`, `clearlinux`, `euleros`, and `fedora` for `${distro}`.
+You MUST choose a distribution (e.g., `ubuntu`) for `${distro}`.
+You can get a supported distributions list in the Kata Containers by running the following.
+
+```
+$ ./rootfs.sh -l
+```
+
+If you want to build the agent without seccomp capability, you need to run the `rootfs.sh` script with `SECCOMP=no` as follows.
+
+```
+$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
+```
 
 > **Note:**
 >

docs/README.md

@@ -11,6 +11,10 @@ For details of the other Kata Containers repositories, see the
 
 * [Installation guides](./install/README.md): Install and run Kata Containers with Docker or Kubernetes
 
+## Tracing
+
+See the [tracing documentation](tracing.md).
+
 ## More User Guides
 
 * [Upgrading](Upgrading.md): how to upgrade from [Clear Containers](https://github.com/clearcontainers) and [runV](https://github.com/hyperhq/runv) to [Kata Containers](https://github.com/kata-containers) and how to upgrade an existing Kata Containers system to the latest version.

docs/Release-Process.md

@@ -64,7 +64,7 @@
 
 ### Check Git-hub Actions
 
-  We make use of [GitHub actions](https://github.com/features/actions) in this [file](https://github.com/kata-containers/kata-containers/blob/main/.github/workflows/main.yaml) in the `kata-containers/kata-containers` repository to build and upload release artifacts. This action is auto triggered with the above step when a new tag is pushed to the `kata-containers/kata-containers` repository.
+  We make use of [GitHub actions](https://github.com/features/actions) in this [file](https://github.com/kata-containers/kata-containers/blob/main/.github/workflows/release.yaml) in the `kata-containers/kata-containers` repository to build and upload release artifacts. This action is auto triggered with the above step when a new tag is pushed to the `kata-containers/kata-containers` repository.
 
   Check the [actions status page](https://github.com/kata-containers/kata-containers/actions) to verify all steps in the actions workflow have completed successfully. On success, a static tarball containing Kata release artifacts will be uploaded to the [Release page](https://github.com/kata-containers/kata-containers/releases).
 

docs/design/host-cgroups.md

@@ -242,8 +242,8 @@ On the other hand, running all non vCPU threads under a dedicated overhead cgrou
 accurate metrics on the actual Kata Container pod overhead, allowing for tuning the overhead
 cgroup size and constraints accordingly.
 
-[linux-config]: https://github.com/opencontainers/runtime-spec/blob/master/config-linux.md
-[cgroupspath]: https://github.com/opencontainers/runtime-spec/blob/master/config-linux.md#cgroups-path
+[linux-config]: https://github.com/opencontainers/runtime-spec/blob/main/config-linux.md
+[cgroupspath]: https://github.com/opencontainers/runtime-spec/blob/main/config-linux.md#cgroups-path
 
 # Supported cgroups
 

docs/design/kata-2-0-metrics.md

@@ -207,7 +207,7 @@ Metrics for Firecracker vmm.
 | `kata_firecracker_uart`: <br> Metrics specific to the UART device. | `GAUGE` |  | <ul><li>`item`<ul><li>`error_count`</li><li>`flush_count`</li><li>`missed_read_count`</li><li>`missed_write_count`</li><li>`read_count`</li><li>`write_count`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
 | `kata_firecracker_vcpu`: <br> Metrics specific to VCPUs' mode of functioning. | `GAUGE` |  | <ul><li>`item`<ul><li>`exit_io_in`</li><li>`exit_io_out`</li><li>`exit_mmio_read`</li><li>`exit_mmio_write`</li><li>`failures`</li><li>`filter_cpuid`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
 | `kata_firecracker_vmm`: <br> Metrics specific to the machine manager as a whole. | `GAUGE` |  | <ul><li>`item`<ul><li>`device_events`</li><li>`panic_count`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
-| `kata_firecracker_vsock`: <br> Vsock-related metrics. | `GAUGE` |  | <ul><li>`item`<ul><li>`activate_fails`</li><li>`cfg_fails`</li><li>`conn_event_fails`</li><li>`conns_added`</li><li>`conns_killed`</li><li>`conns_removed`</li><li>`ev_queue_event_fails`</li><li>`killq_resync`</li><li>`muxer_event_fails`</li><li>`rx_bytes_count`</li><li>`rx_packets_count`</li><li>`rx_queue_event_count`</li><li>`rx_queue_event_fails`</li><li>`rx_read_fails`</li><li>`tx_bytes_count`</li><li>`tx_flush_fails`</li><li>`tx_packets_count`</li><li>`tx_queue_event_count`</li><li>`tx_queue_event_fails`</li><li>`tx_write_fails`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
+| `kata_firecracker_vsock`: <br> VSOCK-related metrics. | `GAUGE` |  | <ul><li>`item`<ul><li>`activate_fails`</li><li>`cfg_fails`</li><li>`conn_event_fails`</li><li>`conns_added`</li><li>`conns_killed`</li><li>`conns_removed`</li><li>`ev_queue_event_fails`</li><li>`killq_resync`</li><li>`muxer_event_fails`</li><li>`rx_bytes_count`</li><li>`rx_packets_count`</li><li>`rx_queue_event_count`</li><li>`rx_queue_event_fails`</li><li>`rx_read_fails`</li><li>`tx_bytes_count`</li><li>`tx_flush_fails`</li><li>`tx_packets_count`</li><li>`tx_queue_event_count`</li><li>`tx_queue_event_fails`</li><li>`tx_write_fails`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
 
 ### Kata guest OS metrics
 

docs/design/kata-design-requirements.md

@@ -30,7 +30,7 @@ The Kata Containers runtime **MUST** implement the following command line option
 The Kata Containers project **MUST** provide two interfaces for CRI shims to manage hardware
 virtualization based Kubernetes pods and containers:
 -  An OCI and `runc` compatible command line interface, as described in the previous section.
-This interface is used by implementations such as [`CRI-O`](http://cri-o.io) and [`cri-containerd`](https://github.com/containerd/cri-containerd), for example.
+This interface is used by implementations such as [`CRI-O`](http://cri-o.io) and [`containerd`](https://github.com/containerd/containerd), for example.
 - A hardware virtualization runtime library API for CRI shims to consume and provide a more
 CRI native implementation. The [`frakti`](https://github.com/kubernetes/frakti) CRI shim is an example of such a consumer.
 

docs/how-to/README.md

@@ -5,7 +5,7 @@
 - [Run Kata containers with `crictl`](run-kata-with-crictl.md)
 - [Run Kata Containers with Kubernetes](run-kata-with-k8s.md)
 - [How to use Kata Containers and Containerd](containerd-kata.md)
-- [How to use Kata Containers and CRI (containerd plugin) with Kubernetes](how-to-use-k8s-with-cri-containerd-and-kata.md)
+- [How to use Kata Containers and CRI (containerd) with Kubernetes](how-to-use-k8s-with-cri-containerd-and-kata.md)
 - [Kata Containers and service mesh for Kubernetes](service-mesh.md)
 - [How to import Kata Containers logs into Fluentd](how-to-import-kata-logs-with-fluentd.md)
 

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -34,8 +34,6 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.agent.enable_tracing` | `boolean` | enable tracing for the agent |
 | `io.katacontainers.config.agent.container_pipe_size` | uint32 | specify the size of the std(in/out) pipes created for containers |
 | `io.katacontainers.config.agent.kernel_modules` | string | the list of kernel modules and their parameters that will be loaded in the guest kernel. Semicolon separated list of kernel modules and their parameters. These modules will be loaded in the guest kernel using `modprobe`(8). E.g., `e1000e InterruptThrottleRate=3000,3000,3000 EEE=1; i915 enable_ppgtt=0` |
-| `io.katacontainers.config.agent.trace_mode` | string | the trace mode for the agent |
-| `io.katacontainers.config.agent.trace_type` | string | the trace type for the agent |
 
 ## Hypervisor Options
 | Key | Value Type | Comments |

docs/how-to/how-to-use-k8s-with-cri-containerd-and-kata.md

@@ -3,7 +3,7 @@
 This document describes how to set up a single-machine Kubernetes (k8s) cluster.
 
 The Kubernetes cluster will use the
-[CRI containerd plugin](https://github.com/containerd/containerd/tree/main/pkg/cri) and
+[CRI containerd](https://github.com/containerd/containerd/) and
 [Kata Containers](https://katacontainers.io) to launch untrusted workloads.
 
 ## Requirements
@@ -71,12 +71,12 @@ $ for service in ${services}; do
     service_dir="/etc/systemd/system/${service}.service.d/"
     sudo mkdir -p ${service_dir}
 
-    cat << EOT | sudo tee "${service_dir}/proxy.conf"
+    cat << EOF | sudo tee "${service_dir}/proxy.conf"
 [Service]
 Environment="HTTP_PROXY=${http_proxy}"
 Environment="HTTPS_PROXY=${https_proxy}"
 Environment="NO_PROXY=${no_proxy}"
-EOT
+EOF
 done
 
 $ sudo systemctl daemon-reload
@@ -172,7 +172,7 @@ If a pod has the `runtimeClassName` set to `kata`, the CRI plugin runs the pod w
 - Create an pod configuration that using Kata Containers runtime
 
   ```bash
-  $ cat << EOT | tee nginx-kata.yaml
+  $ cat << EOF | tee nginx-kata.yaml
   apiVersion: v1
   kind: Pod
   metadata:
@@ -183,7 +183,7 @@ If a pod has the `runtimeClassName` set to `kata`, the CRI plugin runs the pod w
     - name: nginx
       image: nginx
       
-  EOT
+  EOF
   ```
 
 - Create the pod

docs/how-to/how-to-use-kata-containers-with-acrn.md

@@ -22,7 +22,7 @@ This document requires the presence of the ACRN hypervisor and Kata Containers o
 
 - ACRN supported [Hardware](https://projectacrn.github.io/latest/hardware.html#supported-hardware).
   > **Note:** Please make sure to have a minimum of 4 logical processors (HT) or cores.
-- ACRN [software](https://projectacrn.github.io/latest/tutorials/kbl-nuc-sdc.html#use-the-script-to-set-up-acrn-automatically) setup.
+- ACRN [software](https://projectacrn.github.io/latest/tutorials/run_kata_containers.html) setup.
 - For networking, ACRN supports either MACVTAP or TAP. If MACVTAP is not enabled in the Service OS, please follow the below steps to update the kernel:
 
   ```sh

docs/how-to/privileged.md

@@ -16,9 +16,9 @@ from the host, a potentially undesirable side-effect that decreases the security
 
 The following sections document how to configure this behavior in different container runtimes.
 
-#### Containerd and CRI
+#### Containerd
 
-The Containerd CRI allows configuring the privileged host devices behavior for each runtime in the CRI config. This is
+The Containerd allows configuring the privileged host devices behavior for each runtime in the containerd config. This is
 done with the `privileged_without_host_devices` option. Setting this to `true` will disable hot plugging of the host 
 devices into the guest, even when privileged is enabled.
 
@@ -41,7 +41,7 @@ See below example config:
 ```
 
  - [Kata Containers with Containerd and CRI documentation](how-to-use-k8s-with-cri-containerd-and-kata.md)
- - [Containerd CRI config documentation](https://github.com/containerd/cri/blob/master/docs/config.md)
+ - [Containerd CRI config documentation](https://github.com/containerd/containerd/blob/main/docs/cri/config.md)
 
 #### CRI-O
 

docs/how-to/run-kata-with-k8s.md

@@ -9,7 +9,7 @@ Kubernetes CRI (Container Runtime Interface) implementations allow using any
 OCI-compatible runtime with Kubernetes, such as the Kata Containers runtime.
 
 Kata Containers support both the [CRI-O](https://github.com/kubernetes-incubator/cri-o) and
-[CRI-containerd](https://github.com/containerd/cri) CRI implementations.
+[containerd](https://github.com/containerd/containerd) CRI implementations.
 
 After choosing one CRI implementation, you must make the appropriate configuration
 to ensure it integrates with Kata Containers.
@@ -20,7 +20,7 @@ required to spawn pods and containers, and this is the preferred way to run Kata
 An equivalent shim implementation for CRI-O is planned.
 
 ### CRI-O
-For CRI-O installation instructions, refer to the [CRI-O Tutorial](https://github.com/kubernetes-incubator/cri-o/blob/master/tutorial.md) page.
+For CRI-O installation instructions, refer to the [CRI-O Tutorial](https://github.com/cri-o/cri-o/blob/main/tutorial.md) page.
 
 The following sections show how to set up the CRI-O configuration file (default path: `/etc/crio/crio.conf`) for Kata.
 
@@ -30,7 +30,7 @@ Unless otherwise stated, all the following settings are specific to the `crio.ru
 # runtime used and options for how to set up and manage the OCI runtime.
 [crio.runtime]
 ```
-A comprehensive documentation of the configuration file can be found [here](https://github.com/cri-o/cri-o/blob/master/docs/crio.conf.5.md).
+A comprehensive documentation of the configuration file can be found [here](https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md).
 
 > **Note**: After any change to this file, the CRI-O daemon have to be restarted with:
 >````
@@ -111,11 +111,7 @@ manage_ns_lifecycle = true
 ```
 
 
-### containerd with CRI plugin
-
-If you select containerd with `cri` plugin, follow the "Getting Started for Developers"
-instructions [here](https://github.com/containerd/cri#getting-started-for-developers)
-to properly install it.
+### containerd
 
 To customize containerd to select Kata Containers runtime, follow our
 "Configure containerd to use Kata Containers" internal documentation
@@ -160,7 +156,7 @@ $ sudo systemctl restart kubelet
 # If using CRI-O
 $ sudo kubeadm init --ignore-preflight-errors=all --cri-socket /var/run/crio/crio.sock --pod-network-cidr=10.244.0.0/16
 
-# If using CRI-containerd
+# If using containerd
 $ sudo kubeadm init --ignore-preflight-errors=all --cri-socket /run/containerd/containerd.sock --pod-network-cidr=10.244.0.0/16
 
 $ export KUBECONFIG=/etc/kubernetes/admin.conf

docs/how-to/service-mesh.md

@@ -34,7 +34,7 @@ as the proxy starts.
 
 Follow the [instructions](../install/README.md)
 to get Kata Containers properly installed and configured with Kubernetes.
-You can choose between CRI-O and CRI-containerd, both are supported
+You can choose between CRI-O and containerd, both are supported
 through this document.
 
 For both cases, select the workloads as _trusted_ by default. This way,
@@ -159,7 +159,7 @@ containers with `privileged: true` to `privileged: false`.
 There is no difference between Istio and Linkerd in this section. It is
 about which CRI implementation you use.
 
-For both CRI-O and CRI-containerd, you have to add an annotation indicating
+For both CRI-O and containerd, you have to add an annotation indicating
 the workload for this deployment is not _trusted_, which will trigger
 `kata-runtime` to be called instead of `runc`.
 
@@ -193,9 +193,9 @@ spec:
 ...
 ```
 
-__CRI-containerd:__
+__containerd:__
 
-Add the following annotation for CRI-containerd
+Add the following annotation for containerd
 ```yaml
 io.kubernetes.cri.untrusted-workload: "true"
 ```

docs/tracing.md

@@ -0,0 +1,213 @@
+# Overview
+
+This document explains how to trace Kata Containers components.
+
+# Introduction
+
+The Kata Containers runtime and agent are able to generate
+[OpenTelemetry][opentelemetry] trace spans, which allow the administrator to
+observe what those components are doing and how much time they are spending on
+each operation.
+
+# OpenTelemetry summary
+
+An OpenTelemetry-enabled application creates a number of trace "spans". A span
+contains the following attributes:
+
+- A name
+- A pair of timestamps (recording the start time and end time of some operation)
+- A reference to the span's parent span
+
+All spans need to be *finished*, or *completed*, to allow the OpenTelemetry
+framework to generate the final trace information (by effectively closing the
+transaction encompassing the initial (root) span and all its children).
+
+For Kata, the root span represents the total amount of time taken to run a
+particular component from startup to its shutdown (the "run time").
+
+# Architecture
+
+## Runtime tracing architecture
+
+The runtime, which runs in the host environment, has been modified to
+optionally generate trace spans which are sent to a trace collector on the
+host.
+
+## Agent tracing architecture
+
+An OpenTelemetry system (such as [Jaeger][jaeger-tracing]) uses a collector to
+gather up trace spans from the application for viewing and processing. For an
+application to use the collector, it must run in the same context as
+the collector.
+
+This poses a problem for tracing the Kata Containers agent since it does not
+run in the same context as the collector: it runs inside a virtual machine (VM).
+
+To allow spans from the agent to be sent to the trace collector, Kata provides
+a [trace forwarder][trace-forwarder] component. This runs in the same context
+as the collector (generally on the host system) and listens on a
+[`VSOCK`][vsock] channel for traces generated by the agent, forwarding them on
+to the trace collector.
+
+> **Note:**
+>
+> This design supports agent tracing without having to make changes to the
+> image, but also means that [custom images][osbuilder] can also benefit from
+> agent tracing.
+
+The following diagram summarises the architecture used to trace the Kata
+Containers agent:
+
+```
++--------------------------------------------+
+| Host                                       |
+|                                            |
+| +---------------+                          |
+| | OpenTelemetry |                          |
+| | Trace         |                          |
+| | Collector     |                          |
+| +---------------+                          |
+|       ^                  +---------------+ |
+|       | spans            | Kata VM       | |
+| +-----+-----+            |               | |
+| | Kata      |    spans   o     +-------+ | |
+| | Trace     |<-----------------| Kata  | | |
+| | Forwarder |    VSOCK   o     | Agent | | |
+| +-----------+    Channel |     +-------+ | |
+|                          +---------------+ |
++--------------------------------------------+
+```
+
+# Agent tracing prerequisites
+
+- You must have a trace collector running.
+
+  Although the collector normally runs on the host, it can also be run from
+  inside a Docker image configured to expose the appropriate host ports to the
+  collector.
+
+  The [Jaeger "all-in-one" Docker image][jaeger-all-in-one] method
+  is the quickest and simplest way to run the collector for testing.
+
+- If you wish to trace the agent, you must start the
+  [trace forwarder][trace-forwarder].
+
+> **Notes:**
+>
+> - If agent tracing is enabled but the forwarder is not running,
+>   the agent will log an error (signalling that it cannot generate trace
+>   spans), but continue to work as normal.
+>
+> - The trace forwarder requires a trace collector (such as Jaeger) to be
+>   running before it is started. If a collector is not running, the trace
+>   forwarder will exit with an error.
+
+# Enable tracing
+
+By default, tracing is disabled for all components. To enable _any_ form of
+tracing an `enable_tracing` option must be enabled for at least one component.
+
+> **Note:** 
+>
+> Enabling this option will only allow tracing for subsequently
+> started containers.
+
+## Enable runtime tracing
+
+To enable runtime tracing, set the tracing option as shown:
+
+```toml
+[runtime]
+enable_tracing = true
+```
+
+## Enable agent tracing
+
+To enable agent tracing, set the tracing option as shown:
+
+```toml
+[agent.kata]
+enable_tracing = true
+```
+
+> **Note:**
+>
+> If both agent tracing and runtime tracing are enabled, the resulting trace
+> spans will be "collated": expanding individual runtime spans in the Jaeger
+> web UI will show the agent trace spans resulting from the runtime
+> operation.
+
+# Appendices
+
+## Agent tracing requirements
+
+### Host environment
+
+- The host kernel must support the VSOCK socket type.
+
+  This will be available if the kernel is built with the
+  `CONFIG_VHOST_VSOCK` configuration option.
+
+- The VSOCK kernel module must be loaded:
+
+   ```
+   $ sudo modprobe vhost_vsock
+   ```
+
+### Guest environment
+
+- The guest kernel must support the VSOCK socket type:
+
+  This will be available if the kernel is built with the
+  `CONFIG_VIRTIO_VSOCKETS` configuration option.
+
+  > **Note:** The default Kata Containers guest kernel provides this feature.
+
+## Agent tracing limitations
+
+- Agent tracing is only "completed" when the workload and the Kata agent
+  process have exited.
+
+  Although trace information *can* be inspected before the workload and agent
+  have exited, it is incomplete. This is shown as `<trace-without-root-span>`
+  in the Jaeger web UI.
+
+  If the workload is still running, the trace transaction -- which spans the entire
+  runtime of the Kata agent -- will not have been completed. To view the complete
+  trace details, wait for the workload to end, or stop the container.
+
+## Performance impact
+
+[OpenTelemetry][opentelemetry] is designed for high performance. It combines
+the best of two previous generation projects (OpenTracing and OpenCensus) and
+uses a very efficient mechanism to capture trace spans. Further, the trace
+points inserted into the agent are generated dynamically at compile time. This
+is advantageous since new versions of the agent will automatically benefit
+from improvements in the tracing infrastructure. Overall, the impact of
+enabling runtime and agent tracing should be extremely low.
+
+## Agent shutdown behaviour
+ 
+In normal operation, the Kata runtime manages the VM shutdown and performs
+certain optimisations to speed up this process. However, if agent tracing is
+enabled, the agent itself is responsible for shutting down the VM. This it to
+ensure all agent trace transactions are completed. This means there will be a
+small performance impact for container shutdown when agent tracing is enabled
+as the runtime must wait for the VM to shutdown fully.
+
+## Set up a tracing development environment
+
+If you want to debug, further develop, or test tracing,
+[enabling full debug][enable-full-debug]
+is highly recommended. For working with the agent, you may also wish to
+[enable a debug console][setup-debug-console]
+to allow you to access the VM environment.
+
+[enable-full-debug]: https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md#enable-full-debug
+[jaeger-all-in-one]: https://www.jaegertracing.io/docs/getting-started/
+[jaeger-tracing]: https://www.jaegertracing.io
+[opentelemetry]: https://opentelemetry.io
+[osbuilder]: https://github.com/kata-containers/kata-containers/blob/main/tools/osbuilder
+[setup-debug-console]: https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md#set-up-a-debug-console
+[trace-forwarder]: /src/trace-forwarder
+[vsock]: https://wiki.qemu.org/Features/VirtioVsock

docs/use-cases/using-Intel-SGX-and-kata.md

@@ -1,107 +1,113 @@
 # Kata Containers with SGX
 
-Intel® Software Guard Extensions (SGX) is a set of instructions that increases the security
+Intel Software Guard Extensions (SGX) is a set of instructions that increases the security
 of applications code and data, giving them more protections from disclosure or modification.
 
-> **Note:** At the time of writing this document, SGX patches have not landed on the Linux kernel
-> project, so specific versions for guest and host kernels must be installed to enable SGX.
+This document guides you to run containers with SGX enclaves with Kata Containers in Kubernetes.
 
-## Check if SGX is enabled
+## Preconditions
 
-Run the following command to check if your host supports SGX.
+* Intel SGX capable bare metal nodes
+* Host kernel Linux 5.13 or later with SGX and SGX KVM enabled:
 
 ```sh
-$ grep -o sgx /proc/cpuinfo
+$ grep SGX /boot/config-`uname -r`
+CONFIG_X86_SGX=y
+CONFIG_X86_SGX_KVM=y
 ```
 
-Continue to the following section if the output of the above command is empty,
-otherwise continue to section [Install Guest kernel with SGX support](#install-guest-kernel-with-sgx-support)
+* Kubernetes cluster configured with:
+   * [`kata-deploy`](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy) based Kata Containers installation
+   * [Intel SGX Kubernetes device plugin](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/main/cmd/sgx_plugin#deploying-with-pre-built-images)
 
-## Install Host kernel with SGX support
+> Note: Kata Containers supports creating VM sandboxes with Intel® SGX enabled
+> using [cloud-hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor/) VMM only. QEMU support is waiting to get the
+> Intel SGX enabled QEMU upstream release.
 
-The following commands were tested on Fedora 32, they might work on other distros too.
+## Installation
+
+### Kata Containers Guest Kernel
+
+Follow the instructions to [setup](../../tools/packaging/kernel/README.md#setup-kernel-source-code) and [build](../../tools/packaging/kernel/README.md#build-the-kernel) the experimental guest kernel. Then, install as:
 
 ```sh
-$ git clone --depth=1 https://github.com/intel/kvm-sgx
-$ pushd kvm-sgx
-$ cp /boot/config-$(uname -r) .config
-$ yes "" | make oldconfig
-$ # In the following step, enable: INTEL_SGX and INTEL_SGX_VIRTUALIZATION
-$ make menuconfig
-$ make -j$(($(nproc)-1)) bzImage
-$ make -j$(($(nproc)-1)) modules
-$ sudo make modules_install
-$ sudo make install
-$ popd
-$ sudo reboot
+$ sudo cp kata-linux-experimental-*/vmlinux /opt/kata/share/kata-containers/vmlinux.sgx
+$ sudo sed -i 's|vmlinux.container|vmlinux.sgx|g' \
+  /opt/kata/share/defaults/kata-containers/configuration-clh.toml
 ```
 
-> **Notes:**
-> * Run: `mokutil --sb-state` to check whether secure boot is enabled, if so, you will need to sign the kernel.
-> * You'll lose SGX support when a new distro kernel is installed and the system rebooted.
-
-Once you have restarted your system with the new brand Linux Kernel with SGX support, run
-the following command to make sure it's enabled. If the output is empty, go to the BIOS
-setup and enable SGX manually.
-
-```sh
-$ grep -o sgx /proc/cpuinfo
-```
-
-## Install Guest kernel with SGX support
-
-Install the guest kernel in the Kata Containers directory, this way it can be used to run
-Kata Containers.
-
-```sh
-$ curl -LOk https://github.com/devimc/kvm-sgx/releases/download/v0.0.1/kata-virtiofs-sgx.tar.gz
-$ sudo tar -xf kata-virtiofs-sgx.tar.gz -C /usr/share/kata-containers/
-$ sudo sed -i 's|kernel =|kernel = "/usr/share/kata-containers/vmlinux-virtiofs-sgx.container"|g' \
-  /usr/share/defaults/kata-containers/configuration.toml
-```
-
-## Run Kata Containers with SGX enabled
+### Kata Containers Configuration
 
 Before running a Kata Container make sure that your version of `crio` or `containerd`
 supports annotations.
+
 For `containerd` check in `/etc/containerd/config.toml` that the list of `pod_annotations` passed
 to the `sandbox` are: `["io.katacontainers.*", "sgx.intel.com/epc"]`.
 
-> `sgx.yaml`
+## Usage
+
+With the following sample job deployed using `kubectl apply -f`:
+
 ```yaml
-apiVersion: v1
-kind: Pod
+apiVersion: batch/v1
+kind: Job
 metadata:
-  name: sgx
-  annotations:
-    sgx.intel.com/epc: "32Mi"
+  name: oesgx-demo-job
+  labels:
+    jobgroup: oesgx-demo
 spec:
-  terminationGracePeriodSeconds: 0
-  runtimeClassName: kata
-  containers:
-  - name: c1
-    image: busybox
-    command:
-        - sh
-    stdin: true
-    tty: true
-    volumeMounts:
-    - mountPath: /dev/sgx/
-      name: test-volume
-  volumes:
-  - name: test-volume
-    hostPath:
-      path: /dev/sgx/
-      type: Directory
+  template:
+    metadata:
+      labels:
+        jobgroup: oesgx-demo
+    spec:
+      runtimeClassName: kata-clh
+      initContainers:
+        - name: init-sgx
+          image: busybox
+          command: ['sh', '-c', 'mkdir /dev/sgx; ln -s /dev/sgx_enclave /dev/sgx/enclave; ln -s /dev/sgx_provision /dev/sgx/provision']
+          volumeMounts:
+          - mountPath: /dev
+            name: dev-mount
+      restartPolicy: Never
+      containers:
+        -
+          name: eosgx-demo-job-1
+          image: oeciteam/oe-helloworld:latest
+          imagePullPolicy: IfNotPresent
+          securityContext:
+            readOnlyRootFilesystem: true
+            capabilities:
+              add: ["IPC_LOCK"]
+          resources:
+            limits:
+              sgx.intel.com/epc: "512Ki"
+      volumes:
+        - name: dev-mount
+          hostPath:
+            path: /dev
 ```
 
+You'll see the enclave output:
+
 ```sh
-$ kubectl apply -f sgx.yaml
-$ kubectl exec -ti sgx ls /dev/sgx/
-enclave    provision
+$ kubectl logs oesgx-demo-job-wh42g
+Hello world from the enclave
+Enclave called into host to print: Hello World!
 ```
 
-The output of the latest command shouldn't be empty, otherwise check
-your system environment to make sure SGX is fully supported.
+### Notes
 
-[1]: github.com/cloud-hypervisor/cloud-hypervisor/
+* The Kata VM's SGX Encrypted Page Cache (EPC) memory size is based on the sum of `sgx.intel.com/epc`
+resource requests within the pod.
+* `init-sgx` can be removed from the YAML configuration file if the Kata rootfs is modified with the
+necessary udev rules.
+   See the [note on SGX backwards compatibility](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/main/cmd/sgx_plugin#backwards-compatibility-note).
+* Intel SGX DCAP attestation is known to work from Kata sandboxes but it comes with one limitation: If
+the Intel SGX `aesm` daemon runs on the bare metal node and DCAP `out-of-proc` attestation is used,
+containers within the Kata sandbox cannot get the access to the host's `/var/run/aesmd/aesm.sock`
+because socket passthrough is not supported. An alternative is to deploy the `aesm` daemon as a side-car
+container.
+* Projects like [Gramine Shielded Containers (GSC)](https://gramine-gsc.readthedocs.io/en/latest/) are
+also known to work. For GSC specifically, the Kata guest kernel needs to have the `CONFIG_NUMA=y`
+enabled and at least one CPU online when running the GSC container.

pkg/logging/Cargo.toml

@@ -7,15 +7,15 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-serde_json = "1.0.39"
+serde_json = "1.0.73"
 # slog:
 # - Dynamic keys required to allow HashMap keys to be slog::Serialized.
 # - The 'max_*' features allow changing the log level at runtime
 #   (by stopping the compiler from removing log calls).
-slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_info"] }
-slog-json = "2.3.0"
-slog-async = "2.3.0"
-slog-scope = "4.1.2"
+slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug"] }
+slog-json = "2.4.0"
+slog-async = "2.7.0"
+slog-scope = "4.4.0"
 
 [dev-dependencies]
-tempfile = "3.1.0"
+tempfile = "3.2.0"

snap/snapcraft.yaml

@@ -59,7 +59,7 @@ parts:
 
       yq_version=3.4.1
       yq_url="https://${yq_pkg}/releases/download/${yq_version}/yq_${goos}_${goarch}"
-      curl -o "${yq_path}" -LSsf "${yq_url}"
+      curl -o "${yq_path}" -L "${yq_url}"
       chmod +x "${yq_path}"
 
       kata_dir=gopath/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
@@ -139,7 +139,7 @@ parts:
       cp kata-containers*.img ${kata_image_dir}
 
   runtime:
-    after: [godeps, image]
+    after: [godeps, image, cloud-hypervisor]
     plugin: nil
     build-attributes: [no-patchelf]
     override-build: |
@@ -185,6 +185,7 @@ parts:
       - flex
     override-build: |
       yq=${SNAPCRAFT_STAGE}/yq
+      export PATH="${PATH}:${SNAPCRAFT_STAGE}"
       export GOPATH=${SNAPCRAFT_STAGE}/gopath
       kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
       versions_file="${kata_dir}/versions.yaml"
@@ -199,10 +200,17 @@ parts:
       kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
 
       cd ${kata_dir}/tools/packaging/kernel
+      kernel_dir_prefix="kata-linux-"
 
       # Setup and build kernel
-      ./build-kernel.sh -v ${kernel_version} -d setup
-      kernel_dir_prefix="kata-linux-"
+      if [ "$(uname -m)" = "x86_64" ]; then
+        kernel_version="$(${yq} r $versions_file assets.kernel-experimental.tag)"
+        kernel_version=${kernel_version#v}
+        kernel_dir_prefix="kata-linux-experimental-"
+        ./build-kernel.sh -e -v ${kernel_version} -d setup
+      else
+        ./build-kernel.sh -v ${kernel_version} -d setup
+      fi
       cd ${kernel_dir_prefix}*
       make -j $(($(nproc)-1)) EXTRAVERSION=".container"
 
@@ -327,6 +335,22 @@ parts:
       # Hack: move qemu to /
       "snap/kata-containers/current/": "./"
 
+  cloud-hypervisor:
+    plugin: nil
+    after: [godeps]
+    override-build: |
+      export GOPATH=${SNAPCRAFT_STAGE}/gopath
+      yq=${SNAPCRAFT_STAGE}/yq
+      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
+      versions_file="${kata_dir}/versions.yaml"
+      version="$(${yq} r ${versions_file} assets.hypervisor.cloud_hypervisor.version)"
+      url="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/${version}"
+      curl -L ${url}/cloud-hypervisor-static -o cloud-hypervisor
+      curl -LO ${url}/clh-remote
+
+      install -D cloud-hypervisor ${SNAPCRAFT_PART_INSTALL}/usr/bin/cloud-hypervisor
+      install -D clh-remote ${SNAPCRAFT_PART_INSTALL}/usr/bin/clh-remote
+
 apps:
   runtime:
     command: usr/bin/kata-runtime

src/agent/Cargo.toml

@@ -6,7 +6,6 @@ edition = "2018"
 
 [dependencies]
 oci = { path = "oci" }
-logging = { path = "../../pkg/logging" }
 rustjail = { path = "rustjail" }
 protocols = { path = "protocols" }
 lazy_static = "1.3.0"
@@ -20,11 +19,12 @@ scan_fmt = "0.2.3"
 scopeguard = "1.0.0"
 thiserror = "1.0.26"
 regex = "1"
+serial_test = "0.5.1"
 
 # Async helpers
 async-trait = "0.1.42"
 async-recursion = "0.3.2"
-futures = "0.3.12"
+futures = "0.3.17"
 
 # Async runtime
 tokio = { version = "1", features = ["full"] }
@@ -35,21 +35,20 @@ rtnetlink = "0.8.0"
 netlink-packet-utils = "0.4.1"
 ipnetwork = "0.17.0"
 
-# slog:
-# - Dynamic keys required to allow HashMap keys to be slog::Serialized.
-# - The 'max_*' features allow changing the log level at runtime
-#   (by stopping the compiler from removing log calls).
-slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_info"] }
+# Note: this crate sets the slog 'max_*' features which allows the log level
+# to be modified at runtime.
+logging = { path = "../../pkg/logging" }
+slog = "2.5.2"
 slog-scope = "4.1.2"
 
 # Redirect ttrpc log calls
 slog-stdlog = "4.0.0"
 log = "0.4.11"
 
-prometheus = { version = "0.9.0", features = ["process"] }
-procfs = "0.7.9"
+prometheus = { version = "0.13.0", features = ["process"] }
+procfs = "0.12.0"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.2.5" }
+cgroups = { package = "cgroups-rs", version = "0.2.8" }
 
 # Tracing
 tracing = "0.1.26"
@@ -74,3 +73,6 @@ members = [
 
 [profile.release]
 lto = true
+
+[features]
+seccomp = ["rustjail/seccomp"]

src/agent/Makefile

@@ -27,6 +27,20 @@ COMMIT_MSG = $(if $(COMMIT),$(COMMIT),unknown)
 # Exported to allow cargo to see it
 export VERSION_COMMIT := $(if $(COMMIT),$(VERSION)-$(COMMIT),$(VERSION))
 
+EXTRA_RUSTFEATURES :=
+
+##VAR SECCOMP=yes|no define if agent enables seccomp feature
+SECCOMP := yes
+
+# Enable seccomp feature of rust build
+ifeq ($(SECCOMP),yes)
+    override EXTRA_RUSTFEATURES += seccomp
+endif
+
+ifneq ($(EXTRA_RUSTFEATURES),)
+    override EXTRA_RUSTFEATURES := --features $(EXTRA_RUSTFEATURES)
+endif
+
 include ../../utils.mk
 
 TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET)
@@ -90,15 +104,14 @@ default: $(TARGET) show-header
 $(TARGET): $(GENERATED_CODE) $(TARGET_PATH)
 
 $(TARGET_PATH): $(SOURCES) | show-summary
-	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE)
+	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
 
 $(GENERATED_FILES): %: %.in
 	@sed $(foreach r,$(GENERATED_REPLACEMENTS),-e 's|@$r@|$($r)|g') "$<" > "$@"
 
 ##TARGET optimize: optimized  build
 optimize: $(SOURCES) | show-summary show-header
-	@RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny-warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE)
-
+	@RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny-warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
 
 ##TARGET clippy: run clippy linter
 clippy: $(GENERATED_CODE)
@@ -127,7 +140,7 @@ vendor:
 
 #TARGET test: run cargo tests
 test:
-	@cargo test --all --target $(TRIPLE) -- --nocapture
+	@cargo test --all --target $(TRIPLE) $(EXTRA_RUSTFEATURES) -- --nocapture
 
 ##TARGET check: run test
 check: clippy format

src/agent/README.md

@@ -19,6 +19,7 @@ After that, we drafted the initial code here, and any contributions are welcome.
 | I/O stream              | :white_check_mark: |
 | Cgroups                 | :white_check_mark: |
 | Capabilities, `rlimit`, readonly path, masked path, users | :white_check_mark: |
+| Seccomp                 | :white_check_mark: |
 | container stats (`stats_container`)                     | :white_check_mark: |
 | Hooks                   | :white_check_mark: |
 | **Agent Features & APIs** |

src/agent/oci/Cargo.toml

@@ -5,7 +5,7 @@ authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
 edition = "2018"
 
 [dependencies]
-serde = "1.0.91"
-serde_derive = "1.0.91"
-serde_json = "1.0.39"
-libc = "0.2.58"
+serde = "1.0.131"
+serde_derive = "1.0.131"
+serde_json = "1.0.73"
+libc = "0.2.112"

src/agent/rustjail/Cargo.toml

@@ -23,14 +23,18 @@ scan_fmt = "0.2"
 regex = "1.1"
 path-absolutize = "1.2.0"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.2.5" }
+cgroups = { package = "cgroups-rs", version = "0.2.8" }
 rlimit = "0.5.3"
 
 tokio = { version = "1.2.0", features = ["sync", "io-util", "process", "time", "macros"] }
 futures = "0.3"
 async-trait = "0.1.31"
 inotify = "0.9.2"
+libseccomp = { version = "0.1.3", optional = true }
 
 [dev-dependencies]
 serial_test = "0.5.0"
 tempfile = "3.1.0"
+
+[features]
+seccomp = ["libseccomp"]

src/agent/rustjail/src/container.rs

@@ -25,6 +25,8 @@ use crate::cgroups::mock::Manager as FsManager;
 use crate::cgroups::Manager;
 use crate::log_child;
 use crate::process::Process;
+#[cfg(feature = "seccomp")]
+use crate::seccomp;
 use crate::specconv::CreateOpts;
 use crate::{mount, validator};
 
@@ -151,7 +153,7 @@ lazy_static! {
             },
             LinuxDevice {
                 path: "/dev/full".to_string(),
-                r#type: String::from("c"),
+                r#type: "c".to_string(),
                 major: 1,
                 minor: 7,
                 file_mode: Some(0o666),
@@ -593,11 +595,22 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         })?;
     }
 
-    // NoNewPeiviledges, Drop capabilities
+    // NoNewPrivileges
     if oci_process.no_new_privileges {
         capctl::prctl::set_no_new_privs().map_err(|_| anyhow!("cannot set no new privileges"))?;
     }
 
+    // Without NoNewPrivileges, we need to set seccomp
+    // before dropping capabilities because the calling thread
+    // must have the CAP_SYS_ADMIN.
+    #[cfg(feature = "seccomp")]
+    if !oci_process.no_new_privileges {
+        if let Some(ref scmp) = linux.seccomp {
+            seccomp::init_seccomp(scmp)?;
+        }
+    }
+
+    // Drop capabilities
     if oci_process.capabilities.is_some() {
         let c = oci_process.capabilities.as_ref().unwrap();
         capabilities::drop_privileges(cfd_log, c)?;
@@ -641,7 +654,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     let exec_file = Path::new(&args[0]);
     log_child!(cfd_log, "process command: {:?}", &args);
     if !exec_file.exists() {
-        find_file(exec_file).ok_or_else(|| anyhow!("the file {} is not exist", &args[0]))?;
+        find_file(exec_file).ok_or_else(|| anyhow!("the file {} was not found", &args[0]))?;
     }
 
     // notify parent that the child's ready to start
@@ -669,6 +682,16 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         unistd::read(fd, &mut buf)?;
     }
 
+    // With NoNewPrivileges, we should set seccomp as close to
+    // do_exec as possible in order to reduce the amount of
+    // system calls in the seccomp profiles.
+    #[cfg(feature = "seccomp")]
+    if oci_process.no_new_privileges {
+        if let Some(ref scmp) = linux.seccomp {
+            seccomp::init_seccomp(scmp)?;
+        }
+    }
+
     do_exec(&args);
 }
 

src/agent/rustjail/src/lib.rs

@@ -34,6 +34,8 @@ pub mod container;
 pub mod mount;
 pub mod pipestream;
 pub mod process;
+#[cfg(feature = "seccomp")]
+pub mod seccomp;
 pub mod specconv;
 pub mod sync;
 pub mod sync_with_async;

src/agent/rustjail/src/mount.rs

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use anyhow::{anyhow, bail, Context, Result};
+use anyhow::{anyhow, Context, Result};
 use libc::uid_t;
 use nix::errno::Errno;
 use nix::fcntl::{self, OFlag};
@@ -19,7 +19,7 @@ use std::fs::{self, OpenOptions};
 use std::mem::MaybeUninit;
 use std::os::unix;
 use std::os::unix::io::RawFd;
-use std::path::{Path, PathBuf};
+use std::path::{Component, Path, PathBuf};
 
 use path_absolutize::*;
 use std::fs::File;
@@ -745,7 +745,7 @@ fn mount_from(
         let _ = fs::create_dir_all(&dir).map_err(|e| {
             log_child!(
                 cfd_log,
-                "creat dir {}: {}",
+                "create dir {}: {}",
                 dir.to_str().unwrap(),
                 e.to_string()
             )
@@ -828,18 +828,35 @@ fn default_symlinks() -> Result<()> {
     }
     Ok(())
 }
+
+fn dev_rel_path(path: &str) -> Option<&Path> {
+    let path = Path::new(path);
+
+    if !path.starts_with("/dev")
+        || path == Path::new("/dev")
+        || path.components().any(|c| c == Component::ParentDir)
+    {
+        return None;
+    }
+    path.strip_prefix("/").ok()
+}
+
 fn create_devices(devices: &[LinuxDevice], bind: bool) -> Result<()> {
-    let op: fn(&LinuxDevice) -> Result<()> = if bind { bind_dev } else { mknod_dev };
+    let op: fn(&LinuxDevice, &Path) -> Result<()> = if bind { bind_dev } else { mknod_dev };
     let old = stat::umask(Mode::from_bits_truncate(0o000));
     for dev in DEFAULT_DEVICES.iter() {
-        op(dev)?;
+        let path = Path::new(&dev.path[1..]);
+        op(dev, path).context(format!("Creating container device {:?}", dev))?;
     }
     for dev in devices {
-        if !dev.path.starts_with("/dev") || dev.path.contains("..") {
+        let path = dev_rel_path(&dev.path).ok_or_else(|| {
             let msg = format!("{} is not a valid device path", dev.path);
-            bail!(anyhow!(msg));
+            anyhow!(msg)
+        })?;
+        if let Some(dir) = path.parent() {
+            fs::create_dir_all(dir).context(format!("Creating container device {:?}", dev))?;
         }
-        op(dev)?;
+        op(dev, path).context(format!("Creating container device {:?}", dev))?;
     }
     stat::umask(old);
     Ok(())
@@ -861,21 +878,21 @@ lazy_static! {
     };
 }
 
-fn mknod_dev(dev: &LinuxDevice) -> Result<()> {
+fn mknod_dev(dev: &LinuxDevice, relpath: &Path) -> Result<()> {
     let f = match LINUXDEVICETYPE.get(dev.r#type.as_str()) {
         Some(v) => v,
         None => return Err(anyhow!("invalid spec".to_string())),
     };
 
     stat::mknod(
-        &dev.path[1..],
+        relpath,
         *f,
         Mode::from_bits_truncate(dev.file_mode.unwrap_or(0)),
         nix::sys::stat::makedev(dev.major as u64, dev.minor as u64),
     )?;
 
     unistd::chown(
-        &dev.path[1..],
+        relpath,
         Some(Uid::from_raw(dev.uid.unwrap_or(0) as uid_t)),
         Some(Gid::from_raw(dev.gid.unwrap_or(0) as uid_t)),
     )?;
@@ -883,9 +900,9 @@ fn mknod_dev(dev: &LinuxDevice) -> Result<()> {
     Ok(())
 }
 
-fn bind_dev(dev: &LinuxDevice) -> Result<()> {
+fn bind_dev(dev: &LinuxDevice, relpath: &Path) -> Result<()> {
     let fd = fcntl::open(
-        &dev.path[1..],
+        relpath,
         OFlag::O_RDWR | OFlag::O_CREAT,
         Mode::from_bits_truncate(0o644),
     )?;
@@ -894,7 +911,7 @@ fn bind_dev(dev: &LinuxDevice) -> Result<()> {
 
     mount(
         Some(&*dev.path),
-        &dev.path[1..],
+        relpath,
         None::<&str>,
         MsFlags::MS_BIND,
         None::<&str>,
@@ -1258,11 +1275,12 @@ mod tests {
             uid: Some(unistd::getuid().as_raw()),
             gid: Some(unistd::getgid().as_raw()),
         };
+        let path = Path::new("fifo");
 
-        let ret = mknod_dev(&dev);
+        let ret = mknod_dev(&dev, path);
         assert!(ret.is_ok(), "Should pass. Got: {:?}", ret);
 
-        let ret = stat::stat("fifo");
+        let ret = stat::stat(path);
         assert!(ret.is_ok(), "Should pass. Got: {:?}", ret);
     }
     #[test]
@@ -1379,4 +1397,26 @@ mod tests {
             assert!(result == t.result, "{}", msg);
         }
     }
+
+    #[test]
+    fn test_dev_rel_path() {
+        // Valid device paths
+        assert_eq!(dev_rel_path("/dev/sda").unwrap(), Path::new("dev/sda"));
+        assert_eq!(dev_rel_path("//dev/sda").unwrap(), Path::new("dev/sda"));
+        assert_eq!(
+            dev_rel_path("/dev/vfio/99").unwrap(),
+            Path::new("dev/vfio/99")
+        );
+        assert_eq!(dev_rel_path("/dev/...").unwrap(), Path::new("dev/..."));
+        assert_eq!(dev_rel_path("/dev/a..b").unwrap(), Path::new("dev/a..b"));
+        assert_eq!(dev_rel_path("/dev//foo").unwrap(), Path::new("dev/foo"));
+
+        // Bad device paths
+        assert!(dev_rel_path("/devfoo").is_none());
+        assert!(dev_rel_path("/etc/passwd").is_none());
+        assert!(dev_rel_path("/dev/../etc/passwd").is_none());
+        assert!(dev_rel_path("dev/foo").is_none());
+        assert!(dev_rel_path("").is_none());
+        assert!(dev_rel_path("/dev").is_none());
+    }
 }

src/agent/rustjail/src/process.rs

@@ -24,6 +24,16 @@ use tokio::io::{split, ReadHalf, WriteHalf};
 use tokio::sync::Mutex;
 use tokio::sync::Notify;
 
+macro_rules! close_process_stream {
+    ($self: ident, $stream:ident, $stream_type: ident) => {
+        if $self.$stream.is_some() {
+            $self.close_stream(StreamType::$stream_type);
+            let _ = unistd::close($self.$stream.unwrap());
+            $self.$stream = None;
+        }
+    };
+}
+
 #[derive(Debug, PartialEq, Eq, Hash, Clone)]
 pub enum StreamType {
     Stdin,
@@ -147,6 +157,22 @@ impl Process {
         notify.notify_one();
     }
 
+    pub fn close_stdin(&mut self) {
+        close_process_stream!(self, term_master, TermMaster);
+        close_process_stream!(self, parent_stdin, ParentStdin);
+
+        self.notify_term_close();
+    }
+
+    pub fn cleanup_process_stream(&mut self) {
+        close_process_stream!(self, parent_stdin, ParentStdin);
+        close_process_stream!(self, parent_stdout, ParentStdout);
+        close_process_stream!(self, parent_stderr, ParentStderr);
+        close_process_stream!(self, term_master, TermMaster);
+
+        self.notify_term_close();
+    }
+
     fn get_fd(&self, stream_type: &StreamType) -> Option<RawFd> {
         match stream_type {
             StreamType::Stdin => self.stdin,

src/agent/rustjail/src/seccomp.rs

@@ -0,0 +1,237 @@
+// Copyright 2021 Sony Group Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use anyhow::{anyhow, Result};
+use libseccomp::*;
+use oci::{LinuxSeccomp, LinuxSeccompArg};
+use std::str::FromStr;
+
+fn get_filter_attr_from_flag(flag: &str) -> Result<ScmpFilterAttr> {
+    match flag {
+        "SECCOMP_FILTER_FLAG_TSYNC" => Ok(ScmpFilterAttr::CtlTsync),
+        "SECCOMP_FILTER_FLAG_LOG" => Ok(ScmpFilterAttr::CtlLog),
+        "SECCOMP_FILTER_FLAG_SPEC_ALLOW" => Ok(ScmpFilterAttr::CtlSsb),
+        _ => Err(anyhow!("Invalid seccomp flag")),
+    }
+}
+
+// get_rule_conditions gets rule conditions for a system call from the args.
+fn get_rule_conditions(args: &[LinuxSeccompArg]) -> Result<Vec<ScmpArgCompare>> {
+    let mut conditions: Vec<ScmpArgCompare> = Vec::new();
+
+    for arg in args {
+        if arg.op.is_empty() {
+            return Err(anyhow!("seccomp opreator is required"));
+        }
+
+        let cond = ScmpArgCompare::new(
+            arg.index,
+            ScmpCompareOp::from_str(&arg.op)?,
+            arg.value,
+            Some(arg.value_two),
+        );
+
+        conditions.push(cond);
+    }
+
+    Ok(conditions)
+}
+
+// init_seccomp creates a seccomp filter and loads it for the current process
+// including all the child processes.
+pub fn init_seccomp(scmp: &LinuxSeccomp) -> Result<()> {
+    let def_action = ScmpAction::from_str(scmp.default_action.as_str(), Some(libc::EPERM as u32))?;
+
+    // Create a new filter context
+    let mut filter = ScmpFilterContext::new_filter(def_action)?;
+
+    // Add extra architectures
+    for arch in &scmp.architectures {
+        let scmp_arch = ScmpArch::from_str(arch)?;
+        filter.add_arch(scmp_arch)?;
+    }
+
+    // Unset no new privileges bit
+    filter.set_no_new_privs_bit(false)?;
+
+    // Add a rule for each system call
+    for syscall in &scmp.syscalls {
+        if syscall.names.is_empty() {
+            return Err(anyhow!("syscall name is required"));
+        }
+
+        let action = ScmpAction::from_str(&syscall.action, Some(syscall.errno_ret))?;
+        if action == def_action {
+            continue;
+        }
+
+        for name in &syscall.names {
+            let syscall_num = get_syscall_from_name(name, None)?;
+
+            if syscall.args.is_empty() {
+                filter.add_rule(action, syscall_num, None)?;
+            } else {
+                let conditions = get_rule_conditions(&syscall.args)?;
+                filter.add_rule(action, syscall_num, Some(&conditions))?;
+            }
+        }
+    }
+
+    // Set filter attributes for each seccomp flag
+    for flag in &scmp.flags {
+        let scmp_attr = get_filter_attr_from_flag(flag)?;
+        filter.set_filter_attr(scmp_attr, 1)?;
+    }
+
+    // Load the filter
+    filter.load()?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::skip_if_not_root;
+    use libc::{dup3, process_vm_readv, EPERM, O_CLOEXEC};
+    use std::io::Error;
+    use std::ptr::null;
+
+    macro_rules! syscall_assert {
+        ($e1: expr, $e2: expr) => {
+            let mut errno: i32 = 0;
+            if $e1 < 0 {
+                errno = -Error::last_os_error().raw_os_error().unwrap();
+            }
+            assert_eq!(errno, $e2);
+        };
+    }
+
+    #[test]
+    fn test_get_filter_attr_from_flag() {
+        skip_if_not_root!();
+
+        assert_eq!(
+            get_filter_attr_from_flag("SECCOMP_FILTER_FLAG_TSYNC").unwrap(),
+            ScmpFilterAttr::CtlTsync
+        );
+
+        assert_eq!(get_filter_attr_from_flag("ERROR").is_err(), true);
+    }
+
+    #[test]
+    fn test_init_seccomp() {
+        skip_if_not_root!();
+
+        let data = r#"{
+            "defaultAction": "SCMP_ACT_ALLOW",
+            "architectures": [
+            ],
+            "flags": [
+                "SECCOMP_FILTER_FLAG_LOG"
+            ],
+            "syscalls": [
+                {
+                   "names": [
+                        "dup3"
+                    ],
+                    "action": "SCMP_ACT_ERRNO"
+                },
+                {
+                   "names": [
+                        "process_vm_readv"
+                    ],
+                    "action": "SCMP_ACT_ERRNO",
+                    "errnoRet": 111,
+                    "args": [
+                        {
+                            "index": 0,
+                            "value": 10,
+                            "op": "SCMP_CMP_EQ"
+                        }
+                    ]
+                },
+                {
+                   "names": [
+                        "process_vm_readv"
+                    ],
+                    "action": "SCMP_ACT_ERRNO",
+                    "errnoRet": 111,
+                    "args": [
+                        {
+                            "index": 0,
+                            "value": 20,
+                            "op": "SCMP_CMP_EQ"
+                        }
+                    ]
+                },
+                {
+                   "names": [
+                        "process_vm_readv"
+                    ],
+                    "action": "SCMP_ACT_ERRNO",
+                    "errnoRet": 222,
+                    "args": [
+                        {
+                            "index": 0,
+                            "value": 30,
+                            "op": "SCMP_CMP_EQ"
+                        },
+                        {
+                            "index": 2,
+                            "value": 40,
+                            "op": "SCMP_CMP_EQ"
+                        }
+                    ]
+                }
+            ]
+        }"#;
+
+        let mut scmp: oci::LinuxSeccomp = serde_json::from_str(data).unwrap();
+        let mut arch: Vec<oci::Arch>;
+
+        if cfg!(target_endian = "little") {
+            // For little-endian architectures
+            arch = vec![
+                "SCMP_ARCH_X86".to_string(),
+                "SCMP_ARCH_X32".to_string(),
+                "SCMP_ARCH_X86_64".to_string(),
+                "SCMP_ARCH_AARCH64".to_string(),
+                "SCMP_ARCH_ARM".to_string(),
+                "SCMP_ARCH_PPC64LE".to_string(),
+            ];
+        } else {
+            // For big-endian architectures
+            arch = vec!["SCMP_ARCH_S390X".to_string()];
+        }
+
+        scmp.architectures.append(&mut arch);
+
+        init_seccomp(&scmp).unwrap();
+
+        // Basic syscall with simple rule
+        syscall_assert!(unsafe { dup3(0, 1, O_CLOEXEC) }, -EPERM);
+
+        // Syscall with permitted arguments
+        syscall_assert!(unsafe { process_vm_readv(1, null(), 0, null(), 0, 0) }, 0);
+
+        // Multiple arguments with OR rules with ERRNO
+        syscall_assert!(
+            unsafe { process_vm_readv(10, null(), 0, null(), 0, 0) },
+            -111
+        );
+        syscall_assert!(
+            unsafe { process_vm_readv(20, null(), 0, null(), 0, 0) },
+            -111
+        );
+
+        // Multiple arguments with AND rules with ERRNO
+        syscall_assert!(unsafe { process_vm_readv(30, null(), 0, null(), 0, 0) }, 0);
+        syscall_assert!(
+            unsafe { process_vm_readv(30, null(), 40, null(), 0, 0) },
+            -222
+        );
+    }
+}

src/agent/samples/configuration-all-endpoints.toml

@@ -4,18 +4,38 @@ server_addr = 'vsock://8:2048'
 
 [endpoints]
 # All endpoints are allowed
-allowed = [ "CreateContainer", "StartContainer", "RemoveContainer",
-            "ExecProcess",  "SignalProcess", "WaitProcess",
-            "UpdateContainer", "StatsContainer", "PauseContainer", "ResumeContainer",
-            "WriteStdin", "ReadStdout", "ReadStderr", "CloseStdin", "TtyWinResize",
-            "UpdateInterface", "UpdateRoutes", "ListInterfaces", "ListRoutes", "AddARPNeighbors",
-            "StartTracing", "StopTracing", "GetMetrics",
-            "CreateSandbox", "DestroySandbox",
-            "OnlineCPUMem",
-            "ReseedRandomDev",
-            "GetGuestDetails",
-            "MemHotplugByProbe",
-            "SetGuestDateTime",
-            "CopyFile",
-            "GetOOMEvent",
-            "AddSwap"]
+allowed = [
+        "AddARPNeighborsRequest",
+        "AddSwapRequest",
+        "CloseStdinRequest",
+        "CopyFileRequest",
+        "CreateContainerRequest",
+        "CreateSandboxRequest",
+        "DestroySandboxRequest",
+        "ExecProcessRequest",
+        "GetMetricsRequest",
+        "GetOOMEventRequest",
+        "GuestDetailsRequest",
+        "ListInterfacesRequest",
+        "ListRoutesRequest",
+        "MemHotplugByProbeRequest",
+        "OnlineCPUMemRequest",
+        "PauseContainerRequest",
+        "PullImageRequest",
+        "ReadStreamRequest",
+        "RemoveContainerRequest",
+        "ReseedRandomDevRequest",
+        "ResumeContainerRequest",
+        "SetGuestDateTimeRequest",
+        "SignalProcessRequest",
+        "StartContainerRequest",
+        "StartTracingRequest",
+        "StatsContainerRequest",
+        "StopTracingRequest",
+        "TtyWinResizeRequest",
+        "UpdateContainerRequest",
+        "UpdateInterfaceRequest",
+        "UpdateRoutesRequest",
+        "WaitProcessRequest",
+        "WriteStreamRequest"
+]

src/agent/src/config.rs

@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 //
-use crate::tracer;
+use crate::rpc;
 use anyhow::{bail, ensure, Context, Result};
 use serde::Deserialize;
 use std::collections::HashSet;
@@ -33,7 +33,7 @@ const VSOCK_PORT: u16 = 1024;
 // Environment variables used for development and testing
 const SERVER_ADDR_ENV_VAR: &str = "KATA_AGENT_SERVER_ADDR";
 const LOG_LEVEL_ENV_VAR: &str = "KATA_AGENT_LOG_LEVEL";
-const TRACE_TYPE_ENV_VAR: &str = "KATA_AGENT_TRACE_TYPE";
+const TRACING_ENV_VAR: &str = "KATA_AGENT_TRACING";
 
 const ERR_INVALID_LOG_LEVEL: &str = "invalid log level";
 const ERR_INVALID_LOG_LEVEL_PARAM: &str = "invalid log level parameter";
@@ -73,8 +73,9 @@ pub struct AgentConfig {
     pub container_pipe_size: i32,
     pub server_addr: String,
     pub unified_cgroup_hierarchy: bool,
-    pub tracing: tracer::TraceType,
+    pub tracing: bool,
     pub endpoints: AgentEndpoints,
+    pub supports_seccomp: bool,
 }
 
 #[derive(Debug, Deserialize)]
@@ -88,7 +89,7 @@ pub struct AgentConfigBuilder {
     pub container_pipe_size: Option<i32>,
     pub server_addr: Option<String>,
     pub unified_cgroup_hierarchy: Option<bool>,
-    pub tracing: Option<tracer::TraceType>,
+    pub tracing: Option<bool>,
     pub endpoints: Option<EndpointsConfig>,
 }
 
@@ -148,8 +149,9 @@ impl Default for AgentConfig {
             container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
             server_addr: format!("{}:{}", VSOCK_ADDR, VSOCK_PORT),
             unified_cgroup_hierarchy: false,
-            tracing: tracer::TraceType::Disabled,
+            tracing: false,
             endpoints: Default::default(),
+            supports_seccomp: rpc::have_seccomp(),
         }
     }
 }
@@ -213,11 +215,11 @@ impl AgentConfig {
             // Support "bare" tracing option for backwards compatibility with
             // Kata 1.x.
             if param == &TRACE_MODE_OPTION {
-                config.tracing = tracer::TraceType::Isolated;
+                config.tracing = true;
                 continue;
             }
 
-            parse_cmdline_param!(param, TRACE_MODE_OPTION, config.tracing, get_trace_type);
+            parse_cmdline_param!(param, TRACE_MODE_OPTION, config.tracing, get_bool_value);
 
             // parse cmdline options
             parse_cmdline_param!(param, LOG_LEVEL_OPTION, config.log_level, get_log_level);
@@ -277,10 +279,10 @@ impl AgentConfig {
             }
         }
 
-        if let Ok(value) = env::var(TRACE_TYPE_ENV_VAR) {
-            if let Ok(result) = value.parse::<tracer::TraceType>() {
-                config.tracing = result;
-            }
+        if let Ok(value) = env::var(TRACING_ENV_VAR) {
+            let name_value = format!("{}={}", TRACING_ENV_VAR, value);
+
+            config.tracing = get_bool_value(&name_value)?;
         }
 
         // We did not get a configuration file: allow all endpoints.
@@ -343,25 +345,6 @@ fn get_log_level(param: &str) -> Result<slog::Level> {
     logrus_to_slog_level(fields[1])
 }
 
-#[instrument]
-fn get_trace_type(param: &str) -> Result<tracer::TraceType> {
-    ensure!(!param.is_empty(), "invalid trace type parameter");
-
-    let fields: Vec<&str> = param.split('=').collect();
-    ensure!(
-        fields[0] == TRACE_MODE_OPTION,
-        "invalid trace type key name"
-    );
-
-    if fields.len() == 1 {
-        return Ok(tracer::TraceType::Isolated);
-    }
-
-    let result = fields[1].parse::<tracer::TraceType>()?;
-
-    Ok(result)
-}
-
 #[instrument]
 fn get_hotplug_timeout(param: &str) -> Result<time::Duration> {
     let fields: Vec<&str> = param.split('=').collect();
@@ -446,10 +429,6 @@ mod tests {
     use std::time;
     use tempfile::tempdir;
 
-    const ERR_INVALID_TRACE_TYPE_PARAM: &str = "invalid trace type parameter";
-    const ERR_INVALID_TRACE_TYPE: &str = "invalid trace type";
-    const ERR_INVALID_TRACE_TYPE_KEY: &str = "invalid trace type key name";
-
     // Parameters:
     //
     // 1: expected Result
@@ -500,7 +479,7 @@ mod tests {
             container_pipe_size: i32,
             server_addr: &'a str,
             unified_cgroup_hierarchy: bool,
-            tracing: tracer::TraceType,
+            tracing: bool,
         }
 
         impl Default for TestData<'_> {
@@ -515,7 +494,7 @@ mod tests {
                     container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
                     server_addr: TEST_SERVER_ADDR,
                     unified_cgroup_hierarchy: false,
-                    tracing: tracer::TraceType::Disabled,
+                    tracing: false,
                 }
             }
         }
@@ -774,49 +753,115 @@ mod tests {
             },
             TestData {
                 contents: "trace",
-                tracing: tracer::TraceType::Disabled,
+                tracing: false,
                 ..Default::default()
             },
             TestData {
                 contents: ".trace",
-                tracing: tracer::TraceType::Disabled,
+                tracing: false,
                 ..Default::default()
             },
             TestData {
                 contents: "agent.tracer",
-                tracing: tracer::TraceType::Disabled,
+                tracing: false,
                 ..Default::default()
             },
             TestData {
                 contents: "agent.trac",
-                tracing: tracer::TraceType::Disabled,
+                tracing: false,
                 ..Default::default()
             },
             TestData {
                 contents: "agent.trace",
-                tracing: tracer::TraceType::Isolated,
+                tracing: true,
                 ..Default::default()
             },
             TestData {
-                contents: "agent.trace=isolated",
-                tracing: tracer::TraceType::Isolated,
+                contents: "agent.trace=true",
+                tracing: true,
                 ..Default::default()
             },
             TestData {
-                contents: "agent.trace=disabled",
-                tracing: tracer::TraceType::Disabled,
+                contents: "agent.trace=false",
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.trace=0",
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.trace=1",
+                tracing: true,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.trace=a",
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.trace=foo",
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.trace=.",
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.trace=,",
+                tracing: false,
                 ..Default::default()
             },
             TestData {
                 contents: "",
-                env_vars: vec!["KATA_AGENT_TRACE_TYPE=isolated"],
-                tracing: tracer::TraceType::Isolated,
+                env_vars: vec!["KATA_AGENT_TRACING="],
+                tracing: false,
                 ..Default::default()
             },
             TestData {
                 contents: "",
-                env_vars: vec!["KATA_AGENT_TRACE_TYPE=disabled"],
-                tracing: tracer::TraceType::Disabled,
+                env_vars: vec!["KATA_AGENT_TRACING=''"],
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_TRACING=0"],
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_TRACING=."],
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_TRACING=,"],
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_TRACING=foo"],
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_TRACING=1"],
+                tracing: true,
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_TRACING=true"],
+                tracing: true,
                 ..Default::default()
             },
         ];
@@ -1302,64 +1347,6 @@ Caused by:
         }
     }
 
-    #[test]
-    fn test_get_trace_type() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            param: &'a str,
-            result: Result<tracer::TraceType>,
-        }
-
-        let tests = &[
-            TestData {
-                param: "",
-                result: Err(anyhow!(ERR_INVALID_TRACE_TYPE_PARAM)),
-            },
-            TestData {
-                param: "agent.tracer",
-                result: Err(anyhow!(ERR_INVALID_TRACE_TYPE_KEY)),
-            },
-            TestData {
-                param: "agent.trac",
-                result: Err(anyhow!(ERR_INVALID_TRACE_TYPE_KEY)),
-            },
-            TestData {
-                param: "agent.trace=",
-                result: Err(anyhow!(ERR_INVALID_TRACE_TYPE)),
-            },
-            TestData {
-                param: "agent.trace==",
-                result: Err(anyhow!(ERR_INVALID_TRACE_TYPE)),
-            },
-            TestData {
-                param: "agent.trace=foo",
-                result: Err(anyhow!(ERR_INVALID_TRACE_TYPE)),
-            },
-            TestData {
-                param: "agent.trace",
-                result: Ok(tracer::TraceType::Isolated),
-            },
-            TestData {
-                param: "agent.trace=isolated",
-                result: Ok(tracer::TraceType::Isolated),
-            },
-            TestData {
-                param: "agent.trace=disabled",
-                result: Ok(tracer::TraceType::Disabled),
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let result = get_trace_type(d.param);
-
-            let msg = format!("{}: result: {:?}", msg, result);
-
-            assert_result!(d.result, result, msg);
-        }
-    }
-
     #[test]
     fn test_config_builder_from_string() {
         let config = AgentConfig::from_str(

src/agent/src/device.rs

@@ -7,7 +7,10 @@ use libc::{c_uint, major, minor};
 use nix::sys::stat;
 use regex::Regex;
 use std::collections::HashMap;
+use std::ffi::OsStr;
+use std::fmt;
 use std::fs;
+use std::os::unix::ffi::OsStrExt;
 use std::os::unix::fs::MetadataExt;
 use std::path::Path;
 use std::str::FromStr;
@@ -46,6 +49,9 @@ pub const DRIVER_LOCAL_TYPE: &str = "local";
 pub const DRIVER_WATCHABLE_BIND_TYPE: &str = "watchable-bind";
 // VFIO device to be bound to a guest kernel driver
 pub const DRIVER_VFIO_GK_TYPE: &str = "vfio-gk";
+// VFIO device to be bound to vfio-pci and made available inside the
+// container as a VFIO device node
+pub const DRIVER_VFIO_TYPE: &str = "vfio";
 
 #[derive(Debug)]
 struct DevIndexEntry {
@@ -62,6 +68,83 @@ pub fn online_device(path: &str) -> Result<()> {
     Ok(())
 }
 
+// Force a given PCI device to bind to the given driver, does
+// basically the same thing as
+//    driverctl set-override <PCI address> <driver>
+#[instrument]
+pub fn pci_driver_override<T, U>(syspci: T, dev: pci::Address, drv: U) -> Result<()>
+where
+    T: AsRef<OsStr> + std::fmt::Debug,
+    U: AsRef<OsStr> + std::fmt::Debug,
+{
+    let syspci = Path::new(&syspci);
+    let drv = drv.as_ref();
+    info!(sl!(), "rebind_pci_driver: {} => {:?}", dev, drv);
+
+    let devpath = syspci.join("devices").join(dev.to_string());
+    let overridepath = &devpath.join("driver_override");
+
+    fs::write(overridepath, drv.as_bytes())?;
+
+    let drvpath = &devpath.join("driver");
+    let need_unbind = match fs::read_link(drvpath) {
+        Ok(d) if d.file_name() == Some(drv) => return Ok(()), // Nothing to do
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => false, // No current driver
+        Err(e) => return Err(anyhow!("Error checking driver on {}: {}", dev, e)),
+        Ok(_) => true, // Current driver needs unbinding
+    };
+    if need_unbind {
+        let unbindpath = &drvpath.join("unbind");
+        fs::write(unbindpath, dev.to_string())?;
+    }
+    let probepath = syspci.join("drivers_probe");
+    fs::write(probepath, dev.to_string())?;
+    Ok(())
+}
+
+// Represents an IOMMU group
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct IommuGroup(u32);
+
+impl fmt::Display for IommuGroup {
+    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+        write!(f, "{}", self.0)
+    }
+}
+
+// Determine the IOMMU group of a PCI device
+#[instrument]
+fn pci_iommu_group<T>(syspci: T, dev: pci::Address) -> Result<Option<IommuGroup>>
+where
+    T: AsRef<OsStr> + std::fmt::Debug,
+{
+    let syspci = Path::new(&syspci);
+    let grouppath = syspci
+        .join("devices")
+        .join(dev.to_string())
+        .join("iommu_group");
+
+    match fs::read_link(&grouppath) {
+        // Device has no group
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
+        Err(e) => Err(anyhow!("Error reading link {:?}: {}", &grouppath, e)),
+        Ok(group) => {
+            if let Some(group) = group.file_name() {
+                if let Some(group) = group.to_str() {
+                    if let Ok(group) = group.parse::<u32>() {
+                        return Ok(Some(IommuGroup(group)));
+                    }
+                }
+            }
+            Err(anyhow!(
+                "Unexpected IOMMU group link {:?} => {:?}",
+                grouppath,
+                group
+            ))
+        }
+    }
+}
+
 // pcipath_to_sysfs fetches the sysfs path for a PCI path, relative to
 // the sysfs path for the PCI host bridge, based on the PCI path
 // provided.
@@ -296,6 +379,33 @@ pub async fn wait_for_pci_device(
     Ok(addr)
 }
 
+#[derive(Debug)]
+struct VfioMatcher {
+    syspath: String,
+}
+
+impl VfioMatcher {
+    fn new(grp: IommuGroup) -> VfioMatcher {
+        VfioMatcher {
+            syspath: format!("/devices/virtual/vfio/{}", grp),
+        }
+    }
+}
+
+impl UeventMatcher for VfioMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.devpath == self.syspath
+    }
+}
+
+#[instrument]
+async fn get_vfio_device_name(sandbox: &Arc<Mutex<Sandbox>>, grp: IommuGroup) -> Result<String> {
+    let matcher = VfioMatcher::new(grp);
+
+    let uev = wait_for_uevent(sandbox, matcher).await?;
+    Ok(format!("{}/{}", SYSTEM_DEV_PATH, &uev.devname))
+}
+
 /// Scan SCSI bus for the given SCSI address(SCSI-Id and LUN)
 #[instrument]
 fn scan_scsi_bus(scsi_addr: &str) -> Result<()> {
@@ -326,24 +436,27 @@ fn scan_scsi_bus(scsi_addr: &str) -> Result<()> {
     Ok(())
 }
 
-// update_spec_device_list takes a device description provided by the caller,
-// trying to find it on the guest. Once this device has been identified, the
-// "real" information that can be read from inside the VM is used to update
-// the same device in the list of devices provided through the OCI spec.
-// This is needed to update information about minor/major numbers that cannot
-// be predicted from the caller.
+// update_spec_device updates the device list in the OCI spec to make
+// it include details appropriate for the VM, instead of the host.  It
+// is given the host path to the device (to locate the device in the
+// original OCI spec) and the VM path which it uses to determine the
+// VM major/minor numbers, and the final path with which to present
+// the device in the (inner) container
 #[instrument]
-fn update_spec_device_list(device: &Device, spec: &mut Spec, devidx: &DevIndex) -> Result<()> {
+fn update_spec_device(
+    spec: &mut Spec,
+    devidx: &DevIndex,
+    host_path: &str,
+    vm_path: &str,
+    final_path: &str,
+) -> Result<()> {
     let major_id: c_uint;
     let minor_id: c_uint;
 
     // If no container_path is provided, we won't be able to match and
     // update the device in the OCI spec device list. This is an error.
-    if device.container_path.is_empty() {
-        return Err(anyhow!(
-            "container_path cannot empty for device {:?}",
-            device
-        ));
+    if host_path.is_empty() {
+        return Err(anyhow!("Host path cannot empty for device"));
     }
 
     let linux = spec
@@ -351,11 +464,11 @@ fn update_spec_device_list(device: &Device, spec: &mut Spec, devidx: &DevIndex)
         .as_mut()
         .ok_or_else(|| anyhow!("Spec didn't container linux field"))?;
 
-    if !Path::new(&device.vm_path).exists() {
-        return Err(anyhow!("vm_path:{} doesn't exist", device.vm_path));
+    if !Path::new(vm_path).exists() {
+        return Err(anyhow!("vm_path:{} doesn't exist", vm_path));
     }
 
-    let meta = fs::metadata(&device.vm_path)?;
+    let meta = fs::metadata(vm_path)?;
     let dev_id = meta.rdev();
     unsafe {
         major_id = major(dev_id);
@@ -364,24 +477,27 @@ fn update_spec_device_list(device: &Device, spec: &mut Spec, devidx: &DevIndex)
 
     info!(
         sl!(),
-        "got the device: dev_path: {}, major: {}, minor: {}\n", &device.vm_path, major_id, minor_id
+        "update_spec_device(): vm_path={}, major: {}, minor: {}\n", vm_path, major_id, minor_id
     );
 
-    if let Some(idxdata) = devidx.0.get(device.container_path.as_str()) {
+    if let Some(idxdata) = devidx.0.get(host_path) {
         let dev = &mut linux.devices[idxdata.idx];
         let host_major = dev.major;
         let host_minor = dev.minor;
 
         dev.major = major_id as i64;
         dev.minor = minor_id as i64;
+        dev.path = final_path.to_string();
 
         info!(
             sl!(),
-            "change the device from major: {} minor: {} to vm device major: {} minor: {}",
+            "change the device from path: {} major: {} minor: {} to vm device path: {} major: {} minor: {}",
+            host_path,
             host_major,
             host_minor,
-            major_id,
-            minor_id
+            dev.path,
+            dev.major,
+            dev.minor,
         );
 
         // Resources must be updated since they are used to identify
@@ -402,7 +518,7 @@ fn update_spec_device_list(device: &Device, spec: &mut Spec, devidx: &DevIndex)
     } else {
         Err(anyhow!(
             "Should have found a matching device {} in the spec",
-            device.vm_path
+            vm_path
         ))
     }
 }
@@ -420,7 +536,13 @@ async fn virtiommio_blk_device_handler(
         return Err(anyhow!("Invalid path for virtio mmio blk device"));
     }
 
-    update_spec_device_list(device, spec, devidx)
+    update_spec_device(
+        spec,
+        devidx,
+        &device.container_path,
+        &device.vm_path,
+        &device.container_path,
+    )
 }
 
 // device.Id should be a PCI path string
@@ -436,7 +558,13 @@ async fn virtio_blk_device_handler(
 
     dev.vm_path = get_virtio_blk_pci_device_name(sandbox, &pcipath).await?;
 
-    update_spec_device_list(&dev, spec, devidx)
+    update_spec_device(
+        spec,
+        devidx,
+        &dev.container_path,
+        &dev.vm_path,
+        &dev.container_path,
+    )
 }
 
 // device.id should be a CCW path string
@@ -451,7 +579,13 @@ async fn virtio_blk_ccw_device_handler(
     let mut dev = device.clone();
     let ccw_device = ccw::Device::from_str(&device.id)?;
     dev.vm_path = get_virtio_blk_ccw_device_name(sandbox, &ccw_device).await?;
-    update_spec_device_list(&dev, spec, devidx)
+    update_spec_device(
+        spec,
+        devidx,
+        &dev.container_path,
+        &dev.vm_path,
+        &dev.container_path,
+    )
 }
 
 #[cfg(not(target_arch = "s390x"))]
@@ -475,7 +609,13 @@ async fn virtio_scsi_device_handler(
 ) -> Result<()> {
     let mut dev = device.clone();
     dev.vm_path = get_scsi_device_name(sandbox, &device.id).await?;
-    update_spec_device_list(&dev, spec, devidx)
+    update_spec_device(
+        spec,
+        devidx,
+        &dev.container_path,
+        &dev.vm_path,
+        &dev.container_path,
+    )
 }
 
 #[instrument]
@@ -489,7 +629,13 @@ async fn virtio_nvdimm_device_handler(
         return Err(anyhow!("Invalid path for nvdimm device"));
     }
 
-    update_spec_device_list(device, spec, devidx)
+    update_spec_device(
+        spec,
+        devidx,
+        &device.container_path,
+        &device.vm_path,
+        &device.container_path,
+    )
 }
 
 fn split_vfio_option(opt: &str) -> Option<(&str, &str)> {
@@ -507,19 +653,54 @@ fn split_vfio_option(opt: &str) -> Option<(&str, &str)> {
 // Each option should have the form "DDDD:BB:DD.F=<pcipath>"
 //     DDDD:BB:DD.F is the device's PCI address in the host
 //     <pcipath> is a PCI path to the device in the guest (see pci.rs)
-async fn vfio_gk_device_handler(
+async fn vfio_device_handler(
     device: &Device,
-    _: &mut Spec,
+    spec: &mut Spec,
     sandbox: &Arc<Mutex<Sandbox>>,
-    _: &DevIndex,
+    devidx: &DevIndex,
 ) -> Result<()> {
+    let vfio_in_guest = device.field_type != DRIVER_VFIO_GK_TYPE;
+    let mut group = None;
+
     for opt in device.options.iter() {
         let (_, pcipath) =
             split_vfio_option(opt).ok_or_else(|| anyhow!("Malformed VFIO option {:?}", opt))?;
         let pcipath = pci::Path::from_str(pcipath)?;
 
-        wait_for_pci_device(sandbox, &pcipath).await?;
+        let guestdev = wait_for_pci_device(sandbox, &pcipath).await?;
+        if vfio_in_guest {
+            pci_driver_override(SYSFS_BUS_PCI_PATH, guestdev, "vfio-pci")?;
+
+            let devgroup = pci_iommu_group(SYSFS_BUS_PCI_PATH, guestdev)?;
+            if devgroup.is_none() {
+                // Devices must have an IOMMU group to be usable via VFIO
+                return Err(anyhow!("{} has no IOMMU group", guestdev));
+            }
+
+            if group.is_some() && group != devgroup {
+                // If PCI devices associated with the same VFIO device
+                // (and therefore group) in the host don't end up in
+                // the same group in the guest, something has gone
+                // horribly wrong
+                return Err(anyhow!(
+                    "{} is not in guest IOMMU group {}",
+                    guestdev,
+                    group.unwrap()
+                ));
+            }
+
+            group = devgroup;
+        }
     }
+
+    if vfio_in_guest {
+        // If there are any devices at all, logic above ensures that group is not None
+        let group = group.unwrap();
+        let vmpath = get_vfio_device_name(sandbox, group).await?;
+
+        update_spec_device(spec, devidx, &device.container_path, &vmpath, &vmpath)?;
+    }
+
     Ok(())
 }
 
@@ -592,7 +773,9 @@ async fn add_device(
         DRIVER_MMIO_BLK_TYPE => virtiommio_blk_device_handler(device, spec, sandbox, devidx).await,
         DRIVER_NVDIMM_TYPE => virtio_nvdimm_device_handler(device, spec, sandbox, devidx).await,
         DRIVER_SCSI_TYPE => virtio_scsi_device_handler(device, spec, sandbox, devidx).await,
-        DRIVER_VFIO_GK_TYPE => vfio_gk_device_handler(device, spec, sandbox, devidx).await,
+        DRIVER_VFIO_GK_TYPE | DRIVER_VFIO_TYPE => {
+            vfio_device_handler(device, spec, sandbox, devidx).await
+        }
         _ => Err(anyhow!("Unknown device type {}", device.field_type)),
     }
 }
@@ -657,28 +840,28 @@ mod tests {
     }
 
     #[test]
-    fn test_update_spec_device_list() {
+    fn test_update_spec_device() {
         let (major, minor) = (7, 2);
-        let mut device = Device::default();
         let mut spec = Spec::default();
 
         // container_path empty
+        let container_path = "";
+        let vm_path = "";
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_err());
 
-        device.container_path = "/dev/null".to_string();
-
         // linux is empty
+        let container_path = "/dev/null";
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_err());
 
         spec.linux = Some(Linux::default());
 
         // linux.devices is empty
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_err());
 
         spec.linux.as_mut().unwrap().devices = vec![oci::LinuxDevice {
@@ -690,26 +873,32 @@ mod tests {
 
         // vm_path empty
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_err());
 
-        device.vm_path = "/dev/null".to_string();
+        let vm_path = "/dev/null";
 
         // guest and host path are not the same
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
-        assert!(res.is_err(), "device={:?} spec={:?}", device, spec);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
+        assert!(
+            res.is_err(),
+            "container_path={:?} vm_path={:?} spec={:?}",
+            container_path,
+            vm_path,
+            spec
+        );
 
-        spec.linux.as_mut().unwrap().devices[0].path = device.container_path.clone();
+        spec.linux.as_mut().unwrap().devices[0].path = container_path.to_string();
 
         // spec.linux.resources is empty
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_ok());
 
         // update both devices and cgroup lists
         spec.linux.as_mut().unwrap().devices = vec![oci::LinuxDevice {
-            path: device.container_path.clone(),
+            path: container_path.to_string(),
             major,
             minor,
             ..oci::LinuxDevice::default()
@@ -725,12 +914,12 @@ mod tests {
         });
 
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_ok());
     }
 
     #[test]
-    fn test_update_spec_device_list_guest_host_conflict() {
+    fn test_update_spec_device_guest_host_conflict() {
         let null_rdev = fs::metadata("/dev/null").unwrap().rdev();
         let zero_rdev = fs::metadata("/dev/zero").unwrap().rdev();
         let full_rdev = fs::metadata("/dev/full").unwrap().rdev();
@@ -781,20 +970,14 @@ mod tests {
         };
         let devidx = DevIndex::new(&spec);
 
-        let dev_a = Device {
-            container_path: "/dev/a".to_string(),
-            vm_path: "/dev/zero".to_string(),
-            ..Device::default()
-        };
+        let container_path_a = "/dev/a";
+        let vm_path_a = "/dev/zero";
 
         let guest_major_a = stat::major(zero_rdev) as i64;
         let guest_minor_a = stat::minor(zero_rdev) as i64;
 
-        let dev_b = Device {
-            container_path: "/dev/b".to_string(),
-            vm_path: "/dev/full".to_string(),
-            ..Device::default()
-        };
+        let container_path_b = "/dev/b";
+        let vm_path_b = "/dev/full";
 
         let guest_major_b = stat::major(full_rdev) as i64;
         let guest_minor_b = stat::minor(full_rdev) as i64;
@@ -811,7 +994,13 @@ mod tests {
         assert_eq!(Some(host_major_b), specresources.devices[1].major);
         assert_eq!(Some(host_minor_b), specresources.devices[1].minor);
 
-        let res = update_spec_device_list(&dev_a, &mut spec, &devidx);
+        let res = update_spec_device(
+            &mut spec,
+            &devidx,
+            container_path_a,
+            vm_path_a,
+            container_path_a,
+        );
         assert!(res.is_ok());
 
         let specdevices = &spec.linux.as_ref().unwrap().devices;
@@ -826,7 +1015,13 @@ mod tests {
         assert_eq!(Some(host_major_b), specresources.devices[1].major);
         assert_eq!(Some(host_minor_b), specresources.devices[1].minor);
 
-        let res = update_spec_device_list(&dev_b, &mut spec, &devidx);
+        let res = update_spec_device(
+            &mut spec,
+            &devidx,
+            container_path_b,
+            vm_path_b,
+            container_path_b,
+        );
         assert!(res.is_ok());
 
         let specdevices = &spec.linux.as_ref().unwrap().devices;
@@ -843,7 +1038,7 @@ mod tests {
     }
 
     #[test]
-    fn test_update_spec_device_list_char_block_conflict() {
+    fn test_update_spec_device_char_block_conflict() {
         let null_rdev = fs::metadata("/dev/null").unwrap().rdev();
 
         let guest_major = stat::major(null_rdev) as i64;
@@ -892,11 +1087,8 @@ mod tests {
         };
         let devidx = DevIndex::new(&spec);
 
-        let dev = Device {
-            container_path: "/dev/char".to_string(),
-            vm_path: "/dev/null".to_string(),
-            ..Device::default()
-        };
+        let container_path = "/dev/char";
+        let vm_path = "/dev/null";
 
         let specresources = spec.linux.as_ref().unwrap().resources.as_ref().unwrap();
         assert_eq!(Some(host_major), specresources.devices[0].major);
@@ -904,7 +1096,7 @@ mod tests {
         assert_eq!(Some(host_major), specresources.devices[1].major);
         assert_eq!(Some(host_minor), specresources.devices[1].minor);
 
-        let res = update_spec_device_list(&dev, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_ok());
 
         // Only the char device, not the block device should be updated
@@ -915,6 +1107,43 @@ mod tests {
         assert_eq!(Some(host_minor), specresources.devices[1].minor);
     }
 
+    #[test]
+    fn test_update_spec_device_final_path() {
+        let null_rdev = fs::metadata("/dev/null").unwrap().rdev();
+        let guest_major = stat::major(null_rdev) as i64;
+        let guest_minor = stat::minor(null_rdev) as i64;
+
+        let host_path = "/dev/host";
+        let host_major: i64 = 99;
+        let host_minor: i64 = 99;
+
+        let mut spec = Spec {
+            linux: Some(Linux {
+                devices: vec![oci::LinuxDevice {
+                    path: host_path.to_string(),
+                    r#type: "c".to_string(),
+                    major: host_major,
+                    minor: host_minor,
+                    ..oci::LinuxDevice::default()
+                }],
+                ..Linux::default()
+            }),
+            ..Spec::default()
+        };
+        let devidx = DevIndex::new(&spec);
+
+        let vm_path = "/dev/null";
+        let final_path = "/dev/final";
+
+        let res = update_spec_device(&mut spec, &devidx, host_path, vm_path, final_path);
+        assert!(res.is_ok());
+
+        let specdevices = &spec.linux.as_ref().unwrap().devices;
+        assert_eq!(guest_major, specdevices[0].major);
+        assert_eq!(guest_minor, specdevices[0].minor);
+        assert_eq!(final_path, specdevices[0].path);
+    }
+
     #[test]
     fn test_pcipath_to_sysfs() {
         let testdir = tempdir().expect("failed to create tmpdir");
@@ -1142,6 +1371,27 @@ mod tests {
         assert!(!matcher_a.is_match(&uev_b));
     }
 
+    #[tokio::test]
+    async fn test_vfio_matcher() {
+        let grpa = IommuGroup(1);
+        let grpb = IommuGroup(22);
+
+        let mut uev_a = crate::uevent::Uevent::default();
+        uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
+        uev_a.devname = format!("vfio/{}", grpa);
+        uev_a.devpath = format!("/devices/virtual/vfio/{}", grpa);
+        let matcher_a = VfioMatcher::new(grpa);
+
+        let mut uev_b = uev_a.clone();
+        uev_b.devpath = format!("/devices/virtual/vfio/{}", grpb);
+        let matcher_b = VfioMatcher::new(grpb);
+
+        assert!(matcher_a.is_match(&uev_a));
+        assert!(matcher_b.is_match(&uev_b));
+        assert!(!matcher_b.is_match(&uev_a));
+        assert!(!matcher_a.is_match(&uev_b));
+    }
+
     #[test]
     fn test_split_vfio_option() {
         assert_eq!(
@@ -1151,4 +1401,81 @@ mod tests {
         assert_eq!(split_vfio_option("0000:01:00.0=02/01=rubbish"), None);
         assert_eq!(split_vfio_option("0000:01:00.0"), None);
     }
+
+    #[test]
+    fn test_pci_driver_override() {
+        let testdir = tempdir().expect("failed to create tmpdir");
+        let syspci = testdir.path(); // Path to mock /sys/bus/pci
+
+        let dev0 = pci::Address::new(0, 0, pci::SlotFn::new(0, 0).unwrap());
+        let dev0path = syspci.join("devices").join(dev0.to_string());
+        let dev0drv = dev0path.join("driver");
+        let dev0override = dev0path.join("driver_override");
+
+        let drvapath = syspci.join("drivers").join("drv_a");
+        let drvaunbind = drvapath.join("unbind");
+
+        let probepath = syspci.join("drivers_probe");
+
+        // Start mocking dev0 as being unbound
+        fs::create_dir_all(&dev0path).unwrap();
+
+        pci_driver_override(syspci, dev0, "drv_a").unwrap();
+        assert_eq!(fs::read_to_string(&dev0override).unwrap(), "drv_a");
+        assert_eq!(fs::read_to_string(&probepath).unwrap(), dev0.to_string());
+
+        // Now mock dev0 already being attached to drv_a
+        fs::create_dir_all(&drvapath).unwrap();
+        std::os::unix::fs::symlink(&drvapath, dev0drv).unwrap();
+        std::fs::remove_file(&probepath).unwrap();
+
+        pci_driver_override(syspci, dev0, "drv_a").unwrap(); // no-op
+        assert_eq!(fs::read_to_string(&dev0override).unwrap(), "drv_a");
+        assert!(!probepath.exists());
+
+        // Now try binding to a different driver
+        pci_driver_override(syspci, dev0, "drv_b").unwrap();
+        assert_eq!(fs::read_to_string(&dev0override).unwrap(), "drv_b");
+        assert_eq!(fs::read_to_string(&probepath).unwrap(), dev0.to_string());
+        assert_eq!(fs::read_to_string(&drvaunbind).unwrap(), dev0.to_string());
+    }
+
+    #[test]
+    fn test_pci_iommu_group() {
+        let testdir = tempdir().expect("failed to create tmpdir"); // mock /sys
+        let syspci = testdir.path().join("bus").join("pci");
+
+        // Mock dev0, which has no group
+        let dev0 = pci::Address::new(0, 0, pci::SlotFn::new(0, 0).unwrap());
+        let dev0path = syspci.join("devices").join(dev0.to_string());
+
+        fs::create_dir_all(&dev0path).unwrap();
+
+        // Test dev0
+        assert!(pci_iommu_group(&syspci, dev0).unwrap().is_none());
+
+        // Mock dev1, which is in group 12
+        let dev1 = pci::Address::new(0, 1, pci::SlotFn::new(0, 0).unwrap());
+        let dev1path = syspci.join("devices").join(dev1.to_string());
+        let dev1group = dev1path.join("iommu_group");
+
+        fs::create_dir_all(&dev1path).unwrap();
+        std::os::unix::fs::symlink("../../../kernel/iommu_groups/12", &dev1group).unwrap();
+
+        // Test dev1
+        assert_eq!(
+            pci_iommu_group(&syspci, dev1).unwrap(),
+            Some(IommuGroup(12))
+        );
+
+        // Mock dev2, which has a bogus group (dir instead of symlink)
+        let dev2 = pci::Address::new(0, 2, pci::SlotFn::new(0, 0).unwrap());
+        let dev2path = syspci.join("devices").join(dev2.to_string());
+        let dev2group = dev2path.join("iommu_group");
+
+        fs::create_dir_all(&dev2group).unwrap();
+
+        // Test dev2
+        assert!(pci_iommu_group(&syspci, dev2).is_err());
+    }
 }

src/agent/src/linux_abi.rs

@@ -83,6 +83,8 @@ pub const SYSFS_MEMORY_ONLINE_PATH: &str = "/sys/devices/system/memory";
 
 pub const SYSFS_SCSI_HOST_PATH: &str = "/sys/class/scsi_host";
 
+pub const SYSFS_BUS_PCI_PATH: &str = "/sys/bus/pci";
+
 pub const SYSFS_CGROUPPATH: &str = "/sys/fs/cgroup";
 pub const SYSFS_ONLINE_FILE: &str = "online";
 
@@ -94,6 +96,7 @@ pub const SYSTEM_DEV_PATH: &str = "/dev";
 // Linux UEvent related consts.
 pub const U_EVENT_ACTION: &str = "ACTION";
 pub const U_EVENT_ACTION_ADD: &str = "add";
+pub const U_EVENT_ACTION_REMOVE: &str = "remove";
 pub const U_EVENT_DEV_PATH: &str = "DEVPATH";
 pub const U_EVENT_SUB_SYSTEM: &str = "SUBSYSTEM";
 pub const U_EVENT_SEQ_NUM: &str = "SEQNUM";

src/agent/src/main.rs

@@ -196,8 +196,8 @@ async fn real_main() -> std::result::Result<(), Box<dyn std::error::Error>> {
         ttrpc_log_guard = Ok(slog_stdlog::init().map_err(|e| e)?);
     }
 
-    if config.tracing != tracer::TraceType::Disabled {
-        let _ = tracer::setup_tracing(NAME, &logger, &config)?;
+    if config.tracing {
+        tracer::setup_tracing(NAME, &logger)?;
     }
 
     let root_span = span!(tracing::Level::TRACE, "root-span");
@@ -229,23 +229,29 @@ async fn real_main() -> std::result::Result<(), Box<dyn std::error::Error>> {
     // Wait for all threads to finish
     let results = join_all(tasks).await;
 
-    for result in results {
-        if let Err(e) = result {
-            return Err(anyhow!(e).into());
-        }
-    }
-
     // force flushing spans
     drop(span_guard);
     drop(root_span);
 
-    if config.tracing != tracer::TraceType::Disabled {
+    if config.tracing {
         tracer::end_tracing();
     }
 
     eprintln!("{} shutdown complete", NAME);
 
-    Ok(())
+    let mut wait_errors: Vec<tokio::task::JoinError> = vec![];
+    for result in results {
+        if let Err(e) = result {
+            eprintln!("wait task error: {:#?}", e);
+            wait_errors.push(e);
+        }
+    }
+
+    if wait_errors.is_empty() {
+        Ok(())
+    } else {
+        Err(anyhow!("wait all tasks failed: {:#?}", wait_errors).into())
+    }
 }
 
 fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {

src/agent/src/metrics.rs

@@ -23,50 +23,50 @@ macro_rules! sl {
 lazy_static! {
 
     static ref     AGENT_SCRAPE_COUNT: IntCounter =
-    prometheus::register_int_counter!(format!("{}_{}",NAMESPACE_KATA_AGENT,"scrape_count").as_ref(), "Metrics scrape count").unwrap();
+    prometheus::register_int_counter!(format!("{}_{}",NAMESPACE_KATA_AGENT,"scrape_count"), "Metrics scrape count").unwrap();
 
     static ref     AGENT_THREADS: Gauge =
-    prometheus::register_gauge!(format!("{}_{}",NAMESPACE_KATA_AGENT,"threads").as_ref(), "Agent process threads").unwrap();
+    prometheus::register_gauge!(format!("{}_{}",NAMESPACE_KATA_AGENT,"threads"), "Agent process threads").unwrap();
 
     static ref     AGENT_TOTAL_TIME: Gauge =
-    prometheus::register_gauge!(format!("{}_{}",NAMESPACE_KATA_AGENT,"total_time").as_ref(), "Agent process total time").unwrap();
+    prometheus::register_gauge!(format!("{}_{}",NAMESPACE_KATA_AGENT,"total_time"), "Agent process total time").unwrap();
 
     static ref     AGENT_TOTAL_VM: Gauge =
-    prometheus::register_gauge!(format!("{}_{}",NAMESPACE_KATA_AGENT,"total_vm").as_ref(), "Agent process total VM size").unwrap();
+    prometheus::register_gauge!(format!("{}_{}",NAMESPACE_KATA_AGENT,"total_vm"), "Agent process total VM size").unwrap();
 
     static ref     AGENT_TOTAL_RSS: Gauge =
-    prometheus::register_gauge!(format!("{}_{}",NAMESPACE_KATA_AGENT,"total_rss").as_ref(), "Agent process total RSS size").unwrap();
+    prometheus::register_gauge!(format!("{}_{}",NAMESPACE_KATA_AGENT,"total_rss"), "Agent process total RSS size").unwrap();
 
     static ref     AGENT_PROC_STATUS: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_AGENT,"proc_status").as_ref(), "Agent process status.", &["item"]).unwrap();
+    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_AGENT,"proc_status"), "Agent process status.", &["item"]).unwrap();
 
     static ref     AGENT_IO_STAT: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_AGENT,"io_stat").as_ref(), "Agent process IO statistics.", &["item"]).unwrap();
+    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_AGENT,"io_stat"), "Agent process IO statistics.", &["item"]).unwrap();
 
     static ref     AGENT_PROC_STAT: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_AGENT,"proc_stat").as_ref(), "Agent process statistics.", &["item"]).unwrap();
+    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_AGENT,"proc_stat"), "Agent process statistics.", &["item"]).unwrap();
 
     // guest os metrics
     static ref     GUEST_LOAD: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"load").as_ref() , "Guest system load.", &["item"]).unwrap();
+    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"load") , "Guest system load.", &["item"]).unwrap();
 
     static ref     GUEST_TASKS: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"tasks").as_ref() , "Guest system load.", &["item"]).unwrap();
+    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"tasks") , "Guest system load.", &["item"]).unwrap();
 
     static ref     GUEST_CPU_TIME: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"cpu_time").as_ref() , "Guest CPU statistics.", &["cpu","item"]).unwrap();
+    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"cpu_time") , "Guest CPU statistics.", &["cpu","item"]).unwrap();
 
     static ref     GUEST_VM_STAT: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"vm_stat").as_ref() , "Guest virtual memory statistics.", &["item"]).unwrap();
+    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"vm_stat") , "Guest virtual memory statistics.", &["item"]).unwrap();
 
     static ref     GUEST_NETDEV_STAT: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"netdev_stat").as_ref() , "Guest net devices statistics.", &["interface","item"]).unwrap();
+    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"netdev_stat") , "Guest net devices statistics.", &["interface","item"]).unwrap();
 
     static ref     GUEST_DISKSTAT: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"diskstat").as_ref() , "Disks statistics in system.", &["disk","item"]).unwrap();
+    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"diskstat") , "Disks statistics in system.", &["disk","item"]).unwrap();
 
     static ref     GUEST_MEMINFO: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"meminfo").as_ref() , "Statistics about memory usage in the system.", &["item"]).unwrap();
+    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"meminfo") , "Statistics about memory usage in the system.", &["item"]).unwrap();
 }
 
 #[instrument]
@@ -348,17 +348,17 @@ fn set_gauge_vec_cpu_time(gv: &prometheus::GaugeVec, cpu: &str, cpu_time: &procf
     gv.with_label_values(&[cpu, "idle"])
         .set(cpu_time.idle as f64);
     gv.with_label_values(&[cpu, "iowait"])
-        .set(cpu_time.iowait.unwrap_or(0.0) as f64);
+        .set(cpu_time.iowait.unwrap_or(0) as f64);
     gv.with_label_values(&[cpu, "irq"])
-        .set(cpu_time.irq.unwrap_or(0.0) as f64);
+        .set(cpu_time.irq.unwrap_or(0) as f64);
     gv.with_label_values(&[cpu, "softirq"])
-        .set(cpu_time.softirq.unwrap_or(0.0) as f64);
+        .set(cpu_time.softirq.unwrap_or(0) as f64);
     gv.with_label_values(&[cpu, "steal"])
-        .set(cpu_time.steal.unwrap_or(0.0) as f64);
+        .set(cpu_time.steal.unwrap_or(0) as f64);
     gv.with_label_values(&[cpu, "guest"])
-        .set(cpu_time.guest.unwrap_or(0.0) as f64);
+        .set(cpu_time.guest.unwrap_or(0) as f64);
     gv.with_label_values(&[cpu, "guest_nice"])
-        .set(cpu_time.guest_nice.unwrap_or(0.0) as f64);
+        .set(cpu_time.guest_nice.unwrap_or(0) as f64);
 }
 
 #[instrument]
@@ -470,7 +470,7 @@ fn set_gauge_vec_proc_status(gv: &prometheus::GaugeVec, status: &procfs::process
     gv.with_label_values(&["vmswap"])
         .set(status.vmswap.unwrap_or(0) as f64);
     gv.with_label_values(&["hugetlbpages"])
-        .set(status.hugetblpages.unwrap_or(0) as f64);
+        .set(status.hugetlbpages.unwrap_or(0) as f64);
     gv.with_label_values(&["voluntary_ctxt_switches"])
         .set(status.voluntary_ctxt_switches.unwrap_or(0) as f64);
     gv.with_label_values(&["nonvoluntary_ctxt_switches"])

src/agent/src/mount.rs

@@ -405,14 +405,18 @@ async fn bind_watcher_storage_handler(
     logger: &Logger,
     storage: &Storage,
     sandbox: Arc<Mutex<Sandbox>>,
+    cid: Option<String>,
 ) -> Result<()> {
     let mut locked = sandbox.lock().await;
-    let container_id = locked.id.clone();
 
-    locked
-        .bind_watcher
-        .add_container(container_id, iter::once(storage.clone()), logger)
-        .await
+    if let Some(cid) = cid {
+        locked
+            .bind_watcher
+            .add_container(cid, iter::once(storage.clone()), logger)
+            .await
+    } else {
+        Ok(())
+    }
 }
 
 // mount_storage performs the mount described by the storage structure.
@@ -518,6 +522,7 @@ pub async fn add_storages(
     logger: Logger,
     storages: Vec<Storage>,
     sandbox: Arc<Mutex<Sandbox>>,
+    cid: Option<String>,
 ) -> Result<Vec<String>> {
     let mut mount_list = Vec::new();
 
@@ -548,7 +553,8 @@ pub async fn add_storages(
             }
             DRIVER_NVDIMM_TYPE => nvdimm_storage_handler(&logger, &storage, sandbox.clone()).await,
             DRIVER_WATCHABLE_BIND_TYPE => {
-                bind_watcher_storage_handler(&logger, &storage, sandbox.clone()).await?;
+                bind_watcher_storage_handler(&logger, &storage, sandbox.clone(), cid.clone())
+                    .await?;
                 // Don't register watch mounts, they're handled separately by the watcher.
                 Ok(String::new())
             }

src/agent/src/netlink.rs

@@ -6,6 +6,7 @@
 use anyhow::{anyhow, Context, Result};
 use futures::{future, StreamExt, TryStreamExt};
 use ipnetwork::{IpNetwork, Ipv4Network, Ipv6Network};
+use nix::errno::Errno;
 use protobuf::RepeatedField;
 use protocols::types::{ARPNeighbor, IPAddress, IPFamily, Interface, Route};
 use rtnetlink::{new_connection, packet, IpVersion};
@@ -363,14 +364,17 @@ impl Handle {
                     request = request.gateway(ip);
                 }
 
-                request.execute().await.with_context(|| {
-                    format!(
-                        "Failed to add IP v6 route (src: {}, dst: {}, gtw: {})",
-                        route.get_source(),
-                        route.get_dest(),
-                        route.get_gateway()
-                    )
-                })?;
+                if let Err(rtnetlink::Error::NetlinkError(message)) = request.execute().await {
+                    if Errno::from_i32(message.code.abs()) != Errno::EEXIST {
+                        return Err(anyhow!(
+                            "Failed to add IP v6 route (src: {}, dst: {}, gtw: {},Err: {})",
+                            route.get_source(),
+                            route.get_dest(),
+                            route.get_gateway(),
+                            message
+                        ));
+                    }
+                }
             } else {
                 let dest_addr = if !route.dest.is_empty() {
                     Ipv4Network::from_str(&route.dest)?
@@ -401,7 +405,17 @@ impl Handle {
                     request = request.gateway(ip);
                 }
 
-                request.execute().await?;
+                if let Err(rtnetlink::Error::NetlinkError(message)) = request.execute().await {
+                    if Errno::from_i32(message.code.abs()) != Errno::EEXIST {
+                        return Err(anyhow!(
+                            "Failed to add IP v4 route (src: {}, dst: {}, gtw: {},Err: {})",
+                            route.get_source(),
+                            route.get_dest(),
+                            route.get_gateway(),
+                            message
+                        ));
+                    }
+                }
             }
         }
 

src/agent/src/rpc.rs

@@ -148,6 +148,10 @@ impl AgentService {
         };
 
         info!(sl!(), "receive createcontainer, spec: {:?}", &oci);
+        info!(
+            sl!(),
+            "receive createcontainer, storages: {:?}", &req.storages
+        );
 
         // Some devices need some extra processing (the ones invoked with
         // --device for instance), and that's what this call is doing. It
@@ -163,7 +167,13 @@ impl AgentService {
         // After all those storages have been processed, no matter the order
         // here, the agent will rely on rustjail (using the oci.Mounts
         // list) to bind mount all of them inside the container.
-        let m = add_storages(sl!(), req.storages.to_vec(), self.sandbox.clone()).await?;
+        let m = add_storages(
+            sl!(),
+            req.storages.to_vec(),
+            self.sandbox.clone(),
+            Some(req.container_id.clone()),
+        )
+        .await?;
         {
             sandbox = self.sandbox.clone();
             s = sandbox.lock().await;
@@ -433,7 +443,7 @@ impl AgentService {
             .get_container(&cid)
             .ok_or_else(|| anyhow!("Invalid container id"))?;
 
-        let mut p = match ctr.processes.get_mut(&pid) {
+        let p = match ctr.processes.get_mut(&pid) {
             Some(p) => p,
             None => {
                 // Lost race, pick up exit code from channel
@@ -444,7 +454,7 @@ impl AgentService {
 
         // need to close all fd
         // ignore errors for some fd might be closed by stream
-        let _ = cleanup_process(&mut p);
+        p.cleanup_process_stream();
 
         resp.status = p.exit_code;
         // broadcast exit code to all parallel watchers
@@ -573,6 +583,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
     ) -> ttrpc::Result<Empty> {
         trace_rpc_call!(ctx, "remove_container", req);
         is_allowed!(req);
+
         match self.do_remove_container(req).await {
             Err(e) => Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string())),
             Ok(_) => Ok(Empty::new()),
@@ -776,19 +787,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
             )
         })?;
 
-        if p.term_master.is_some() {
-            p.close_stream(StreamType::TermMaster);
-            let _ = unistd::close(p.term_master.unwrap());
-            p.term_master = None;
-        }
-
-        if p.parent_stdin.is_some() {
-            p.close_stream(StreamType::ParentStdin);
-            let _ = unistd::close(p.parent_stdin.unwrap());
-            p.parent_stdin = None;
-        }
-
-        p.notify_term_close();
+        p.close_stdin();
 
         Ok(Empty::new())
     }
@@ -1014,7 +1013,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
                 .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
         }
 
-        match add_storages(sl!(), req.storages.to_vec(), self.sandbox.clone()).await {
+        match add_storages(sl!(), req.storages.to_vec(), self.sandbox.clone(), None).await {
             Ok(m) => {
                 let sandbox = self.sandbox.clone();
                 let mut s = sandbox.lock().await;
@@ -1334,11 +1333,19 @@ fn get_memory_info(block_size: bool, hotplug: bool) -> Result<(u64, bool)> {
     Ok((size, plug))
 }
 
+pub fn have_seccomp() -> bool {
+    if cfg!(feature = "seccomp") {
+        return true;
+    }
+
+    false
+}
+
 fn get_agent_details() -> AgentDetails {
     let mut detail = AgentDetails::new();
 
     detail.set_version(AGENT_VERSION.to_string());
-    detail.set_supports_seccomp(false);
+    detail.set_supports_seccomp(have_seccomp());
     detail.init_daemon = unistd::getpid() == Pid::from_raw(1);
 
     detail.device_handlers = RepeatedField::new();
@@ -1653,11 +1660,6 @@ fn setup_bundle(cid: &str, spec: &mut Spec) -> Result<PathBuf> {
         readonly: spec_root.readonly,
     });
 
-    info!(
-        sl!(),
-        "{:?}",
-        spec.process.as_ref().unwrap().console_size.as_ref()
-    );
     let _ = spec.save(config_path.to_str().unwrap());
 
     let olddir = unistd::getcwd().context("cannot getcwd")?;
@@ -1666,37 +1668,6 @@ fn setup_bundle(cid: &str, spec: &mut Spec) -> Result<PathBuf> {
     Ok(olddir)
 }
 
-fn cleanup_process(p: &mut Process) -> Result<()> {
-    if p.parent_stdin.is_some() {
-        p.close_stream(StreamType::ParentStdin);
-        unistd::close(p.parent_stdin.unwrap())?;
-    }
-
-    if p.parent_stdout.is_some() {
-        p.close_stream(StreamType::ParentStdout);
-        unistd::close(p.parent_stdout.unwrap())?;
-    }
-
-    if p.parent_stderr.is_some() {
-        p.close_stream(StreamType::ParentStderr);
-        unistd::close(p.parent_stderr.unwrap())?;
-    }
-
-    if p.term_master.is_some() {
-        p.close_stream(StreamType::TermMaster);
-        unistd::close(p.term_master.unwrap())?;
-    }
-
-    p.notify_term_close();
-
-    p.parent_stdin = None;
-    p.parent_stdout = None;
-    p.parent_stderr = None;
-    p.term_master = None;
-
-    Ok(())
-}
-
 fn load_kernel_module(module: &protocols::agent::KernelModule) -> Result<()> {
     if module.name.is_empty() {
         return Err(anyhow!("Kernel module name is empty"));
@@ -1749,6 +1720,7 @@ mod tests {
             fd: -1,
             mh: MessageHeader::default(),
             metadata: std::collections::HashMap::new(),
+            timeout_nano: 0,
         }
     }
 

src/agent/src/sandbox.rs

@@ -464,7 +464,10 @@ mod tests {
         baremount(src, dst, "bind", MsFlags::MS_BIND, "", logger)
     }
 
+    use serial_test::serial;
+
     #[tokio::test]
+    #[serial]
     async fn set_sandbox_storage() {
         let logger = slog::Logger::root(slog::Discard, o!());
         let mut s = Sandbox::new(&logger).unwrap();
@@ -499,6 +502,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn remove_sandbox_storage() {
         skip_if_not_root!();
 
@@ -555,6 +559,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn unset_and_remove_sandbox_storage() {
         skip_if_not_root!();
 
@@ -606,6 +611,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn unset_sandbox_storage() {
         let logger = slog::Logger::root(slog::Discard, o!());
         let mut s = Sandbox::new(&logger).unwrap();
@@ -689,6 +695,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn get_container_entry_exist() {
         skip_if_not_root!();
         let logger = slog::Logger::root(slog::Discard, o!());
@@ -702,6 +709,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn get_container_no_entry() {
         let logger = slog::Logger::root(slog::Discard, o!());
         let mut s = Sandbox::new(&logger).unwrap();
@@ -711,6 +719,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn add_and_get_container() {
         skip_if_not_root!();
         let logger = slog::Logger::root(slog::Discard, o!());
@@ -722,6 +731,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn update_shared_pidns() {
         skip_if_not_root!();
         let logger = slog::Logger::root(slog::Discard, o!());
@@ -740,6 +750,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn add_guest_hooks() {
         let logger = slog::Logger::root(slog::Discard, o!());
         let mut s = Sandbox::new(&logger).unwrap();
@@ -763,6 +774,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn test_sandbox_set_destroy() {
         let logger = slog::Logger::root(slog::Discard, o!());
         let mut s = Sandbox::new(&logger).unwrap();

src/agent/src/tracer.rs

@@ -3,61 +3,17 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use crate::config::AgentConfig;
 use anyhow::Result;
 use opentelemetry::sdk::propagation::TraceContextPropagator;
 use opentelemetry::{global, sdk::trace::Config, trace::TracerProvider};
-use serde::Deserialize;
 use slog::{info, o, Logger};
 use std::collections::HashMap;
-use std::error::Error;
-use std::fmt;
-use std::str::FromStr;
 use tracing_opentelemetry::OpenTelemetryLayer;
 use tracing_subscriber::layer::SubscriberExt;
 use tracing_subscriber::Registry;
 use ttrpc::r#async::TtrpcContext;
 
-#[derive(Debug, Deserialize, PartialEq)]
-pub enum TraceType {
-    Disabled,
-    Isolated,
-}
-
-#[derive(Debug)]
-pub struct TraceTypeError {
-    details: String,
-}
-
-impl TraceTypeError {
-    fn new(msg: &str) -> TraceTypeError {
-        TraceTypeError {
-            details: msg.into(),
-        }
-    }
-}
-
-impl Error for TraceTypeError {}
-
-impl fmt::Display for TraceTypeError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}", self.details)
-    }
-}
-
-impl FromStr for TraceType {
-    type Err = TraceTypeError;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        match s {
-            "isolated" => Ok(TraceType::Isolated),
-            "disabled" => Ok(TraceType::Disabled),
-            _ => Err(TraceTypeError::new("invalid trace type")),
-        }
-    }
-}
-
-pub fn setup_tracing(name: &'static str, logger: &Logger, _agent_cfg: &AgentConfig) -> Result<()> {
+pub fn setup_tracing(name: &'static str, logger: &Logger) -> Result<()> {
     let logger = logger.new(o!("subsystem" => "vsock-tracer"));
 
     let exporter = vsock_exporter::Exporter::builder()

src/agent/src/uevent.rs

@@ -97,10 +97,18 @@ impl Uevent {
         })
     }
 
+    #[instrument]
+    async fn process_remove(&self, logger: &Logger, sandbox: &Arc<Mutex<Sandbox>>) {
+        let mut sb = sandbox.lock().await;
+        sb.uevent_map.remove(&self.devpath);
+    }
+
     #[instrument]
     async fn process(&self, logger: &Logger, sandbox: &Arc<Mutex<Sandbox>>) {
         if self.action == U_EVENT_ACTION_ADD {
             return self.process_add(logger, sandbox).await;
+        } else if self.action == U_EVENT_ACTION_REMOVE {
+            return self.process_remove(logger, sandbox).await;
         }
         debug!(*logger, "ignoring event"; "uevent" => format!("{:?}", self));
     }

src/agent/src/watcher.rs

@@ -49,7 +49,7 @@ struct Storage {
     /// the source becomes too large, either in number of files (>16) or total size (>1MB).
     watch: bool,
 
-    /// The list of files to watch from the source mount point and updated in the target one.
+    /// The list of files, directories, symlinks to watch from the source mount point and updated in the target one.
     watched_files: HashMap<PathBuf, SystemTime>,
 }
 
@@ -79,6 +79,20 @@ impl Drop for Storage {
     }
 }
 
+async fn copy(from: impl AsRef<Path>, to: impl AsRef<Path>) -> Result<()> {
+    if fs::symlink_metadata(&from).await?.file_type().is_symlink() {
+        // if source is a symlink, create new symlink with same link source. If
+        // the symlink exists, remove and create new one:
+        if fs::symlink_metadata(&to).await.is_ok() {
+            fs::remove_file(&to).await?;
+        }
+        fs::symlink(fs::read_link(&from).await?, &to).await?;
+    } else {
+        fs::copy(from, to).await?;
+    }
+    Ok(())
+}
+
 impl Storage {
     async fn new(storage: protos::Storage) -> Result<Storage> {
         let entry = Storage {
@@ -93,6 +107,17 @@ impl Storage {
     async fn update_target(&self, logger: &Logger, source_path: impl AsRef<Path>) -> Result<()> {
         let source_file_path = source_path.as_ref();
 
+        // if we are creating a directory: just create it, nothing more to do
+        if source_file_path.symlink_metadata()?.file_type().is_dir() {
+            let dest_file_path = self.make_target_path(&source_file_path)?;
+
+            fs::create_dir_all(&dest_file_path)
+                .await
+                .with_context(|| format!("Unable to mkdir all for {}", dest_file_path.display()))?;
+            return Ok(());
+        }
+
+        // Assume we are dealing with either a file or a symlink now:
         let dest_file_path = if self.source_mount_point.is_file() {
             // Simple file to file copy
             // Assume target mount is a file path
@@ -110,19 +135,13 @@ impl Storage {
             dest_file_path
         };
 
-        debug!(
-            logger,
-            "Copy from {} to {}",
-            source_file_path.display(),
-            dest_file_path.display()
-        );
-        fs::copy(&source_file_path, &dest_file_path)
+        copy(&source_file_path, &dest_file_path)
             .await
             .with_context(|| {
                 format!(
                     "Copy from {} to {} failed",
                     source_file_path.display(),
-                    dest_file_path.display()
+                    dest_file_path.display(),
                 )
             })?;
 
@@ -135,7 +154,7 @@ impl Storage {
         let mut remove_list = Vec::new();
         let mut updated_files: Vec<PathBuf> = Vec::new();
 
-        // Remove deleted files for tracking list
+        // Remove deleted files for tracking list.
         self.watched_files.retain(|st, _| {
             if st.exists() {
                 true
@@ -147,10 +166,19 @@ impl Storage {
 
         // Delete from target
         for path in remove_list {
-            // File has been deleted, remove it from target mount
             let target = self.make_target_path(path)?;
-            debug!(logger, "Removing file from mount: {}", target.display());
-            let _ = fs::remove_file(target).await;
+            // The target may be a directory or a file. If it is a directory that is removed,
+            // we'll remove all files under that directory as well. Because of this, there's a
+            // chance the target (a subdirectory or file under a prior removed target) was already
+            // removed. Make sure we check if the target exists before checking the metadata, and
+            // don't return an error if the remove fails
+            if target.exists() && target.symlink_metadata()?.file_type().is_dir() {
+                debug!(logger, "Removing a directory: {}", target.display());
+                let _ = fs::remove_dir_all(target).await;
+            } else {
+                debug!(logger, "Removing a file: {}", target.display());
+                let _ = fs::remove_file(target).await;
+            }
         }
 
         // Scan new & changed files
@@ -182,15 +210,16 @@ impl Storage {
         let mut size: u64 = 0;
         debug!(logger, "Scanning path: {}", path.display());
 
-        if path.is_file() {
-            let metadata = path
-                .metadata()
-                .with_context(|| format!("Failed to query metadata for: {}", path.display()))?;
+        let metadata = path
+            .symlink_metadata()
+            .with_context(|| format!("Failed to query metadata for: {}", path.display()))?;
 
-            let modified = metadata
-                .modified()
-                .with_context(|| format!("Failed to get modified date for: {}", path.display()))?;
+        let modified = metadata
+            .modified()
+            .with_context(|| format!("Failed to get modified date for: {}", path.display()))?;
 
+        // Treat files and symlinks the same:
+        if path.is_file() || metadata.file_type().is_symlink() {
             size += metadata.len();
 
             // Insert will return old entry if any
@@ -212,6 +241,16 @@ impl Storage {
                 }
             );
         } else {
+            // Handling regular directories - check  to see if this directory is already being tracked, and
+            // track if not:
+            if self
+                .watched_files
+                .insert(path.to_path_buf(), modified)
+                .is_none()
+            {
+                update_list.push(path.to_path_buf());
+            }
+
             // Scan dir recursively
             let mut entries = fs::read_dir(path)
                 .await
@@ -612,7 +651,7 @@ mod tests {
             .unwrap();
 
         // setup storage3: many files, but still watchable
-        for i in 1..MAX_ENTRIES_PER_STORAGE + 1 {
+        for i in 1..MAX_ENTRIES_PER_STORAGE {
             fs::write(src3_path.join(format!("{}.txt", i)), "original").unwrap();
         }
 
@@ -622,6 +661,9 @@ mod tests {
             ..Default::default()
         };
 
+        // delay 20 ms between writes to files in order to ensure filesystem timestamps are unique
+        thread::sleep(Duration::from_millis(20));
+
         entries
             .add(std::iter::once(storage0), &logger)
             .await
@@ -674,7 +716,7 @@ mod tests {
             std::fs::read_dir(entries.0[3].target_mount_point.as_path())
                 .unwrap()
                 .count(),
-            MAX_ENTRIES_PER_STORAGE
+            MAX_ENTRIES_PER_STORAGE - 1
         );
 
         // Add two files to storage 0, verify it is updated without needing to run check:
@@ -692,6 +734,9 @@ mod tests {
             "updated"
         );
 
+        // delay 20 ms between writes to files in order to ensure filesystem timestamps are unique
+        thread::sleep(Duration::from_millis(20));
+
         //
         // Prepare for second check: update mount sources
         //
@@ -744,7 +789,7 @@ mod tests {
             std::fs::read_dir(entries.0[3].target_mount_point.as_path())
                 .unwrap()
                 .count(),
-            MAX_ENTRIES_PER_STORAGE + 1
+            MAX_ENTRIES_PER_STORAGE
         );
 
         // verify that we can remove files as well, but that it isn't observed until check is run
@@ -822,15 +867,20 @@ mod tests {
         fs::remove_file(source_dir.path().join("big.txt")).unwrap();
         fs::remove_file(source_dir.path().join("too-big.txt")).unwrap();
 
-        // Up to 16 files should be okay:
-        for i in 1..MAX_ENTRIES_PER_STORAGE + 1 {
+        assert_eq!(entry.scan(&logger).await.unwrap(), 0);
+
+        // Up to 15 files should be okay (can watch 15 files + 1 directory)
+        for i in 1..MAX_ENTRIES_PER_STORAGE {
             fs::write(source_dir.path().join(format!("{}.txt", i)), "original").unwrap();
         }
 
-        assert_eq!(entry.scan(&logger).await.unwrap(), MAX_ENTRIES_PER_STORAGE);
+        assert_eq!(
+            entry.scan(&logger).await.unwrap(),
+            MAX_ENTRIES_PER_STORAGE - 1
+        );
 
-        // 17 files is too many:
-        fs::write(source_dir.path().join("17.txt"), "updated").unwrap();
+        // 16 files wll be too many:
+        fs::write(source_dir.path().join("16.txt"), "updated").unwrap();
         thread::sleep(Duration::from_secs(1));
 
         // Expect to receive a MountTooManyFiles error
@@ -843,6 +893,180 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn test_copy() {
+        // prepare tmp src/destination
+        let source_dir = tempfile::tempdir().unwrap();
+        let dest_dir = tempfile::tempdir().unwrap();
+
+        // verify copy of a regular file
+        let src_file = source_dir.path().join("file.txt");
+        let dst_file = dest_dir.path().join("file.txt");
+        fs::write(&src_file, "foo").unwrap();
+        copy(&src_file, &dst_file).await.unwrap();
+        // verify destination:
+        assert!(!fs::symlink_metadata(dst_file)
+            .unwrap()
+            .file_type()
+            .is_symlink());
+
+        // verify copy of a symlink
+        let src_symlink_file = source_dir.path().join("symlink_file.txt");
+        let dst_symlink_file = dest_dir.path().join("symlink_file.txt");
+        tokio::fs::symlink(&src_file, &src_symlink_file)
+            .await
+            .unwrap();
+        copy(src_symlink_file, &dst_symlink_file).await.unwrap();
+        // verify destination:
+        assert!(fs::symlink_metadata(&dst_symlink_file)
+            .unwrap()
+            .file_type()
+            .is_symlink());
+        assert_eq!(fs::read_link(&dst_symlink_file).unwrap(), src_file);
+        assert_eq!(fs::read_to_string(&dst_symlink_file).unwrap(), "foo");
+    }
+
+    #[tokio::test]
+    async fn watch_directory_verify_dir_removal() {
+        let source_dir = tempfile::tempdir().unwrap();
+        let dest_dir = tempfile::tempdir().unwrap();
+
+        let mut entry = Storage::new(protos::Storage {
+            source: source_dir.path().display().to_string(),
+            mount_point: dest_dir.path().display().to_string(),
+            ..Default::default()
+        })
+        .await
+        .unwrap();
+        let logger = slog::Logger::root(slog::Discard, o!());
+
+        // create a path we'll remove later
+        fs::create_dir_all(source_dir.path().join("tmp")).unwrap();
+        fs::write(&source_dir.path().join("tmp/test-file"), "foo").unwrap();
+        assert_eq!(entry.scan(&logger).await.unwrap(), 3); // root, ./tmp, test-file
+
+        // Verify expected directory, file:
+        assert_eq!(
+            std::fs::read_dir(dest_dir.path().join("tmp"))
+                .unwrap()
+                .count(),
+            1
+        );
+        assert_eq!(std::fs::read_dir(&dest_dir).unwrap().count(), 1);
+
+        // Now, remove directory, and verify that the directory (and its file) are removed:
+        fs::remove_dir_all(source_dir.path().join("tmp")).unwrap();
+        thread::sleep(Duration::from_secs(1));
+        assert_eq!(entry.scan(&logger).await.unwrap(), 0);
+
+        assert_eq!(std::fs::read_dir(&dest_dir).unwrap().count(), 0);
+
+        assert_eq!(entry.scan(&logger).await.unwrap(), 0);
+    }
+
+    #[tokio::test]
+    async fn watch_directory_with_symlinks() {
+        // Prepare source directory:
+        // ..2021_10_29_03_10_48.161654083/file.txt
+        // ..data -> ..2021_10_29_03_10_48.161654083
+        // file.txt -> ..data/file.txt
+
+        let source_dir = tempfile::tempdir().unwrap();
+        let actual_dir = source_dir.path().join("..2021_10_29_03_10_48.161654083");
+        let actual_file = actual_dir.join("file.txt");
+        let sym_dir = source_dir.path().join("..data");
+        let sym_file = source_dir.path().join("file.txt");
+
+        let relative_to_dir = PathBuf::from("..2021_10_29_03_10_48.161654083");
+
+        // create backing file/path
+        fs::create_dir_all(&actual_dir).unwrap();
+        fs::write(&actual_file, "two").unwrap();
+
+        // create indirection symlink directory that points to the directory that holds the actual file:
+        tokio::fs::symlink(&relative_to_dir, &sym_dir)
+            .await
+            .unwrap();
+
+        // create presented data file symlink:
+        tokio::fs::symlink(PathBuf::from("..data/file.txt"), sym_file)
+            .await
+            .unwrap();
+
+        let dest_dir = tempfile::tempdir().unwrap();
+
+        // delay 20 ms between writes to files in order to ensure filesystem timestamps are unique
+        thread::sleep(Duration::from_millis(20));
+
+        let mut entry = Storage::new(protos::Storage {
+            source: source_dir.path().display().to_string(),
+            mount_point: dest_dir.path().display().to_string(),
+            ..Default::default()
+        })
+        .await
+        .unwrap();
+
+        let logger = slog::Logger::root(slog::Discard, o!());
+
+        assert_eq!(entry.scan(&logger).await.unwrap(), 5);
+
+        // Should copy no files since nothing is changed since last check
+        assert_eq!(entry.scan(&logger).await.unwrap(), 0);
+
+        // now what, what is updated?
+        fs::write(actual_file, "updated").unwrap();
+
+        // delay 20 ms between writes to files in order to ensure filesystem timestamps are unique
+        thread::sleep(Duration::from_millis(20));
+
+        assert_eq!(entry.scan(&logger).await.unwrap(), 1);
+
+        assert_eq!(
+            fs::read_to_string(dest_dir.path().join("file.txt")).unwrap(),
+            "updated"
+        );
+
+        // Verify that resulting file.txt is a symlink:
+        assert!(
+            tokio::fs::symlink_metadata(dest_dir.path().join("file.txt"))
+                .await
+                .unwrap()
+                .file_type()
+                .is_symlink()
+        );
+
+        // Verify that .data directory is a symlink:
+        assert!(tokio::fs::symlink_metadata(&dest_dir.path().join("..data"))
+            .await
+            .unwrap()
+            .file_type()
+            .is_symlink());
+
+        // Should copy no new files after copy happened
+        assert_eq!(entry.scan(&logger).await.unwrap(), 0);
+
+        // Now, simulate configmap update.
+        //  - create a new actual dir/file,
+        //  - update the symlink directory to point to this one
+        //  - remove old dir/file
+        let new_actual_dir = source_dir.path().join("..2021_10_31");
+        let new_actual_file = new_actual_dir.join("file.txt");
+        fs::create_dir_all(&new_actual_dir).unwrap();
+        fs::write(&new_actual_file, "new configmap").unwrap();
+
+        tokio::fs::remove_file(&sym_dir).await.unwrap();
+        tokio::fs::symlink(PathBuf::from("..2021_10_31"), &sym_dir)
+            .await
+            .unwrap();
+        tokio::fs::remove_dir_all(&actual_dir).await.unwrap();
+
+        assert_eq!(entry.scan(&logger).await.unwrap(), 3); // file, file-dir, symlink
+        assert_eq!(
+            fs::read_to_string(dest_dir.path().join("file.txt")).unwrap(),
+            "new configmap"
+        );
+    }
+
     #[tokio::test]
     async fn watch_directory() {
         // Prepare source directory:
@@ -853,6 +1077,13 @@ mod tests {
         fs::create_dir_all(source_dir.path().join("A/B")).unwrap();
         fs::write(source_dir.path().join("A/B/1.txt"), "two").unwrap();
 
+        // A/C is an empty directory
+        let empty_dir = "A/C";
+        fs::create_dir_all(source_dir.path().join(empty_dir)).unwrap();
+
+        // delay 20 ms between writes to files in order to ensure filesystem timestamps are unique
+        thread::sleep(Duration::from_millis(20));
+
         let dest_dir = tempfile::tempdir().unwrap();
 
         let mut entry = Storage::new(protos::Storage {
@@ -865,13 +1096,14 @@ mod tests {
 
         let logger = slog::Logger::root(slog::Discard, o!());
 
-        assert_eq!(entry.scan(&logger).await.unwrap(), 2);
+        assert_eq!(entry.scan(&logger).await.unwrap(), 6);
+
+        // check empty directory
+        assert!(dest_dir.path().join(empty_dir).exists());
 
         // Should copy no files since nothing is changed since last check
         assert_eq!(entry.scan(&logger).await.unwrap(), 0);
 
-        // Should copy 1 file
-        thread::sleep(Duration::from_secs(1));
         fs::write(source_dir.path().join("A/B/1.txt"), "updated").unwrap();
         assert_eq!(entry.scan(&logger).await.unwrap(), 1);
         assert_eq!(
@@ -879,12 +1111,21 @@ mod tests {
             "updated"
         );
 
+        // delay 20 ms between writes to files in order to ensure filesystem timestamps are unique
+        thread::sleep(Duration::from_millis(20));
+
         // Should copy no new files after copy happened
         assert_eq!(entry.scan(&logger).await.unwrap(), 0);
 
         // Update another file
         fs::write(source_dir.path().join("1.txt"), "updated").unwrap();
         assert_eq!(entry.scan(&logger).await.unwrap(), 1);
+
+        // create another empty directory A/C/D
+        let empty_dir = "A/C/D";
+        fs::create_dir_all(source_dir.path().join(empty_dir)).unwrap();
+        assert_eq!(entry.scan(&logger).await.unwrap(), 1);
+        assert!(dest_dir.path().join(empty_dir).exists());
     }
 
     #[tokio::test]
@@ -909,7 +1150,9 @@ mod tests {
 
         assert_eq!(entry.scan(&logger).await.unwrap(), 1);
 
-        thread::sleep(Duration::from_secs(1));
+        // delay 20 ms between writes to files in order to ensure filesystem timestamps are unique
+        thread::sleep(Duration::from_millis(20));
+
         fs::write(&source_file, "two").unwrap();
         assert_eq!(entry.scan(&logger).await.unwrap(), 1);
         assert_eq!(fs::read_to_string(&dest_file).unwrap(), "two");
@@ -935,8 +1178,9 @@ mod tests {
 
         let logger = slog::Logger::root(slog::Discard, o!());
 
-        assert_eq!(entry.scan(&logger).await.unwrap(), 1);
-        assert_eq!(entry.watched_files.len(), 1);
+        // expect the root directory and the file:
+        assert_eq!(entry.scan(&logger).await.unwrap(), 2);
+        assert_eq!(entry.watched_files.len(), 2);
 
         assert!(target_file.exists());
         assert!(entry.watched_files.contains_key(&source_file));
@@ -946,7 +1190,7 @@ mod tests {
 
         assert_eq!(entry.scan(&logger).await.unwrap(), 0);
 
-        assert_eq!(entry.watched_files.len(), 0);
+        assert_eq!(entry.watched_files.len(), 1);
         assert!(!target_file.exists());
     }
 
@@ -979,7 +1223,10 @@ mod tests {
         );
     }
 
+    use serial_test::serial;
+
     #[tokio::test]
+    #[serial]
     async fn create_tmpfs() {
         skip_if_not_root!();
 
@@ -989,11 +1236,14 @@ mod tests {
         watcher.mount(&logger).await.unwrap();
         assert!(is_mounted(WATCH_MOUNT_POINT_PATH).unwrap());
 
+        thread::sleep(Duration::from_millis(20));
+
         watcher.cleanup();
         assert!(!is_mounted(WATCH_MOUNT_POINT_PATH).unwrap());
     }
 
     #[tokio::test]
+    #[serial]
     async fn spawn_thread() {
         skip_if_not_root!();
 
@@ -1023,6 +1273,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn verify_container_cleanup_watching() {
         skip_if_not_root!();
 

src/agent/vsock-exporter/Cargo.toml

@@ -15,6 +15,6 @@ serde = { version = "1.0.126", features = ["derive"] }
 tokio-vsock = "0.3.1"
 bincode = "1.3.3"
 byteorder = "1.4.3"
-slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_info"] }
+slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug"] }
 async-trait = "0.1.50"
 tokio = "1.2.0"

src/runtime/Makefile

@@ -190,6 +190,7 @@ DEFVALIDVHOSTUSERSTOREPATHS := [\"$(DEFVHOSTUSERSTOREPATH)\"]
 DEFFILEMEMBACKEND := ""
 DEFVALIDFILEMEMBACKENDS := [\"$(DEFFILEMEMBACKEND)\"]
 DEFMSIZE9P := 8192
+DEFVFIOMODE := guest-kernel
 
 # Default cgroup model
 DEFSANDBOXCGROUPONLY ?= false
@@ -459,6 +460,7 @@ USER_VARS += DEFENTROPYSOURCE
 USER_VARS += DEFVALIDENTROPYSOURCES
 USER_VARS += DEFSANDBOXCGROUPONLY
 USER_VARS += DEFBINDMOUNTS
+USER_VARS += DEFVFIOMODE
 USER_VARS += FEATURE_SELINUX
 USER_VARS += BUILDFLAGS
 
@@ -531,7 +533,7 @@ $(NETMON_RUNTIME_OUTPUT): $(SOURCES) VERSION
 runtime: $(RUNTIME_OUTPUT) $(CONFIGS)
 .DEFAULT: default
 
-build: default
+build: all
 
 #Install an executable file
 # params:
@@ -573,10 +575,11 @@ $(MONITOR_OUTPUT): $(SOURCES) $(GENERATED_FILES) $(MAKEFILE_LIST) .git-commit
 
 .PHONY: \
 	check \
-	check-go-static \
 	coverage \
 	default \
 	install \
+	lint \
+	pre-commit \
 	show-header \
 	show-summary \
 	show-variables \
@@ -595,8 +598,6 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit
 
 generate-config: $(CONFIGS)
 
-check: check-go-static
-
 test: install-hook go-test
 
 install-hook:
@@ -610,15 +611,34 @@ go-test: $(GENERATED_FILES)
 	go clean -testcache
 	go test -v -mod=vendor ./...
 
-check-go-static:
-	$(QUIET_CHECK)../../ci/go-no-os-exit.sh ./cmd/kata-runtime
-	$(QUIET_CHECK)../../ci/go-no-os-exit.sh ./virtcontainers
+fast-test: $(GENERATED_FILES)
+	go clean -testcache
+	for s in $$(go list ./...); do if ! go test -failfast -v -mod=vendor -p 1 $$s; then break; fi; done
+
+GOLANGCI_LINT_FILE := ../../../tests/.ci/.golangci.yml
+GOLANGCI_LINT_NAME = golangci-lint
+GOLANGCI_LINT_CMD := $(shell command -v $(GOLANGCI_LINT_NAME) 2>/dev/null)
+lint: all
+	if [ -z $(GOLANGCI_LINT_CMD) ] ; \
+	then \
+		echo "ERROR: command $(GOLANGCI_LINT_NAME) not found. Please install it first." >&2; exit 1; \
+	fi
+
+	if [ -f $(GOLANGCI_LINT_FILE) ] ; \
+	then \
+		echo "running $(GOLANGCI_LINT_NAME)..."; \
+		$(GOLANGCI_LINT_NAME) run -c $(GOLANGCI_LINT_FILE) ; \
+	else \
+		echo "ERROR: file $(GOLANGCI_LINT_FILE) not found. You should clone https://github.com/kata-containers/tests to run $(GOLANGCI_LINT_NAME) locally." >&2; exit 1; \
+	fi;
+
+pre-commit: lint fast-test
 
 coverage:
 	go test -v -mod=vendor -covermode=atomic -coverprofile=coverage.txt ./...
 	go tool cover -html=coverage.txt -o coverage.html
 
-install: default install-runtime install-containerd-shim-v2 install-monitor install-netmon
+install: all install-runtime install-containerd-shim-v2 install-monitor install-netmon
 
 install-bin: $(BINLIST)
 	$(QUIET_INST)$(foreach f,$(BINLIST),$(call INSTALL_EXEC,$f,$(BINDIR)))
@@ -675,6 +695,9 @@ show-usage: show-header
 	@printf "\n"
 	@printf "\tbuild                      : standard build (build everything).\n"
 	@printf "\ttest                       : run tests.\n"
+	@printf "\tpre-commit                 : run $(GOLANGCI_LINT_NAME) and tests locally.\n"
+	@printf "\tlint                       : run $(GOLANGCI_LINT_NAME).\n"
+	@printf "\tfast-test                  : run tests with failfast option.\n"
 	@printf "\tcheck                      : run code checks.\n"
 	@printf "\tclean                      : remove built files.\n"
 	@printf "\tcontainerd-shim-v2         : only build containerd shim v2.\n"

src/runtime/cmd/kata-runtime/exit.go

@@ -1,28 +0,0 @@
-// Copyright (c) 2017 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-package main
-
-import "os"
-
-var atexitFuncs []func()
-
-var exitFunc = os.Exit
-
-// atexit registers a function f that will be run when exit is called. The
-// handlers so registered will be called the in reverse order of their
-// registration.
-func atexit(f func()) {
-	atexitFuncs = append(atexitFuncs, f)
-}
-
-// exit calls all atexit handlers before exiting the process with status.
-func exit(status int) {
-	for i := len(atexitFuncs) - 1; i >= 0; i-- {
-		f := atexitFuncs[i]
-		f()
-	}
-	exitFunc(status)
-}

src/runtime/cmd/kata-runtime/exit_test.go

@@ -1,42 +0,0 @@
-// Copyright (c) 2017 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-package main
-
-import (
-	"os"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-var testFoo string
-
-func testFunc() {
-	testFoo = "bar"
-}
-
-func TestExit(t *testing.T) {
-	assert := assert.New(t)
-
-	var testExitStatus int
-	exitFunc = func(status int) {
-		testExitStatus = status
-	}
-
-	defer func() {
-		exitFunc = os.Exit
-	}()
-
-	// test with no atexit functions added.
-	exit(1)
-	assert.Equal(testExitStatus, 1)
-
-	// test with a function added to the atexit list.
-	atexit(testFunc)
-	exit(0)
-	assert.Equal(testFoo, "bar")
-	assert.Equal(testExitStatus, 0)
-}

src/runtime/cmd/kata-runtime/kata-env.go

@@ -29,7 +29,7 @@ import (
 //
 // XXX: Increment for every change to the output format
 // (meaning any change to the EnvInfo type).
-const formatVersion = "1.0.25"
+const formatVersion = "1.0.26"
 
 // MetaInfo stores information on the format of the output itself
 type MetaInfo struct {
@@ -108,6 +108,7 @@ type HypervisorInfo struct {
 	EntropySource        string
 	SharedFS             string
 	VirtioFSDaemon       string
+	SocketPath           string
 	Msize9p              uint32
 	MemorySlots          uint32
 	PCIeRootPort         uint32
@@ -117,10 +118,8 @@ type HypervisorInfo struct {
 
 // AgentInfo stores agent details
 type AgentInfo struct {
-	TraceMode string
-	TraceType string
-	Debug     bool
-	Trace     bool
+	Debug bool
+	Trace bool
 }
 
 // DistroInfo stores host operating system distribution details.
@@ -131,13 +130,14 @@ type DistroInfo struct {
 
 // HostInfo stores host details
 type HostInfo struct {
-	Kernel             string
-	Architecture       string
-	Distro             DistroInfo
-	CPU                CPUInfo
-	Memory             MemoryInfo
-	VMContainerCapable bool
-	SupportVSocks      bool
+	AvailableGuestProtections []string
+	Kernel                    string
+	Architecture              string
+	Distro                    DistroInfo
+	CPU                       CPUInfo
+	Memory                    MemoryInfo
+	VMContainerCapable        bool
+	SupportVSocks             bool
 }
 
 // NetmonInfo stores netmon details
@@ -157,11 +157,11 @@ type EnvInfo struct {
 	Meta       MetaInfo
 	Image      ImageInfo
 	Initrd     InitrdInfo
-	Agent      AgentInfo
 	Hypervisor HypervisorInfo
-	Netmon     NetmonInfo
 	Runtime    RuntimeInfo
+	Netmon     NetmonInfo
 	Host       HostInfo
+	Agent      AgentInfo
 }
 
 func getMetaInfo() MetaInfo {
@@ -242,14 +242,17 @@ func getHostInfo() (HostInfo, error) {
 
 	memoryInfo := getMemoryInfo()
 
+	availableGuestProtection := vc.AvailableGuestProtections()
+
 	host := HostInfo{
-		Kernel:             hostKernelVersion,
-		Architecture:       arch,
-		Distro:             hostDistro,
-		CPU:                hostCPU,
-		Memory:             memoryInfo,
-		VMContainerCapable: hostVMContainerCapable,
-		SupportVSocks:      supportVSocks,
+		Kernel:                    hostKernelVersion,
+		Architecture:              arch,
+		Distro:                    hostDistro,
+		CPU:                       hostCPU,
+		Memory:                    memoryInfo,
+		AvailableGuestProtections: availableGuestProtection,
+		VMContainerCapable:        hostVMContainerCapable,
+		SupportVSocks:             supportVSocks,
 	}
 
 	return host, nil
@@ -303,13 +306,11 @@ func getAgentInfo(config oci.RuntimeConfig) (AgentInfo, error) {
 	agentConfig := config.AgentConfig
 	agent.Debug = agentConfig.Debug
 	agent.Trace = agentConfig.Trace
-	agent.TraceMode = agentConfig.TraceMode
-	agent.TraceType = agentConfig.TraceType
 
 	return agent, nil
 }
 
-func getHypervisorInfo(config oci.RuntimeConfig) HypervisorInfo {
+func getHypervisorInfo(config oci.RuntimeConfig) (HypervisorInfo, error) {
 	hypervisorPath := config.HypervisorConfig.HypervisorPath
 
 	version, err := getCommandVersion(hypervisorPath)
@@ -317,6 +318,19 @@ func getHypervisorInfo(config oci.RuntimeConfig) HypervisorInfo {
 		version = unknown
 	}
 
+	hypervisorType := config.HypervisorType
+
+	socketPath := unknown
+
+	// It is only reliable to make this call as root since a
+	// non-privileged user may not have access to /dev/vhost-vsock.
+	if os.Geteuid() == 0 {
+		socketPath, err = vc.GetHypervisorSocketTemplate(hypervisorType, &config.HypervisorConfig)
+		if err != nil {
+			return HypervisorInfo{}, err
+		}
+	}
+
 	return HypervisorInfo{
 		Debug:             config.HypervisorConfig.Debug,
 		MachineType:       config.HypervisorConfig.HypervisorMachineType,
@@ -331,7 +345,8 @@ func getHypervisorInfo(config oci.RuntimeConfig) HypervisorInfo {
 
 		HotplugVFIOOnRootBus: config.HypervisorConfig.HotplugVFIOOnRootBus,
 		PCIeRootPort:         config.HypervisorConfig.PCIeRootPort,
-	}
+		SocketPath:           socketPath,
+	}, nil
 }
 
 func getEnvInfo(configFile string, config oci.RuntimeConfig) (env EnvInfo, err error) {
@@ -356,7 +371,10 @@ func getEnvInfo(configFile string, config oci.RuntimeConfig) (env EnvInfo, err e
 		return EnvInfo{}, err
 	}
 
-	hypervisor := getHypervisorInfo(config)
+	hypervisor, err := getHypervisorInfo(config)
+	if err != nil {
+		return EnvInfo{}, err
+	}
 
 	image := ImageInfo{
 		Path: config.HypervisorConfig.ImagePath,

src/runtime/cmd/kata-runtime/kata-env_test.go

@@ -184,10 +184,6 @@ func getExpectedAgentDetails(config oci.RuntimeConfig) (AgentInfo, error) {
 	return AgentInfo{
 		Debug: agentConfig.Debug,
 		Trace: agentConfig.Trace,
-
-		// No trace mode/type set by default
-		TraceMode: "",
-		TraceType: "",
 	}, nil
 }
 
@@ -281,7 +277,7 @@ VERSION_ID="%s"
 }
 
 func getExpectedHypervisor(config oci.RuntimeConfig) HypervisorInfo {
-	return HypervisorInfo{
+	info := HypervisorInfo{
 		Version:           testHypervisorVersion,
 		Path:              config.HypervisorConfig.HypervisorPath,
 		MachineType:       config.HypervisorConfig.HypervisorMachineType,
@@ -296,6 +292,16 @@ func getExpectedHypervisor(config oci.RuntimeConfig) HypervisorInfo {
 		HotplugVFIOOnRootBus: config.HypervisorConfig.HotplugVFIOOnRootBus,
 		PCIeRootPort:         config.HypervisorConfig.PCIeRootPort,
 	}
+
+	if os.Geteuid() == 0 {
+		// This assumes the test hypervisor is a non-hybrid-vsock
+		// one (such as QEMU).
+		info.SocketPath = ""
+	} else {
+		info.SocketPath = unknown
+	}
+
+	return info
 }
 
 func getExpectedImage(config oci.RuntimeConfig) ImageInfo {
@@ -677,14 +683,10 @@ func TestEnvGetAgentInfo(t *testing.T) {
 	assert.True(t, agent.Debug)
 
 	agentConfig.Trace = true
-	agentConfig.TraceMode = "traceMode"
-	agentConfig.TraceType = "traceType"
 	config.AgentConfig = agentConfig
 	agent, err = getAgentInfo(config)
 	assert.NoError(t, err)
 	assert.True(t, agent.Trace)
-	assert.Equal(t, agent.TraceMode, "traceMode")
-	assert.Equal(t, agent.TraceType, "traceType")
 }
 
 func testEnvShowTOMLSettings(t *testing.T, tmpdir string, tmpfile *os.File) error {
@@ -1015,12 +1017,58 @@ func TestGetHypervisorInfo(t *testing.T) {
 	_, config, err := makeRuntimeConfig(tmpdir)
 	assert.NoError(err)
 
-	info := getHypervisorInfo(config)
+	info, err := getHypervisorInfo(config)
+	assert.NoError(err)
 	assert.Equal(info.Version, testHypervisorVersion)
 
 	err = os.Remove(config.HypervisorConfig.HypervisorPath)
 	assert.NoError(err)
 
-	info = getHypervisorInfo(config)
+	info, err = getHypervisorInfo(config)
+	assert.NoError(err)
 	assert.Equal(info.Version, unknown)
 }
+
+func TestGetHypervisorInfoSocket(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	_, config, err := makeRuntimeConfig(tmpdir)
+	assert.NoError(err)
+
+	type TestHypervisorDetails struct {
+		hType       vc.HypervisorType
+		hybridVsock bool
+	}
+
+	hypervisors := []TestHypervisorDetails{
+		{vc.AcrnHypervisor, false},
+		{vc.ClhHypervisor, true},
+		{vc.FirecrackerHypervisor, true},
+		{vc.MockHypervisor, false},
+		{vc.QemuHypervisor, false},
+	}
+
+	for i, details := range hypervisors {
+		msg := fmt.Sprintf("hypervisor[%d]: %+v", i, details)
+
+		config.HypervisorType = details.hType
+
+		info, err := getHypervisorInfo(config)
+		assert.NoError(err, msg)
+
+		if os.Geteuid() == 0 {
+			if !details.hybridVsock {
+				assert.Equal(info.SocketPath, "", msg)
+			} else {
+				assert.NotEmpty(info.SocketPath, msg)
+				assert.True(strings.HasPrefix(info.SocketPath, "/"), msg)
+			}
+		} else {
+			assert.Equal(info.SocketPath, unknown, msg)
+		}
+	}
+}

src/runtime/cmd/kata-runtime/main.go

@@ -30,12 +30,11 @@ import (
 	"github.com/urfave/cli"
 )
 
-// specConfig is the name of the file holding the containers configuration
-const specConfig = "config.json"
-
 // arch is the architecture for the running program
 const arch = goruntime.GOARCH
 
+var exitFunc = os.Exit
+
 var usage = fmt.Sprintf(`%s runtime
 
 %s is a command line program for running applications packaged
@@ -331,7 +330,7 @@ func handleShowConfig(context *cli.Context) {
 			fmt.Fprintf(defaultOutputFile, "%s\n", file)
 		}
 
-		exit(0)
+		exitFunc(0)
 	}
 }
 
@@ -451,7 +450,7 @@ func userWantsUsage(context *cli.Context) bool {
 func fatal(err error) {
 	kataLog.Error(err)
 	fmt.Fprintln(defaultErrorFile, err)
-	exit(1)
+	exitFunc(1)
 }
 
 type fatalWriter struct {

src/runtime/cmd/kata-runtime/main_test.go

@@ -13,7 +13,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path"
 	"path/filepath"
 	"regexp"
@@ -22,9 +21,7 @@ import (
 
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/vcmock"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -38,19 +35,12 @@ const (
 	testFileMode    = os.FileMode(0640)
 	testExeFileMode = os.FileMode(0750)
 
-	// small docker image used to create root filesystems from
-	testDockerImage = "busybox"
-
-	testBundle  = "bundle"
 	testConsole = "/dev/pts/999"
 )
 
 var (
 	// package variables set by calling TestMain()
-	testDir       = ""
-	testBundleDir = ""
-	tc            ktu.TestConstraint
-	ctrEngine     = katautils.CtrEngine{}
+	tc ktu.TestConstraint
 )
 
 // testingImpl is a concrete mock RVC implementation used for testing
@@ -79,57 +69,6 @@ func init() {
 	fmt.Printf("INFO: switching to fake virtcontainers implementation for testing\n")
 	vci = testingImpl
 
-	var err error
-
-	fmt.Printf("INFO: creating test directory\n")
-	testDir, err = ioutil.TempDir("", fmt.Sprintf("%s-", katautils.NAME))
-	if err != nil {
-		panic(fmt.Sprintf("ERROR: failed to create test directory: %v", err))
-	}
-	fmt.Printf("INFO: test directory is %v\n", testDir)
-
-	var output string
-	for _, name := range katautils.DockerLikeCtrEngines {
-		fmt.Printf("INFO: checking for container engine: %s\n", name)
-
-		output, err = ctrEngine.Init(name)
-		if err == nil {
-			break
-		}
-	}
-
-	if ctrEngine.Name == "" {
-		panic(fmt.Sprintf("ERROR: Docker-like container engine not accessible to current user: %v (error %v)",
-			output, err))
-	}
-
-	// Do this now to avoid hitting the test timeout value due to
-	// slow network response.
-	fmt.Printf("INFO: ensuring required container image (%v) is available\n", testDockerImage)
-	// Only hit the network if the image doesn't exist locally
-	_, err = ctrEngine.Inspect(testDockerImage)
-	if err == nil {
-		fmt.Printf("INFO: container image %v already exists locally\n", testDockerImage)
-	} else {
-		fmt.Printf("INFO: pulling container image %v\n", testDockerImage)
-		_, err = ctrEngine.Pull(testDockerImage)
-		if err != nil {
-			panic(err)
-		}
-	}
-
-	testBundleDir = filepath.Join(testDir, testBundle)
-	err = os.MkdirAll(testBundleDir, testDirMode)
-	if err != nil {
-		panic(fmt.Sprintf("ERROR: failed to create bundle directory %v: %v", testBundleDir, err))
-	}
-
-	fmt.Printf("INFO: creating OCI bundle in %v for tests to use\n", testBundleDir)
-	err = realMakeOCIBundle(testBundleDir)
-	if err != nil {
-		panic(fmt.Sprintf("ERROR: failed to create OCI bundle: %v", err))
-	}
-
 	tc = ktu.NewTestConstraint(false)
 }
 
@@ -143,8 +82,6 @@ func resetCLIGlobals() {
 func runUnitTests(m *testing.M) {
 	ret := m.Run()
 
-	os.RemoveAll(testDir)
-
 	os.Exit(ret)
 }
 
@@ -156,7 +93,7 @@ func TestMain(m *testing.M) {
 	if path.Base(os.Args[0]) == katautils.NAME+".coverage" ||
 		path.Base(os.Args[0]) == katautils.NAME {
 		main()
-		exit(0)
+		exitFunc(0)
 	}
 
 	runUnitTests(m)
@@ -232,146 +169,6 @@ func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConf
 	}, nil
 }
 
-// createOCIConfig creates an OCI configuration (spec) file in
-// the bundle directory specified (which must exist).
-func createOCIConfig(bundleDir string) error {
-	if bundleDir == "" {
-		return errors.New("BUG: Need bundle directory")
-	}
-
-	if !katautils.FileExists(bundleDir) {
-		return fmt.Errorf("BUG: Bundle directory %s does not exist", bundleDir)
-	}
-
-	var configCmd string
-
-	// Search for a suitable version of runc to use to generate
-	// the OCI config file.
-	for _, cmd := range []string{"docker-runc", "runc"} {
-		fullPath, err := exec.LookPath(cmd)
-		if err == nil {
-			configCmd = fullPath
-			break
-		}
-	}
-
-	if configCmd == "" {
-		return errors.New("Cannot find command to generate OCI config file")
-	}
-
-	_, err := utils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
-	if err != nil {
-		return err
-	}
-
-	specFile := filepath.Join(bundleDir, specConfig)
-	if !katautils.FileExists(specFile) {
-		return fmt.Errorf("generated OCI config file does not exist: %v", specFile)
-	}
-
-	return nil
-}
-
-// createRootfs creates a minimal root filesystem below the specified
-// directory.
-func createRootfs(dir string) error {
-	err := os.MkdirAll(dir, testDirMode)
-	if err != nil {
-		return err
-	}
-
-	container, err := ctrEngine.Create(testDockerImage)
-	if err != nil {
-		return err
-	}
-
-	err = ctrEngine.GetRootfs(container, dir)
-	if err != nil {
-		return err
-	}
-
-	// Clean up
-	_, err = ctrEngine.Rm(container)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// realMakeOCIBundle will create an OCI bundle (including the "config.json"
-// config file) in the directory specified (which must already exist).
-//
-// XXX: Note that tests should *NOT* call this function - they should
-// XXX: instead call makeOCIBundle().
-func realMakeOCIBundle(bundleDir string) error {
-	if bundleDir == "" {
-		return errors.New("BUG: Need bundle directory")
-	}
-
-	if !katautils.FileExists(bundleDir) {
-		return fmt.Errorf("BUG: Bundle directory %v does not exist", bundleDir)
-	}
-
-	err := createOCIConfig(bundleDir)
-	if err != nil {
-		return err
-	}
-
-	// Note the unusual parameter (a directory, not the config
-	// file to parse!)
-	spec, err := compatoci.ParseConfigJSON(bundleDir)
-	if err != nil {
-		return err
-	}
-
-	// Determine the rootfs directory name the OCI config refers to
-	ociRootPath := spec.Root.Path
-
-	rootfsDir := filepath.Join(bundleDir, ociRootPath)
-
-	if strings.HasPrefix(ociRootPath, "/") {
-		return fmt.Errorf("Cannot handle absolute rootfs as bundle must be unique to each test")
-	}
-
-	err = createRootfs(rootfsDir)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// Create an OCI bundle in the specified directory.
-//
-// Note that the directory will be created, but it's parent is expected to exist.
-//
-// This function works by copying the already-created test bundle. Ideally,
-// the bundle would be recreated for each test, but createRootfs() uses
-// docker which on some systems is too slow, resulting in the tests timing
-// out.
-func makeOCIBundle(bundleDir string) error {
-	from := testBundleDir
-	to := bundleDir
-
-	// only the basename of bundleDir needs to exist as bundleDir
-	// will get created by cp(1).
-	base := filepath.Dir(bundleDir)
-
-	for _, dir := range []string{from, base} {
-		if !katautils.FileExists(dir) {
-			return fmt.Errorf("BUG: directory %v should exist", dir)
-		}
-	}
-
-	output, err := utils.RunCommandFull([]string{"cp", "-a", from, to}, true)
-	if err != nil {
-		return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output)
-	}
-
-	return nil
-}
-
 func createCLIContextWithApp(flagSet *flag.FlagSet, app *cli.App) *cli.Context {
 	ctx := cli.NewContext(app, flagSet, nil)
 
@@ -390,69 +187,6 @@ func createCLIContext(flagset *flag.FlagSet) *cli.Context {
 	return createCLIContextWithApp(flagset, cli.NewApp())
 }
 
-func TestMakeOCIBundle(t *testing.T) {
-	assert := assert.New(t)
-
-	tmpdir, err := ioutil.TempDir(testDir, "")
-	assert.NoError(err)
-	defer os.RemoveAll(tmpdir)
-
-	bundleDir := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundleDir)
-	assert.NoError(err)
-
-	specFile := filepath.Join(bundleDir, specConfig)
-	assert.True(katautils.FileExists(specFile))
-}
-
-func TestCreateOCIConfig(t *testing.T) {
-	assert := assert.New(t)
-
-	tmpdir, err := ioutil.TempDir(testDir, "")
-	assert.NoError(err)
-	defer os.RemoveAll(tmpdir)
-
-	bundleDir := filepath.Join(tmpdir, "bundle")
-
-	err = createOCIConfig(bundleDir)
-	// ENOENT
-	assert.Error(err)
-
-	err = os.MkdirAll(bundleDir, testDirMode)
-	assert.NoError(err)
-
-	err = createOCIConfig(bundleDir)
-	assert.NoError(err)
-
-	specFile := filepath.Join(bundleDir, specConfig)
-	assert.True(katautils.FileExists(specFile))
-}
-
-func TestCreateRootfs(t *testing.T) {
-	assert := assert.New(t)
-
-	tmpdir, err := ioutil.TempDir(testDir, "")
-	assert.NoError(err)
-	defer os.RemoveAll(tmpdir)
-
-	rootfsDir := filepath.Join(tmpdir, "rootfs")
-	assert.False(katautils.FileExists(rootfsDir))
-
-	err = createRootfs(rootfsDir)
-	assert.NoError(err)
-
-	// non-comprehensive list of expected directories
-	expectedDirs := []string{"bin", "dev", "etc", "usr", "var"}
-
-	assert.True(katautils.FileExists(rootfsDir))
-
-	for _, dir := range expectedDirs {
-		dirPath := filepath.Join(rootfsDir, dir)
-		assert.True(katautils.FileExists(dirPath))
-	}
-}
-
 func TestMainUserWantsUsage(t *testing.T) {
 	assert := assert.New(t)
 
@@ -525,7 +259,7 @@ func TestMainBeforeSubCommands(t *testing.T) {
 func TestMainBeforeSubCommandsInvalidLogFile(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir(testDir, "")
+	tmpdir, err := ioutil.TempDir("", "katatest")
 	assert.NoError(err)
 	defer os.RemoveAll(tmpdir)
 
@@ -548,7 +282,7 @@ func TestMainBeforeSubCommandsInvalidLogFile(t *testing.T) {
 func TestMainBeforeSubCommandsInvalidLogFormat(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir(testDir, "")
+	tmpdir, err := ioutil.TempDir("", "katatest")
 	assert.NoError(err)
 	defer os.RemoveAll(tmpdir)
 
@@ -577,7 +311,7 @@ func TestMainBeforeSubCommandsInvalidLogFormat(t *testing.T) {
 func TestMainBeforeSubCommandsLoadConfigurationFail(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir(testDir, "")
+	tmpdir, err := ioutil.TempDir("", "katatest")
 	assert.NoError(err)
 	defer os.RemoveAll(tmpdir)
 
@@ -612,7 +346,7 @@ func TestMainBeforeSubCommandsLoadConfigurationFail(t *testing.T) {
 func TestMainBeforeSubCommandsShowCCConfigPaths(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir(testDir, "")
+	tmpdir, err := ioutil.TempDir("", "katatest")
 	assert.NoError(err)
 	defer os.RemoveAll(tmpdir)
 
@@ -676,7 +410,7 @@ func TestMainBeforeSubCommandsShowCCConfigPaths(t *testing.T) {
 func TestMainFatal(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir(testDir, "")
+	tmpdir, err := ioutil.TempDir("", "katatest")
 	assert.NoError(err)
 	defer os.RemoveAll(tmpdir)
 
@@ -900,7 +634,7 @@ func TestMainCreateRuntime(t *testing.T) {
 func TestMainVersionPrinter(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir("", "")
+	tmpdir, err := ioutil.TempDir("", "katatest")
 	assert.NoError(err)
 	defer os.RemoveAll(tmpdir)
 

src/runtime/cmd/kata-runtime/utils_test.go

@@ -18,7 +18,7 @@ import (
 )
 
 func TestFileExists(t *testing.T) {
-	dir, err := ioutil.TempDir(testDir, "")
+	dir, err := ioutil.TempDir("", "katatest")
 	if err != nil {
 		t.Fatal(err)
 	}

src/runtime/config/configuration-acrn.toml.in

@@ -124,24 +124,17 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_ACRN@"
 
 # Enable agent tracing.
 #
-# If enabled, the default trace mode is "dynamic" and the
-# default trace type is "isolated". The trace mode and type are set
-# explicity with the `trace_type=` and `trace_mode=` options.
+# If enabled, the agent will generate OpenTelemetry trace spans.
 #
 # Notes:
 #
-# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly
-#   setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing`
-#   will NOT activate agent tracing.
-#
-# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for
-#   full details.
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
 #
 # (default: disabled)
 #enable_tracing = true
-#
-#trace_mode = "dynamic"
-#trace_type = "isolated"
 
 # Enable debug console.
 

src/runtime/config/configuration-clh.toml.in

@@ -105,8 +105,7 @@ virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@
 virtio_fs_cache = "@DEFVIRTIOFSCACHE@"
 
 # Block storage driver to be used for the hypervisor in case the container
-# rootfs is backed by a block device. This is virtio-scsi, virtio-blk
-# or nvdimm.
+# rootfs is backed by a block device. This is virtio-blk.
 block_device_driver = "virtio-blk"
 
 # Enable huge pages for VM RAM, default false
@@ -114,6 +113,9 @@ block_device_driver = "virtio-blk"
 # being allocated using huge pages.
 #enable_hugepages = true
 
+# Disable the 'seccomp' feature from Cloud Hypervisor, default false
+# disable_seccomp = true
+
 # This option changes the default hypervisor and kernel parameters
 # to enable debug output where available.
 #
@@ -144,24 +146,17 @@ block_device_driver = "virtio-blk"
 
 # Enable agent tracing.
 #
-# If enabled, the default trace mode is "dynamic" and the
-# default trace type is "isolated". The trace mode and type are set
-# explicity with the `trace_type=` and `trace_mode=` options.
+# If enabled, the agent will generate OpenTelemetry trace spans.
 #
 # Notes:
 #
-# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly
-#   setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing`
-#   will NOT activate agent tracing.
-#
-# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for
-#   full details.
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
 #
 # (default: disabled)
 #enable_tracing = true
-#
-#trace_mode = "dynamic"
-#trace_type = "isolated"
 
 # Enable debug console.
 
@@ -267,6 +262,27 @@ sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
 # These will not be exposed to the container workloads, and are only provided for potential guest services.
 sandbox_bind_mounts=@DEFBINDMOUNTS@
 
+# VFIO Mode
+# Determines how VFIO devices should be be presented to the container.
+# Options:
+#
+#  - vfio
+#    Matches behaviour of OCI runtimes (e.g. runc) as much as
+#    possible.  VFIO devices will appear in the container as VFIO
+#    character devices under /dev/vfio.  The exact names may differ
+#    from the host (they need to match the VM's IOMMU group numbers
+#    rather than the host's)
+#
+#  - guest-kernel
+#    This is a Kata-specific behaviour that's useful in certain cases.
+#    The VFIO device is managed by whatever driver in the VM kernel
+#    claims it.  This means it will appear as one or more device nodes
+#    or network interfaces depending on the nature of the device.
+#    Using this mode requires specially built workloads that know how
+#    to locate the relevant device interfaces within the VM.
+#
+vfio_mode="@DEFVFIOMODE@"
+
 # Enabled experimental feature list, format: ["a", "b"].
 # Experimental features are features not stable enough for production,
 # they may break compatibility, and are prepared for a big version bump.

src/runtime/config/configuration-fc.toml.in

@@ -246,24 +246,17 @@ valid_entropy_sources = @DEFVALIDENTROPYSOURCES@
 
 # Enable agent tracing.
 #
-# If enabled, the default trace mode is "dynamic" and the
-# default trace type is "isolated". The trace mode and type are set
-# explicity with the `trace_type=` and `trace_mode=` options.
+# If enabled, the agent will generate OpenTelemetry trace spans.
 #
 # Notes:
 #
-# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly
-#   setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing`
-#   will NOT activate agent tracing.
-#
-# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for
-#   full details.
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
 #
 # (default: disabled)
 #enable_tracing = true
-#
-#trace_mode = "dynamic"
-#trace_type = "isolated"
 
 # Comma separated list of kernel modules and their parameters.
 # These modules will be loaded in the guest kernel using modprobe(8).

src/runtime/config/configuration-qemu.toml.in

@@ -422,24 +422,17 @@ valid_entropy_sources = @DEFVALIDENTROPYSOURCES@
 
 # Enable agent tracing.
 #
-# If enabled, the default trace mode is "dynamic" and the
-# default trace type is "isolated". The trace mode and type are set
-# explicity with the `trace_type=` and `trace_mode=` options.
+# If enabled, the agent will generate OpenTelemetry trace spans.
 #
 # Notes:
 #
-# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly
-#   setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing`
-#   will NOT activate agent tracing.
-#
-# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for
-#   full details.
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
 #
 # (default: disabled)
 #enable_tracing = true
-#
-#trace_mode = "dynamic"
-#trace_type = "isolated"
 
 # Comma separated list of kernel modules and their parameters.
 # These modules will be loaded in the guest kernel using modprobe(8).
@@ -550,6 +543,27 @@ sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
 # These will not be exposed to the container workloads, and are only provided for potential guest services.
 sandbox_bind_mounts=@DEFBINDMOUNTS@
 
+# VFIO Mode
+# Determines how VFIO devices should be be presented to the container.
+# Options:
+#
+#  - vfio
+#    Matches behaviour of OCI runtimes (e.g. runc) as much as
+#    possible.  VFIO devices will appear in the container as VFIO
+#    character devices under /dev/vfio.  The exact names may differ
+#    from the host (they need to match the VM's IOMMU group numbers
+#    rather than the host's)
+#
+#  - guest-kernel
+#    This is a Kata-specific behaviour that's useful in certain cases.
+#    The VFIO device is managed by whatever driver in the VM kernel
+#    claims it.  This means it will appear as one or more device nodes
+#    or network interfaces depending on the nature of the device.
+#    Using this mode requires specially built workloads that know how
+#    to locate the relevant device interfaces within the VM.
+#
+vfio_mode="@DEFVFIOMODE@"
+
 # Enabled experimental feature list, format: ["a", "b"].
 # Experimental features are features not stable enough for production,
 # they may break compatibility, and are prepared for a big version bump.

src/runtime/data/completions/bash/kata-runtime

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Copyright (c) 2018 Intel Corporation
 #
@@ -53,39 +53,6 @@ _kata_subcmd_seen()
     echo ""
 }
 
-# Return 0 if the specified sub-command requires the name of an
-# *existing* container, else 1.
-_kata_subcmd_needs_existing_container()
-{
-    local subcmd="$1"
-    local cmd
-
-    for cmd in \
-	    'kata-check' \
-	    'kata-env' \
-	    'check' \
-	    'env' \
-	    'create' \
-	    'help' \
-	    'list' \
-	    'version'; do
-        [ "$cmd" = "$subcmd" ] && return 1
-    done
-
-    return 0
-}
-
-# Returns a list of container names
-_kata_get_containers()
-{
-    # Commands that manipulate containers need root privileges.
-    # If the user isn't running as root, don't attempt to obtain a list
-    # as it will result in an error.
-    [ $(id -u) -eq 0 ] || return
-
-    "$_kataruntime" list --quiet
-}
-
 _kata_bash_autocomplete() {
     COMPREPLY=()
 
@@ -104,32 +71,22 @@ _kata_bash_autocomplete() {
     local subcmd_seen=$(_kata_subcmd_seen)
 
     if [ -n "$subcmd_seen" ]; then
-	    _kata_subcmd_needs_existing_container "$subcmd_seen"
-	    local container_cmd=$?
-
-            if [ -n "$cur" ]; then
-                    # Complete with local options and maybe container names
-                    opts=$(_kata_get_subcmd_options "$subcmd_seen")
-                    [ $container_cmd -eq 0 ] && opts="$opts $(_kata_get_containers)"
-            elif [[ "${cur}" == -* ]]; then
-                    # Complete with local options
-                    opts=$(_kata_get_subcmd_options "$subcmd_seen")
-            else
-                    # Potentially complete with container names
-                    [ $container_cmd -eq 0 ] && opts="$(_kata_get_containers)"
-            fi
+        if [ -n "$cur" ]; then
+                # Complete with local options
+                opts=$(_kata_get_subcmd_options "$subcmd_seen")
+        fi
     else
-            if [ -n "$cur" ]; then
-                    # Complete with global options and subcmds
-                    opts="$opts $(_kata_get_global_options)"
-                    opts="$opts $(_kata_get_subcmds)"
-            elif [[ "${cur}" == -* ]]; then
-                    # Complete with global options
-                    opts=$(_kata_get_global_options)
-            else
-                    # Complete with subcmds
-                    opts=$(_kata_get_subcmds)
-            fi
+        if [ -n "$cur" ]; then
+                # Complete with global options and subcmds
+                opts="$opts $(_kata_get_global_options)"
+                opts="$opts $(_kata_get_subcmds)"
+        elif [[ "${cur}" == -* ]]; then
+                # Complete with global options
+                opts=$(_kata_get_global_options)
+        else
+                # Complete with subcmds
+                opts=$(_kata_get_subcmds)
+        fi
     fi
 
     [ -n "$opts" ] && COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )

src/runtime/go.mod

@@ -11,9 +11,10 @@ require (
 	github.com/containerd/containerd v1.5.7
 	github.com/containerd/cri-containerd v1.11.1-0.20190125013620-4dd6735020f5
 	github.com/containerd/fifo v1.0.0
-	github.com/containerd/ttrpc v1.0.2
+	github.com/containerd/ttrpc v1.1.0
 	github.com/containerd/typeurl v1.0.2
 	github.com/containernetworking/plugins v0.9.1
+	github.com/coreos/go-systemd/v22 v22.3.2
 	github.com/cri-o/cri-o v1.0.0-rc2.0.20170928185954-3394b3b2d6af
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/go-ini/ini v1.28.2
@@ -22,6 +23,7 @@ require (
 	github.com/go-openapi/strfmt v0.18.0
 	github.com/go-openapi/swag v0.19.5
 	github.com/go-openapi/validate v0.18.0
+	github.com/godbus/dbus/v5 v5.0.4
 	github.com/gogo/protobuf v1.3.2
 	github.com/hashicorp/go-multierror v1.0.0
 	github.com/intel-go/cpuid v0.0.0-20210602155658-5747e5cec0d9
@@ -38,23 +40,28 @@ require (
 	github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8
 	github.com/sirupsen/logrus v1.8.1
 	github.com/smartystreets/goconvey v1.6.4 // indirect
-	github.com/stretchr/testify v1.6.1
+	github.com/stretchr/testify v1.7.0
 	github.com/urfave/cli v1.22.2
 	github.com/vishvananda/netlink v1.1.1-0.20210924202909-187053b97868
 	github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae
-	go.opentelemetry.io/otel v0.15.0
-	go.opentelemetry.io/otel/exporters/trace/jaeger v0.15.0
-	go.opentelemetry.io/otel/sdk v0.15.0
+	go.opencensus.io v0.23.0 // indirect
+	go.opentelemetry.io/otel v1.0.0
+	go.opentelemetry.io/otel/exporters/jaeger v1.0.0
+	go.opentelemetry.io/otel/sdk v1.0.0
+	go.opentelemetry.io/otel/trace v1.0.0
 	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110
-	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43
+	golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93
 	golang.org/x/sys v0.0.0-20210426230700-d19ff857e887
-	google.golang.org/grpc v1.33.2
+	golang.org/x/text v0.3.5 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb // indirect
+	google.golang.org/grpc v1.36.0
 	k8s.io/apimachinery v0.20.6
 	k8s.io/cri-api v0.20.6
 )
 
 replace (
-	github.com/containerd/containerd => github.com/containerd/containerd v1.5.7
+	github.com/containerd/containerd => github.com/containerd/containerd v1.5.8
 	github.com/opencontainers/runc => github.com/opencontainers/runc v1.0.1
 	github.com/uber-go/atomic => go.uber.org/atomic v1.5.1
 	google.golang.org/genproto => google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8

src/runtime/go.sum

@@ -45,7 +45,6 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/DataDog/sketches-go v0.0.1/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60=
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
@@ -53,8 +52,8 @@ github.com/Microsoft/go-winio v0.4.17 h1:iT12IBVClFevaf8PuVyi3UmZOVh4OqnaLxDTW2O
 github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
 github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600=
-github.com/Microsoft/hcsshim v0.8.21 h1:btRfUDThBE5IKcvI8O8jOiIkujUsAMBSRsYDYmEi6oM=
-github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4=
+github.com/Microsoft/hcsshim v0.8.23 h1:47MSwtKGXet80aIn+7h4YI6fwPmwIghAnsx2aOUrG2M=
+github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
@@ -67,13 +66,10 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
-github.com/apache/thrift v0.13.0 h1:5hryIiq9gtn+MiLVn0wP37kb/uTeRZgN08WoCsAhIhI=
-github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -86,6 +82,7 @@ github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
+github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@@ -97,9 +94,11 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs=
 github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
+github.com/cilium/ebpf v0.6.2 h1:iHsfF/t4aW4heW2YKfeHrVPGdtYTL4C4KocpM8KTSnI=
 github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=
 github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss=
@@ -109,8 +108,8 @@ github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f2
 github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw=
 github.com/containerd/console v1.0.2 h1:Pi6D+aZXM+oUw1czuKgH5IJ+y0jhYcwBJfx5/Ghn9dE=
 github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ=
-github.com/containerd/containerd v1.5.7 h1:rQyoYtj4KddB3bxG6SAqd4+08gePNyJjRqvOIfV3rkM=
-github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c=
+github.com/containerd/containerd v1.5.8 h1:NmkCC1/QxyZFBny8JogwLpOy2f+VEbO/f6bV2Mqtwuw=
+github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s=
 github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ=
 github.com/containerd/continuity v0.1.0 h1:UFRRY5JemiAhPZrr/uE0n8fMTLcZsUvySPr1+D7pgr8=
 github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM=
@@ -124,8 +123,9 @@ github.com/containerd/go-runc v1.0.0 h1:oU+lLv1ULm5taqgV/CJivypVODI4SUz1znWjv3nN
 github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok=
 github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms=
 github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
-github.com/containerd/ttrpc v1.0.2 h1:2/O3oTZN36q2xRolk0a2WWGgh7/Vf/liElg5hFYLX9U=
 github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y=
+github.com/containerd/ttrpc v1.1.0 h1:GbtyLRxb0gOLR0TYQWt3O6B0NvT8tMdorEHqIQo/lWI=
+github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ=
 github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg=
 github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY=
 github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s=
@@ -156,6 +156,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/cri-o/cri-o v1.0.0-rc2.0.20170928185954-3394b3b2d6af h1:H6nLV96F1LkWizYLQtrMtqJBrlJxnpjgisHsTsOS2HU=
 github.com/cri-o/cri-o v1.0.0-rc2.0.20170928185954-3394b3b2d6af/go.mod h1:POmDVglzQ2jWTlL9ZCfZ8d1QjLhmk0oB36O8T0oG75Y=
+github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg=
 github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
 github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
 github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
@@ -181,10 +182,12 @@ github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
+github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
@@ -297,9 +300,11 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -482,6 +487,7 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8 h1:2c1EFnZHIPCW8qKWgHMH/fX2PkSabFc5mrVzfUNdg5U=
 github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
+github.com/seccomp/libseccomp-golang v0.9.1 h1:NJjM5DNFOs0s3kYE1WUOr6G8V97sdt46rlXTMfXGWBo=
 github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
@@ -519,8 +525,10 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -552,14 +560,17 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opentelemetry.io/otel v0.15.0 h1:CZFy2lPhxd4HlhZnYK8gRyDotksO3Ip9rBweY1vVYJw=
-go.opentelemetry.io/otel v0.15.0/go.mod h1:e4GKElweB8W2gWUqbghw0B8t5MCTccc9212eNHnOHwA=
-go.opentelemetry.io/otel/exporters/trace/jaeger v0.15.0 h1:OZY+lMaUJiJ6ls1dDtqKhSPJWEVNytLuRUrzuR852jc=
-go.opentelemetry.io/otel/exporters/trace/jaeger v0.15.0/go.mod h1:4DeFMzRzr2l5o/Lzh4GfOYEH+mnf7ZrEGDzzJEP6ta8=
-go.opentelemetry.io/otel/sdk v0.15.0 h1:Hf2dl1Ad9Hn03qjcAuAq51GP5Pv1SV5puIkS2nRhdd8=
-go.opentelemetry.io/otel/sdk v0.15.0/go.mod h1:Qudkwgq81OcA9GYVlbyZ62wkLieeS1eWxIL0ufxgwoc=
+go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.0.0 h1:qTTn6x71GVBvoafHK/yaRUmFzI4LcONZD0/kXxl5PHI=
+go.opentelemetry.io/otel v1.0.0/go.mod h1:AjRVh9A5/5DE7S+mZtTR6t8vpKKryam+0lREnfmS4cg=
+go.opentelemetry.io/otel/exporters/jaeger v1.0.0 h1:cLhx8llHw02h5JTqGqaRbYn+QVKHmrzD9vEbKnSPk5U=
+go.opentelemetry.io/otel/exporters/jaeger v1.0.0/go.mod h1:q10N1AolE1JjqKrFJK2tYw0iZpmX+HBaXBtuCzRnBGQ=
+go.opentelemetry.io/otel/sdk v1.0.0 h1:BNPMYUONPNbLneMttKSjQhOTlFLOD9U22HNG1KrIN2Y=
+go.opentelemetry.io/otel/sdk v1.0.0/go.mod h1:PCrDHlSy5x1kjezSdL37PhbFUMjrsLRshJ2zCzeXwbM=
+go.opentelemetry.io/otel/trace v1.0.0 h1:TSBr8GTEtKevYMG/2d21M989r5WJYVimhTHBKVEZuh4=
+go.opentelemetry.io/otel/trace v1.0.0/go.mod h1:PXTWqayeFUlJV1YDNhsJYB184+IvAH814St6o6ajzIs=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
@@ -648,8 +659,8 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 h1:ld7aEMNHoBnnDAX15v1T6z31v8HwR2A9FYOuAhWqkwc=
-golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93 h1:alLDrZkL34Y2bnGHfvC1CYBRBXCXgx8AC2vY4MRtYX4=
+golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -659,7 +670,6 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -707,7 +717,6 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -718,6 +727,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 h1:dXfMednGJh/SUUFjTLsWJz3P+TQt9qnR11GgeI3vWKs=
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -726,8 +736,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -778,7 +789,6 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -801,15 +811,14 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
 google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
 google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/api v0.32.0 h1:Le77IccnTqEa8ryp9wIpX5W3zYm7Gf9LhOp9PHcwFts=
-google.golang.org/api v0.32.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
@@ -825,9 +834,9 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa
 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
 google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.33.2 h1:EQyQC3sa8M+p6Ulc8yy9SWSS2GVwyRc83gAbG8lrl4o=
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.36.0 h1:o1bcQ6imQMIOpdrO3SWf2z5RV72WbDwdXuK0MDlc8As=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -837,8 +846,9 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

src/runtime/pkg/containerd-shim-v2/create.go

@@ -311,24 +311,26 @@ func configureNonRootHypervisor(runtimeConfig *oci.RuntimeConfig) error {
 	runtimeConfig.HypervisorConfig.Gid = uint32(gid)
 
 	userTmpDir := path.Join("/run/user/", fmt.Sprint(uid))
-	dir, err := os.Stat(userTmpDir)
-	if os.IsNotExist(err) {
-		if err = os.Mkdir(userTmpDir, vc.DirMode); err != nil {
-			return err
-		}
-		defer func() {
-			if err != nil {
-				if err = os.RemoveAll(userTmpDir); err != nil {
-					shimLog.WithField("userTmpDir", userTmpDir).WithError(err).Warn("failed to remove userTmpDir")
-				}
-			}
-		}()
-		if err = syscall.Chown(userTmpDir, uid, gid); err != nil {
+	_, err = os.Stat(userTmpDir)
+	// Clean up the directory created by the previous run
+	if !os.IsNotExist(err) {
+		if err = os.RemoveAll(userTmpDir); err != nil {
 			return err
 		}
 	}
-	if dir != nil && !dir.IsDir() {
-		return fmt.Errorf("%s is expected to be a directory", userTmpDir)
+
+	if err = os.Mkdir(userTmpDir, vc.DirMode); err != nil {
+		return err
+	}
+	defer func() {
+		if err != nil {
+			if err = os.RemoveAll(userTmpDir); err != nil {
+				shimLog.WithField("userTmpDir", userTmpDir).WithError(err).Warn("failed to remove userTmpDir")
+			}
+		}
+	}()
+	if err = syscall.Chown(userTmpDir, uid, gid); err != nil {
+		return err
 	}
 
 	if err := os.Setenv("XDG_RUNTIME_DIR", userTmpDir); err != nil {

src/runtime/pkg/containerd-shim-v2/create_test.go

@@ -12,7 +12,6 @@ import (
 	"io/ioutil"
 	"os"
 	"path"
-	"path/filepath"
 	"testing"
 
 	"github.com/containerd/containerd/namespaces"
@@ -23,7 +22,6 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
@@ -52,21 +50,12 @@ func TestCreateSandboxSuccess(t *testing.T) {
 		testingImpl.CreateSandboxFunc = nil
 	}()
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
-	defer os.RemoveAll(tmpdir)
+	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
+	// defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(katautils.FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -82,7 +71,7 @@ func TestCreateSandboxSuccess(t *testing.T) {
 	}
 
 	// Rewrite the file
-	err = writeOCIConfigFile(spec, ociConfigFile)
+	err = ktu.WriteOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
 	s := &service{
@@ -110,25 +99,16 @@ func TestCreateSandboxFail(t *testing.T) {
 
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(katautils.FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
-	err = writeOCIConfigFile(spec, ociConfigFile)
+	err = ktu.WriteOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
 	s := &service{
@@ -157,21 +137,12 @@ func TestCreateSandboxConfigFail(t *testing.T) {
 
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, _ := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(katautils.FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -216,21 +187,12 @@ func TestCreateContainerSuccess(t *testing.T) {
 		},
 	}
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(katautils.FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -240,7 +202,7 @@ func TestCreateContainerSuccess(t *testing.T) {
 	spec.Annotations[testSandboxIDAnnotation] = testSandboxID
 
 	// rewrite file
-	err = writeOCIConfigFile(spec, ociConfigFile)
+	err = ktu.WriteOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
 	s := &service{
@@ -265,21 +227,12 @@ func TestCreateContainerSuccess(t *testing.T) {
 func TestCreateContainerFail(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(katautils.FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -287,7 +240,7 @@ func TestCreateContainerFail(t *testing.T) {
 	spec.Annotations[testContainerTypeAnnotation] = testContainerTypeContainer
 	spec.Annotations[testSandboxIDAnnotation] = testSandboxID
 
-	err = writeOCIConfigFile(spec, ociConfigFile)
+	err = ktu.WriteOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
 	// doesn't create sandbox first
@@ -325,21 +278,12 @@ func TestCreateContainerConfigFail(t *testing.T) {
 		sandbox.CreateContainerFunc = nil
 	}()
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(katautils.FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -348,7 +292,7 @@ func TestCreateContainerConfigFail(t *testing.T) {
 	spec.Annotations[testContainerTypeAnnotation] = "errorType"
 	spec.Annotations[testSandboxIDAnnotation] = testSandboxID
 
-	err = writeOCIConfigFile(spec, ociConfigFile)
+	err = ktu.WriteOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
 	s := &service{

src/runtime/pkg/containerd-shim-v2/delete_test.go

@@ -7,14 +7,13 @@
 package containerdshim
 
 import (
-	"io/ioutil"
 	"os"
-	"path/filepath"
 	"testing"
 
 	taskAPI "github.com/containerd/containerd/runtime/v2/task"
 	"github.com/stretchr/testify/assert"
 
+	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/vcmock"
 )
@@ -26,7 +25,7 @@ func TestDeleteContainerSuccessAndFail(t *testing.T) {
 		MockID: testSandboxID,
 	}
 
-	rootPath, bundlePath := testConfigSetup(t)
+	rootPath, bundlePath, _ := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(rootPath)
 	_, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
@@ -43,19 +42,3 @@ func TestDeleteContainerSuccessAndFail(t *testing.T) {
 	s.containers[testContainerID], err = newContainer(s, reqCreate, "", nil, true)
 	assert.NoError(err)
 }
-
-func testConfigSetup(t *testing.T) (rootPath string, bundlePath string) {
-	assert := assert.New(t)
-
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
-
-	bundlePath = filepath.Join(tmpdir, "bundle")
-	err = os.MkdirAll(bundlePath, testDirMode)
-	assert.NoError(err)
-
-	err = createOCIConfig(bundlePath)
-	assert.NoError(err)
-
-	return tmpdir, bundlePath
-}

src/runtime/pkg/containerd-shim-v2/service.go

@@ -935,6 +935,10 @@ func (s *service) Shutdown(ctx context.Context, r *taskAPI.ShutdownRequest) (_ *
 	s.mu.Lock()
 	if len(s.containers) != 0 {
 		s.mu.Unlock()
+
+		span.End()
+		katatrace.StopTracing(s.rootCtx)
+
 		return empty, nil
 	}
 	s.mu.Unlock()

src/runtime/pkg/containerd-shim-v2/service_test.go

@@ -8,9 +8,7 @@ package containerdshim
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"os"
-	"path/filepath"
 	"strings"
 	"testing"
 
@@ -43,13 +41,9 @@ func TestServiceCreate(t *testing.T) {
 
 	assert := assert.New(t)
 
-	tmpdir, _ := ioutil.TempDir("", "")
+	tmpdir, bundleDir, _ := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
-	bundleDir := filepath.Join(tmpdir, "bundle")
-	err := makeOCIBundle(bundleDir)
-	assert.NoError(err)
-
 	ctx := context.Background()
 
 	s, err := newService("foo")

src/runtime/pkg/containerd-shim-v2/stream_test.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"io"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 	"syscall"
 	"testing"
@@ -24,6 +25,11 @@ func TestNewTtyIOFifoReopen(t *testing.T) {
 	var tty *ttyIO
 	assert := assert.New(t)
 	ctx := context.TODO()
+
+	testDir, err := ioutil.TempDir("", "kata-")
+	assert.NoError(err)
+	defer os.RemoveAll(testDir)
+
 	fifoPath, err := ioutil.TempDir(testDir, "fifo-path-")
 	assert.NoError(err)
 	stdout := filepath.Join(fifoPath, "stdout")
@@ -91,8 +97,6 @@ func TestNewTtyIOFifoReopen(t *testing.T) {
 }
 
 func TestIoCopy(t *testing.T) {
-	t.Skip("TestIoCopy is failing randonly, see https://github.com/kata-containers/kata-containers/issues/2042")
-
 	assert := assert.New(t)
 	ctx := context.TODO()
 
@@ -100,6 +104,10 @@ func TestIoCopy(t *testing.T) {
 	testBytes2 := []byte("Test2")
 	testBytes3 := []byte("Test3")
 
+	testDir, err := ioutil.TempDir("", "kata-")
+	assert.NoError(err)
+	defer os.RemoveAll(testDir)
+
 	fifoPath, err := ioutil.TempDir(testDir, "fifo-path-")
 	assert.NoError(err)
 	dstStdoutPath := filepath.Join(fifoPath, "dststdout")
@@ -229,6 +237,18 @@ func TestIoCopy(t *testing.T) {
 			}
 		}
 
+		// check everything works without closed pipes
+		checkFifoWrite(firstW, testBytes1, first)
+		checkFifoRead(firstR, testBytes1, first)
+
+		checkFifoWrite(secondW, testBytes2, second)
+		checkFifoRead(secondR, testBytes2, second)
+
+		if thirdW != nil {
+			checkFifoWrite(thirdW, testBytes3, third)
+			checkFifoRead(thirdR, testBytes3, third)
+		}
+
 		// write to each pipe, and close them immediately
 		// the ioCopy function should copy the data, then stop the corresponding thread
 		checkFifoWrite(firstW, testBytes1, first)

src/runtime/pkg/containerd-shim-v2/utils_test.go

@@ -7,46 +7,28 @@
 package containerdshim
 
 import (
-	"encoding/json"
-	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
-	sysExec "os/exec"
 	"path"
-	"path/filepath"
-	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 
-	"github.com/opencontainers/runtime-spec/specs-go"
-
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/vcmock"
+	"github.com/pkg/errors"
 )
 
 const (
-	// specConf is the name of the file holding the containers configuration
-	specConf = "config.json"
-
 	TestID = "container_test"
 
-	testDirMode  = os.FileMode(0750)
 	testFileMode = os.FileMode(0640)
-	// testExeFileMode         = os.FileMode(0750)
-
-	// small docker image used to create root filesystems from
-	testDockerImage = "busybox"
 
 	testSandboxID   = "777-77-77777777"
 	testContainerID = "42"
-	testBundle      = "bundle"
 	testConsole     = "/dev/pts/888"
 
 	testContainerTypeAnnotation = "io.kubernetes.cri.container-type"
@@ -57,10 +39,7 @@ const (
 
 var (
 	// package variables set by calling TestMain()
-	testDir       = ""
-	testBundleDir = ""
-	tc            ktu.TestConstraint
-	ctrEngine     = katautils.CtrEngine{}
+	tc ktu.TestConstraint
 )
 
 // testingImpl is a concrete mock RVC implementation used for testing
@@ -73,57 +52,6 @@ func init() {
 	fmt.Printf("INFO: switching to fake virtcontainers implementation for testing\n")
 	vci = testingImpl
 
-	var err error
-
-	fmt.Printf("INFO: creating test directory\n")
-	testDir, err = ioutil.TempDir("", "shimV2-")
-	if err != nil {
-		panic(fmt.Sprintf("ERROR: failed to create test directory: %v", err))
-	}
-	fmt.Printf("INFO: test directory is %v\n", testDir)
-
-	var output string
-	for _, name := range katautils.DockerLikeCtrEngines {
-		fmt.Printf("INFO: checking for container engine: %s\n", name)
-
-		output, err = ctrEngine.Init(name)
-		if err == nil {
-			break
-		}
-	}
-
-	if ctrEngine.Name == "" {
-		panic(fmt.Sprintf("ERROR: Docker-like container engine not accessible to current user: %v (error %v)",
-			output, err))
-	}
-
-	// Do this now to avoid hitting the test timeout value due to
-	// slow network response.
-	fmt.Printf("INFO: ensuring required container image (%v) is available\n", testDockerImage)
-	// Only hit the network if the image doesn't exist locally
-	_, err = ctrEngine.Inspect(testDockerImage)
-	if err == nil {
-		fmt.Printf("INFO: container image %v already exists locally\n", testDockerImage)
-	} else {
-		fmt.Printf("INFO: pulling container image %v\n", testDockerImage)
-		_, err = ctrEngine.Pull(testDockerImage)
-		if err != nil {
-			panic(err)
-		}
-	}
-
-	testBundleDir = filepath.Join(testDir, testBundle)
-	err = os.MkdirAll(testBundleDir, testDirMode)
-	if err != nil {
-		panic(fmt.Sprintf("ERROR: failed to create bundle directory %v: %v", testBundleDir, err))
-	}
-
-	fmt.Printf("INFO: creating OCI bundle in %v for tests to use\n", testBundleDir)
-	err = realMakeOCIBundle(testBundleDir)
-	if err != nil {
-		panic(fmt.Sprintf("ERROR: failed to create OCI bundle: %v", err))
-	}
-
 	tc = ktu.NewTestConstraint(false)
 
 	// disable shim management server.
@@ -131,46 +59,6 @@ func init() {
 	defaultStartManagementServerFunc = nil
 }
 
-// createOCIConfig creates an OCI configuration (spec) file in
-// the bundle directory specified (which must exist).
-func createOCIConfig(bundleDir string) error {
-	if bundleDir == "" {
-		return errors.New("BUG: Need bundle directory")
-	}
-
-	if !katautils.FileExists(bundleDir) {
-		return fmt.Errorf("BUG: Bundle directory %s does not exist", bundleDir)
-	}
-
-	var configCmd string
-
-	// Search for a suitable version of runc to use to generate
-	// the OCI config file.
-	for _, cmd := range []string{"docker-runc", "runc"} {
-		fullPath, err := sysExec.LookPath(cmd)
-		if err == nil {
-			configCmd = fullPath
-			break
-		}
-	}
-
-	if configCmd == "" {
-		return errors.New("Cannot find command to generate OCI config file")
-	}
-
-	_, err := utils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
-	if err != nil {
-		return err
-	}
-
-	specFile := filepath.Join(bundleDir, specConf)
-	if !katautils.FileExists(specFile) {
-		return fmt.Errorf("generated OCI config file does not exist: %v", specFile)
-	}
-
-	return nil
-}
-
 func createEmptyFile(path string) (err error) {
 	return ioutil.WriteFile(path, []byte(""), testFileMode)
 }
@@ -221,119 +109,6 @@ func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConf
 	}, nil
 }
 
-// realMakeOCIBundle will create an OCI bundle (including the "config.json"
-// config file) in the directory specified (which must already exist).
-//
-// XXX: Note that tests should *NOT* call this function - they should
-// XXX: instead call makeOCIBundle().
-func realMakeOCIBundle(bundleDir string) error {
-	if bundleDir == "" {
-		return errors.New("BUG: Need bundle directory")
-	}
-
-	if !katautils.FileExists(bundleDir) {
-		return fmt.Errorf("BUG: Bundle directory %v does not exist", bundleDir)
-	}
-
-	err := createOCIConfig(bundleDir)
-	if err != nil {
-		return err
-	}
-
-	// Note the unusual parameter (a directory, not the config
-	// file to parse!)
-	spec, err := compatoci.ParseConfigJSON(bundleDir)
-	if err != nil {
-		return err
-	}
-
-	// Determine the rootfs directory name the OCI config refers to
-	ociRootPath := spec.Root.Path
-
-	rootfsDir := filepath.Join(bundleDir, ociRootPath)
-
-	if strings.HasPrefix(ociRootPath, "/") {
-		return fmt.Errorf("Cannot handle absolute rootfs as bundle must be unique to each test")
-	}
-
-	err = createRootfs(rootfsDir)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// Create an OCI bundle in the specified directory.
-//
-// Note that the directory will be created, but it's parent is expected to exist.
-//
-// This function works by copying the already-created test bundle. Ideally,
-// the bundle would be recreated for each test, but createRootfs() uses
-// docker which on some systems is too slow, resulting in the tests timing
-// out.
-func makeOCIBundle(bundleDir string) error {
-	from := testBundleDir
-	to := bundleDir
-
-	// only the basename of bundleDir needs to exist as bundleDir
-	// will get created by cp(1).
-	base := filepath.Dir(bundleDir)
-
-	for _, dir := range []string{from, base} {
-		if !katautils.FileExists(dir) {
-			return fmt.Errorf("BUG: directory %v should exist", dir)
-		}
-	}
-
-	output, err := utils.RunCommandFull([]string{"cp", "-a", from, to}, true)
-	if err != nil {
-		return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output)
-	}
-
-	return nil
-}
-
-// createRootfs creates a minimal root filesystem below the specified
-// directory.
-func createRootfs(dir string) error {
-	err := os.MkdirAll(dir, testDirMode)
-	if err != nil {
-		return err
-	}
-
-	container, err := ctrEngine.Create(testDockerImage)
-	if err != nil {
-		return err
-	}
-
-	err = ctrEngine.GetRootfs(container, dir)
-	if err != nil {
-		return err
-	}
-
-	// Clean up
-	_, err = ctrEngine.Rm(container)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func writeOCIConfigFile(spec specs.Spec, configPath string) error {
-	if configPath == "" {
-		return errors.New("BUG: need config file path")
-	}
-
-	bytes, err := json.MarshalIndent(spec, "", "\t")
-	if err != nil {
-		return err
-	}
-
-	return ioutil.WriteFile(configPath, bytes, testFileMode)
-}
-
 func TestNoNeedForOutput(t *testing.T) {
 	assert := assert.New(t)
 

src/runtime/pkg/katatestutils/constraints.go

@@ -147,8 +147,15 @@ func getDistroDetails() (name, version string, err error) {
 //   centos: 3.10.0-957.12.1.el7.x86_64
 //   fedora: 5.0.9-200.fc29.x86_64
 //
+// For some self compiled kernel, the kernel version will be with "+" as its suffix
+// For example:
+//   5.12.0-rc4+
+// These kernel version can't be parsed by the current lib and lead to panic
+// therefore the '+' should be removed.
+//
 func fixKernelVersion(version string) string {
-	return strings.Replace(version, "_", "-", -1)
+	version = strings.Replace(version, "_", "-", -1)
+	return strings.Replace(version, "+", "", -1)
 }
 
 // handleDistroName checks that the current distro is compatible with

src/runtime/pkg/katatestutils/constraints_test.go

@@ -21,7 +21,6 @@ import (
 )
 
 const (
-	testFileMode    = os.FileMode(0640)
 	invalidOperator = 1234
 
 	skipUnknownDistroName = "skipping test as cannot determine distro name"

src/runtime/pkg/katatestutils/utils.go

@@ -7,8 +7,203 @@
 package katatestutils
 
 import (
+	"encoding/json"
+	"errors"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strconv"
+	"testing"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	testDirMode  = os.FileMode(0750)
+	testFileMode = os.FileMode(0640)
+
+	busyboxConfigJson = `
+{
+	"ociVersion": "1.0.1-dev",
+	"process": {
+		"terminal": true,
+		"user": {
+			"uid": 0,
+			"gid": 0
+		},
+		"args": [
+			"sh"
+		],
+		"env": [
+			"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+			"TERM=xterm"
+		],
+		"cwd": "/",
+		"capabilities": {
+			"bounding": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			],
+			"effective": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			],
+			"inheritable": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			],
+			"permitted": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			],
+			"ambient": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			]
+		},
+		"rlimits": [
+			{
+				"type": "RLIMIT_NOFILE",
+				"hard": 1024,
+				"soft": 1024
+			}
+		],
+		"noNewPrivileges": true
+	},
+	"root": {
+		"path": "rootfs",
+		"readonly": true
+	},
+	"hostname": "runc",
+	"mounts": [
+		{
+			"destination": "/proc",
+			"type": "proc",
+			"source": "proc"
+		},
+		{
+			"destination": "/dev",
+			"type": "tmpfs",
+			"source": "tmpfs",
+			"options": [
+				"nosuid",
+				"strictatime",
+				"mode=755",
+				"size=65536k"
+			]
+		},
+		{
+			"destination": "/dev/pts",
+			"type": "devpts",
+			"source": "devpts",
+			"options": [
+				"nosuid",
+				"noexec",
+				"newinstance",
+				"ptmxmode=0666",
+				"mode=0620",
+				"gid=5"
+			]
+		},
+		{
+			"destination": "/dev/shm",
+			"type": "tmpfs",
+			"source": "shm",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev",
+				"mode=1777",
+				"size=65536k"
+			]
+		},
+		{
+			"destination": "/dev/mqueue",
+			"type": "mqueue",
+			"source": "mqueue",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev"
+			]
+		},
+		{
+			"destination": "/sys",
+			"type": "sysfs",
+			"source": "sysfs",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev",
+				"ro"
+			]
+		},
+		{
+			"destination": "/sys/fs/cgroup",
+			"type": "cgroup",
+			"source": "cgroup",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev",
+				"relatime",
+				"ro"
+			]
+		}
+	],
+	"linux": {
+		"resources": {
+			"devices": [
+				{
+					"allow": false,
+					"access": "rwm"
+				}
+			]
+		},
+		"namespaces": [
+			{
+				"type": "pid"
+			},
+			{
+				"type": "network"
+			},
+			{
+				"type": "ipc"
+			},
+			{
+				"type": "uts"
+			},
+			{
+				"type": "mount"
+			}
+		],
+		"maskedPaths": [
+			"/proc/acpi",
+			"/proc/asound",
+			"/proc/kcore",
+			"/proc/keys",
+			"/proc/latency_stats",
+			"/proc/timer_list",
+			"/proc/timer_stats",
+			"/proc/sched_debug",
+			"/sys/firmware",
+			"/proc/scsi"
+		],
+		"readonlyPaths": [
+			"/proc/bus",
+			"/proc/fs",
+			"/proc/irq",
+			"/proc/sys",
+			"/proc/sysrq-trigger"
+		]
+	}
+}`
 )
 
 type RuntimeConfigOptions struct {
@@ -23,8 +218,6 @@ type RuntimeConfigOptions struct {
 	NetmonPath           string
 	LogPath              string
 	BlockDeviceDriver    string
-	AgentTraceMode       string
-	AgentTraceType       string
 	SharedFS             string
 	VirtioFSDaemon       string
 	JaegerEndpoint       string
@@ -137,8 +330,6 @@ func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string {
 	[agent.kata]
 	enable_debug = ` + strconv.FormatBool(config.AgentDebug) + `
 	enable_tracing = ` + strconv.FormatBool(config.AgentTrace) + `
-	trace_mode = "` + config.AgentTraceMode + `"` + `
-	trace_type = "` + config.AgentTraceType + `"` + `
 
 	[netmon]
 	path = "` + config.NetmonPath + `"
@@ -158,3 +349,34 @@ func IsInGitHubActions() bool {
 	// https://docs.github.com/en/actions/reference/environment-variables#default-environment-variables
 	return os.Getenv("GITHUB_ACTIONS") == "true"
 }
+
+func SetupOCIConfigFile(t *testing.T) (rootPath string, bundlePath, ociConfigFile string) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "katatest-")
+	assert.NoError(err)
+
+	bundlePath = filepath.Join(tmpdir, "bundle")
+	err = os.MkdirAll(bundlePath, testDirMode)
+	assert.NoError(err)
+
+	ociConfigFile = filepath.Join(bundlePath, "config.json")
+	err = ioutil.WriteFile(ociConfigFile, []byte(busyboxConfigJson), testFileMode)
+	assert.NoError(err)
+
+	return tmpdir, bundlePath, ociConfigFile
+}
+
+// WriteOCIConfigFile using spec to update OCI config file by path configPath
+func WriteOCIConfigFile(spec specs.Spec, configPath string) error {
+	if configPath == "" {
+		return errors.New("BUG: need config file path")
+	}
+
+	bytes, err := json.MarshalIndent(spec, "", "\t")
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(configPath, bytes, testFileMode)
+}

src/runtime/pkg/katautils/config-settings.go.in

@@ -87,6 +87,8 @@ const defaultTxRateLimiterMaxRate = uint64(0)
 const defaultConfidentialGuest = false
 const defaultGuestSwap = false
 const defaultRootlessHypervisor = false
+const defaultDisableSeccomp = false
+const defaultVfioMode = "guest-kernel"
 
 var defaultSGXEPCSize = int64(0)
 

src/runtime/pkg/katautils/config.go

@@ -135,6 +135,7 @@ type hypervisor struct {
 	ConfidentialGuest       bool     `toml:"confidential_guest"`
 	GuestSwap               bool     `toml:"enable_guest_swap"`
 	Rootless                bool     `toml:"rootless"`
+	DisableSeccomp          bool     `toml:"disable_seccomp"`
 }
 
 type runtime struct {
@@ -142,6 +143,7 @@ type runtime struct {
 	JaegerEndpoint      string   `toml:"jaeger_endpoint"`
 	JaegerUser          string   `toml:"jaeger_user"`
 	JaegerPassword      string   `toml:"jaeger_password"`
+	VfioMode            string   `toml:"vfio_mode"`
 	SandboxBindMounts   []string `toml:"sandbox_bind_mounts"`
 	Experimental        []string `toml:"experimental"`
 	Debug               bool     `toml:"enable_debug"`
@@ -153,8 +155,6 @@ type runtime struct {
 }
 
 type agent struct {
-	TraceMode           string   `toml:"trace_mode"`
-	TraceType           string   `toml:"trace_type"`
 	KernelModules       []string `toml:"kernel_modules"`
 	Debug               bool     `toml:"enable_debug"`
 	Tracing             bool     `toml:"enable_tracing"`
@@ -308,11 +308,24 @@ func (h hypervisor) GetEntropySource() string {
 	return h.EntropySource
 }
 
+// Current cpu number should not larger than defaultMaxVCPUs()
+func getCurrentCpuNum() uint32 {
+	var cpu uint32
+	h := hypervisor{}
+
+	cpu = uint32(goruntime.NumCPU())
+	if cpu > h.defaultMaxVCPUs() {
+		cpu = h.defaultMaxVCPUs()
+	}
+
+	return cpu
+}
+
 func (h hypervisor) defaultVCPUs() uint32 {
-	numCPUs := goruntime.NumCPU()
+	numCPUs := getCurrentCpuNum()
 
 	if h.NumVCPUs < 0 || h.NumVCPUs > int32(numCPUs) {
-		return uint32(numCPUs)
+		return numCPUs
 	}
 	if h.NumVCPUs == 0 { // or unspecified
 		return defaultVCPUCount
@@ -489,14 +502,6 @@ func (a agent) trace() bool {
 	return a.Tracing
 }
 
-func (a agent) traceMode() string {
-	return a.TraceMode
-}
-
-func (a agent) traceType() string {
-	return a.TraceType
-}
-
 func (a agent) kernelModules() []string {
 	return a.KernelModules
 }
@@ -875,6 +880,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		VirtioFSExtraArgs:       h.VirtioFSExtraArgs,
 		SGXEPCSize:              defaultSGXEPCSize,
 		EnableAnnotations:       h.EnableAnnotations,
+		DisableSeccomp:          h.DisableSeccomp,
 	}, nil
 }
 
@@ -929,8 +935,6 @@ func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oc
 			LongLiveConn:       true,
 			Debug:              agent.debug(),
 			Trace:              agent.trace(),
-			TraceMode:          agent.traceMode(),
-			TraceType:          agent.traceType(),
 			KernelModules:      agent.kernelModules(),
 			EnableDebugConsole: agent.debugConsoleEnabled(),
 			DialTimeout:        agent.dialTimout(),
@@ -976,10 +980,6 @@ func SetKernelParams(runtimeConfig *oci.RuntimeConfig) error {
 	}
 
 	// next, check for agent specific kernel params
-	err := vc.KataAgentSetDefaultTraceConfigOptions(&runtimeConfig.AgentConfig)
-	if err != nil {
-		return err
-	}
 
 	params := vc.KataAgentKernelParams(runtimeConfig.AgentConfig)
 
@@ -1072,6 +1072,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
 		ConfidentialGuest:       defaultConfidentialGuest,
 		GuestSwap:               defaultGuestSwap,
 		Rootless:                defaultRootlessHypervisor,
+		DisableSeccomp:          defaultDisableSeccomp,
 	}
 }
 
@@ -1081,6 +1082,11 @@ func initConfig() (config oci.RuntimeConfig, err error) {
 		return oci.RuntimeConfig{}, err
 	}
 
+	err = config.VfioMode.VFIOSetMode(defaultVfioMode)
+	if err != nil {
+		return oci.RuntimeConfig{}, err
+	}
+
 	config = oci.RuntimeConfig{
 		HypervisorType:   defaultHypervisor,
 		HypervisorConfig: GetDefaultHypervisorConfig(),
@@ -1127,6 +1133,14 @@ func LoadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat
 		}
 	}
 
+	if tomlConf.Runtime.VfioMode != "" {
+		err = config.VfioMode.VFIOSetMode(tomlConf.Runtime.VfioMode)
+
+		if err != nil {
+			return "", config, err
+		}
+	}
+
 	if !ignoreLogging {
 		err := handleSystemLog("", "")
 		if err != nil {

src/runtime/pkg/katautils/config_test.go

@@ -14,7 +14,6 @@ import (
 	"path"
 	"path/filepath"
 	"reflect"
-	goruntime "runtime"
 	"strings"
 	"syscall"
 	"testing"
@@ -156,7 +155,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		KernelParams:          vc.DeserializeParams(strings.Fields(kernelParams)),
 		HypervisorMachineType: machineType,
 		NumVCPUs:              defaultVCPUCount,
-		DefaultMaxVCPUs:       uint32(goruntime.NumCPU()),
+		DefaultMaxVCPUs:       getCurrentCpuNum(),
 		MemorySize:            defaultMemSize,
 		DisableBlockDeviceUse: disableBlockDevice,
 		BlockDeviceDriver:     defaultBlockDeviceDriver,
@@ -919,13 +918,13 @@ func TestNewClhHypervisorConfig(t *testing.T) {
 func TestHypervisorDefaults(t *testing.T) {
 	assert := assert.New(t)
 
-	numCPUs := goruntime.NumCPU()
+	numCPUs := getCurrentCpuNum()
 
 	h := hypervisor{}
 
 	assert.Equal(h.machineType(), defaultMachineType, "default hypervisor machine type wrong")
 	assert.Equal(h.defaultVCPUs(), defaultVCPUCount, "default vCPU number is wrong")
-	assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong")
+	assert.Equal(h.defaultMaxVCPUs(), numCPUs, "default max vCPU number is wrong")
 	assert.Equal(h.defaultMemSz(), defaultMemSize, "default memory size is wrong")
 
 	machineType := "foo"
@@ -934,23 +933,23 @@ func TestHypervisorDefaults(t *testing.T) {
 
 	// auto inferring
 	h.NumVCPUs = -1
-	assert.Equal(h.defaultVCPUs(), uint32(numCPUs), "default vCPU number is wrong")
+	assert.Equal(h.defaultVCPUs(), numCPUs, "default vCPU number is wrong")
 
 	h.NumVCPUs = 2
 	assert.Equal(h.defaultVCPUs(), uint32(2), "default vCPU number is wrong")
 
 	h.NumVCPUs = int32(numCPUs) + 1
-	assert.Equal(h.defaultVCPUs(), uint32(numCPUs), "default vCPU number is wrong")
+	assert.Equal(h.defaultVCPUs(), numCPUs, "default vCPU number is wrong")
 
 	h.DefaultMaxVCPUs = 2
 	assert.Equal(h.defaultMaxVCPUs(), uint32(2), "default max vCPU number is wrong")
 
-	h.DefaultMaxVCPUs = uint32(numCPUs) + 1
-	assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong")
+	h.DefaultMaxVCPUs = numCPUs + 1
+	assert.Equal(h.defaultMaxVCPUs(), numCPUs, "default max vCPU number is wrong")
 
 	maxvcpus := vc.MaxQemuVCPUs()
 	h.DefaultMaxVCPUs = maxvcpus + 1
-	assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong")
+	assert.Equal(h.defaultMaxVCPUs(), numCPUs, "default max vCPU number is wrong")
 
 	h.MemorySize = 1024
 	assert.Equal(h.defaultMemSz(), uint32(1024), "default memory size is wrong")
@@ -1152,9 +1151,6 @@ func TestAgentDefaults(t *testing.T) {
 
 	a.Tracing = true
 	assert.Equal(a.trace(), a.Tracing)
-
-	assert.Equal(a.traceMode(), a.TraceMode)
-	assert.Equal(a.traceType(), a.TraceType)
 }
 
 func TestGetDefaultConfigFilePaths(t *testing.T) {

src/runtime/pkg/katautils/container_engine.go

@@ -1,77 +0,0 @@
-// Copyright (c) 2019 SUSE LLC
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-package katautils
-
-import (
-	"os/exec"
-
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
-)
-
-type CtrEngine struct {
-	Name string
-}
-
-var (
-	DockerLikeCtrEngines = []string{"docker", "podman"}
-)
-
-func (e *CtrEngine) Init(name string) (string, error) {
-	var out string
-	out, err := utils.RunCommandFull([]string{name, "version"}, true)
-	if err != nil {
-		return out, err
-	}
-
-	e.Name = name
-	return out, nil
-}
-
-func (e *CtrEngine) Inspect(image string) (string, error) {
-	// Only hit the network if the image doesn't exist locally
-	return utils.RunCommand([]string{e.Name, "inspect", "--type=image", image})
-}
-
-func (e *CtrEngine) Pull(image string) (string, error) {
-	return utils.RunCommand([]string{e.Name, "pull", image})
-}
-
-func (e *CtrEngine) Create(image string) (string, error) {
-	return utils.RunCommand([]string{e.Name, "create", image})
-}
-
-func (e *CtrEngine) Rm(ctrID string) (string, error) {
-	return utils.RunCommand([]string{e.Name, "rm", ctrID})
-}
-
-func (e *CtrEngine) GetRootfs(ctrID string, dir string) error {
-	cmd1 := exec.Command(e.Name, "export", ctrID)
-	cmd2 := exec.Command("tar", "-C", dir, "-xvf", "-")
-
-	cmd1Stdout, err := cmd1.StdoutPipe()
-	if err != nil {
-		return err
-	}
-
-	cmd2.Stdin = cmd1Stdout
-
-	err = cmd2.Start()
-	if err != nil {
-		return err
-	}
-
-	err = cmd1.Run()
-	if err != nil {
-		return err
-	}
-
-	err = cmd2.Wait()
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

src/runtime/pkg/katautils/create.go

@@ -112,7 +112,7 @@ func SetEphemeralStorageType(ociSpec specs.Spec) specs.Spec {
 func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec specs.Spec, runtimeConfig oci.RuntimeConfig, rootFs vc.RootFs,
 	containerID, bundlePath, console string, disableOutput, systemdCgroup bool) (_ vc.VCSandbox, _ vc.Process, err error) {
 	span, ctx := katatrace.Trace(ctx, nil, "CreateSandbox", createTracingTags)
-	katatrace.AddTag(span, "container_id", containerID)
+	katatrace.AddTags(span, "container_id", containerID)
 	defer span.End()
 
 	sandboxConfig, err := oci.SandboxConfig(ociSpec, runtimeConfig, bundlePath, containerID, console, disableOutput, systemdCgroup)
@@ -167,7 +167,7 @@ func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec specs.Spec, runtimeCo
 
 	sid := sandbox.ID()
 	kataUtilsLogger = kataUtilsLogger.WithField("sandbox", sid)
-	katatrace.AddTag(span, "sandbox_id", sid)
+	katatrace.AddTags(span, "sandbox_id", sid)
 
 	containers := sandbox.GetAllContainers()
 	if len(containers) != 1 {
@@ -211,7 +211,7 @@ func CreateContainer(ctx context.Context, sandbox vc.VCSandbox, ociSpec specs.Sp
 	var c vc.VCContainer
 
 	span, ctx := katatrace.Trace(ctx, nil, "CreateContainer", createTracingTags)
-	katatrace.AddTag(span, "container_id", containerID)
+	katatrace.AddTags(span, "container_id", containerID)
 	defer span.End()
 
 	ociSpec = SetEphemeralStorageType(ociSpec)
@@ -237,7 +237,7 @@ func CreateContainer(ctx context.Context, sandbox vc.VCSandbox, ociSpec specs.Sp
 		return vc.Process{}, err
 	}
 
-	katatrace.AddTag(span, "sandbox_id", sandboxID)
+	katatrace.AddTags(span, "sandbox_id", sandboxID)
 
 	c, err = sandbox.CreateContainer(ctx, contConfig)
 	if err != nil {

src/runtime/pkg/katautils/create_test.go

@@ -8,7 +8,6 @@ package katautils
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -20,7 +19,6 @@ import (
 	"testing"
 
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
@@ -37,8 +35,6 @@ const (
 )
 
 var (
-	testBundleDir = ""
-
 	// testingImpl is a concrete mock RVC implementation used for testing
 	testingImpl = &vcmock.VCMock{}
 	// mock sandbox
@@ -53,49 +49,6 @@ func init() {
 	tc = ktu.NewTestConstraint(false)
 }
 
-func writeOCIConfigFile(spec specs.Spec, configPath string) error {
-	if configPath == "" {
-		return errors.New("BUG: need config file path")
-	}
-
-	bytes, err := json.MarshalIndent(spec, "", "\t")
-	if err != nil {
-		return err
-	}
-
-	return ioutil.WriteFile(configPath, bytes, testFileMode)
-}
-
-// Create an OCI bundle in the specified directory.
-//
-// Note that the directory will be created, but it's parent is expected to exist.
-//
-// This function works by copying the already-created test bundle. Ideally,
-// the bundle would be recreated for each test, but createRootfs() uses
-// docker which on some systems is too slow, resulting in the tests timing
-// out.
-func makeOCIBundle(bundleDir string) error {
-	from := testBundleDir
-	to := bundleDir
-
-	// only the basename of bundleDir needs to exist as bundleDir
-	// will get created by cp(1).
-	base := filepath.Dir(bundleDir)
-
-	for _, dir := range []string{from, base} {
-		if !FileExists(dir) {
-			return fmt.Errorf("BUG: directory %v should exist", dir)
-		}
-	}
-
-	output, err := utils.RunCommandFull([]string{"cp", "-a", from, to}, true)
-	if err != nil {
-		return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output)
-	}
-
-	return nil
-}
-
 // newTestRuntimeConfig creates a new RuntimeConfig
 func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConfig, error) {
 	if dir == "" {
@@ -262,21 +215,12 @@ func TestSetKernelParamsUserOptionTakesPriority(t *testing.T) {
 func TestCreateSandboxConfigFail(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, _ := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -305,21 +249,12 @@ func TestCreateSandboxFail(t *testing.T) {
 
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, _ := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -375,18 +310,9 @@ func TestCheckForFips(t *testing.T) {
 func TestCreateContainerContainerConfigFail(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -396,7 +322,7 @@ func TestCreateContainerContainerConfigFail(t *testing.T) {
 	spec.Annotations[testContainerTypeAnnotation] = containerType
 
 	// rewrite file
-	err = writeOCIConfigFile(spec, ociConfigFile)
+	err = ktu.WriteOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
 	rootFs := vc.RootFs{Mounted: true}
@@ -412,18 +338,9 @@ func TestCreateContainerContainerConfigFail(t *testing.T) {
 func TestCreateContainerFail(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -433,7 +350,7 @@ func TestCreateContainerFail(t *testing.T) {
 	spec.Annotations[testSandboxIDAnnotation] = testSandboxID
 
 	// rewrite file
-	err = writeOCIConfigFile(spec, ociConfigFile)
+	err = ktu.WriteOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
 	rootFs := vc.RootFs{Mounted: true}
@@ -456,18 +373,9 @@ func TestCreateContainer(t *testing.T) {
 		mockSandbox.CreateContainerFunc = nil
 	}()
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -477,7 +385,7 @@ func TestCreateContainer(t *testing.T) {
 	spec.Annotations[testSandboxIDAnnotation] = testSandboxID
 
 	// rewrite file
-	err = writeOCIConfigFile(spec, ociConfigFile)
+	err = ktu.WriteOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
 	rootFs := vc.RootFs{Mounted: true}

src/runtime/pkg/katautils/hook.go

@@ -35,11 +35,7 @@ func hookLogger() *logrus.Entry {
 func runHook(ctx context.Context, hook specs.Hook, cid, bundlePath string) error {
 	span, _ := katatrace.Trace(ctx, hookLogger(), "runHook", hookTracingTags)
 	defer span.End()
-
-	// FIXME
-	// span.LogFields(
-	// 	log.String("hook-name", hook.Path),
-	// 	log.String("hook-args", strings.Join(hook.Args, " ")))
+	katatrace.AddTags(span, "path", hook.Path, "args", hook.Args)
 
 	state := specs.State{
 		Pid:    syscall.Gettid(),
@@ -96,7 +92,7 @@ func runHook(ctx context.Context, hook specs.Hook, cid, bundlePath string) error
 
 func runHooks(ctx context.Context, hooks []specs.Hook, cid, bundlePath, hookType string) error {
 	span, ctx := katatrace.Trace(ctx, hookLogger(), "runHooks", hookTracingTags)
-	katatrace.AddTag(span, "type", hookType)
+	katatrace.AddTags(span, "type", hookType)
 	defer span.End()
 
 	for _, hook := range hooks {

src/runtime/pkg/katautils/katatrace/tracing.go

@@ -7,14 +7,16 @@ package katatrace
 
 import (
 	"context"
+	"encoding/json"
 
 	"github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel"
-	"go.opentelemetry.io/otel/exporters/trace/jaeger"
-	"go.opentelemetry.io/otel/label"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/jaeger"
 	"go.opentelemetry.io/otel/propagation"
-	export "go.opentelemetry.io/otel/sdk/export/trace"
+	"go.opentelemetry.io/otel/sdk/resource"
 	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
 	"go.opentelemetry.io/otel/trace"
 	otelTrace "go.opentelemetry.io/otel/trace"
 )
@@ -25,10 +27,10 @@ import (
 // https: //github.com/kata-containers/tests/blob/master/tracing/tracing-test.sh
 type kataSpanExporter struct{}
 
-var _ export.SpanExporter = (*kataSpanExporter)(nil)
+var _ sdktrace.SpanExporter = (*kataSpanExporter)(nil)
 
 // ExportSpans exports SpanData to Jaeger.
-func (e *kataSpanExporter) ExportSpans(ctx context.Context, spans []*export.SpanData) error {
+func (e *kataSpanExporter) ExportSpans(ctx context.Context, spans []sdktrace.ReadOnlySpan) error {
 	for _, span := range spans {
 		kataTraceLogger.Tracef("Reporting span %+v", span)
 	}
@@ -39,9 +41,9 @@ func (e *kataSpanExporter) Shutdown(ctx context.Context) error {
 	return nil
 }
 
-// tracerCloser contains a copy of the closer returned by createTracer() which
-// is used by stopTracing().
-var tracerCloser func()
+// tp is the trace provider created in CreateTracer() and used in StopTracing()
+// to flush and shutdown all spans.
+var tp *sdktrace.TracerProvider
 
 var kataTraceLogger = logrus.NewEntry(logrus.New())
 
@@ -61,10 +63,10 @@ type JaegerConfig struct {
 }
 
 // CreateTracer create a tracer
-func CreateTracer(name string, config *JaegerConfig) (func(), error) {
+func CreateTracer(name string, config *JaegerConfig) (*sdktrace.TracerProvider, error) {
 	if !tracing {
 		otel.SetTracerProvider(trace.NewNoopTracerProvider())
-		return func() {}, nil
+		return nil, nil
 	}
 
 	// build kata exporter to log reporting span records
@@ -76,37 +78,32 @@ func CreateTracer(name string, config *JaegerConfig) (func(), error) {
 		collectorEndpoint = "http://localhost:14268/api/traces"
 	}
 
-	jaegerExporter, err := jaeger.NewRawExporter(
-		jaeger.WithCollectorEndpoint(collectorEndpoint,
+	jaegerExporter, err := jaeger.New(
+		jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(collectorEndpoint),
 			jaeger.WithUsername(config.JaegerUser),
 			jaeger.WithPassword(config.JaegerPassword),
-		), jaeger.WithProcess(jaeger.Process{
-			ServiceName: name,
-			Tags: []label.KeyValue{
-				label.String("exporter", "jaeger"),
-				label.String("lib", "opentelemetry"),
-			},
-		}))
+		),
+	)
+
 	if err != nil {
 		return nil, err
 	}
 
 	// build tracer provider, that combining both jaeger exporter and kata exporter.
 	tp := sdktrace.NewTracerProvider(
-		sdktrace.WithConfig(
-			sdktrace.Config{
-				DefaultSampler: sdktrace.AlwaysSample(),
-			},
-		),
+		sdktrace.WithSampler(sdktrace.AlwaysSample()),
 		sdktrace.WithSyncer(kataExporter),
 		sdktrace.WithSyncer(jaegerExporter),
+		sdktrace.WithResource(resource.NewSchemaless(
+			semconv.ServiceNameKey.String(name),
+			attribute.String("exporter", "jaeger"),
+			attribute.String("lib", "opentelemetry"),
+		)),
 	)
 
-	tracerCloser = jaegerExporter.Flush
-
 	otel.SetTracerProvider(tp)
 	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))
-	return tracerCloser, nil
+	return tp, nil
 }
 
 // StopTracing ends all tracing, reporting the spans to the collector.
@@ -121,9 +118,8 @@ func StopTracing(ctx context.Context) {
 	}
 
 	// report all possible spans to the collector
-	if tracerCloser != nil {
-		tracerCloser()
-	}
+	tp.ForceFlush(ctx)
+	tp.Shutdown(ctx)
 }
 
 // Trace creates a new tracing span based on the specified name and parent context.
@@ -138,12 +134,12 @@ func Trace(parent context.Context, logger *logrus.Entry, name string, tags ...ma
 		parent = context.Background()
 	}
 
-	var otelTags []label.KeyValue
+	var otelTags []attribute.KeyValue
 	// do not append tags if tracing is disabled
 	if tracing {
 		for _, tagSet := range tags {
 			for k, v := range tagSet {
-				otelTags = append(otelTags, label.Key(k).String(v))
+				otelTags = append(otelTags, attribute.Key(k).String(v))
 			}
 		}
 	}
@@ -163,8 +159,63 @@ func Trace(parent context.Context, logger *logrus.Entry, name string, tags ...ma
 	return span, ctx
 }
 
-// AddTag adds an additional key-value pair to a tracing span. This can be used to
-// provide dynamic tags that are determined at runtime.
-func AddTag(span otelTrace.Span, key string, value interface{}) {
-	span.SetAttributes(label.Any(key, value))
+func addTag(span otelTrace.Span, key string, value interface{}) {
+	// do not append tags if tracing is disabled
+	if !tracing {
+		return
+	}
+	if value == nil {
+		span.SetAttributes(attribute.String(key, "nil"))
+		return
+	}
+
+	switch value := value.(type) {
+	case string:
+		span.SetAttributes(attribute.String(key, value))
+	case bool:
+		span.SetAttributes(attribute.Bool(key, value))
+	case int:
+		span.SetAttributes(attribute.Int(key, value))
+	case int8:
+		span.SetAttributes(attribute.Int(key, int(value)))
+	case int16:
+		span.SetAttributes(attribute.Int(key, int(value)))
+	case int64:
+		span.SetAttributes(attribute.Int64(key, value))
+	case float64:
+		span.SetAttributes(attribute.Float64(key, value))
+	default:
+		content, err := json.Marshal(value)
+		if content == nil && err == nil {
+			span.SetAttributes(attribute.String(key, "nil"))
+		} else if content != nil && err == nil {
+			span.SetAttributes(attribute.String(key, string(content)))
+		} else {
+			kataTraceLogger.WithField("type", "bug").Error("span attribute value error")
+		}
+	}
+}
+
+// AddTag adds additional key-value pairs to a tracing span. This can be used to provide
+// dynamic tags that are determined at runtime and tags with a non-string value.
+// Must have an even number of keyValues with keys being strings.
+func AddTags(span otelTrace.Span, keyValues ...interface{}) {
+	if !tracing {
+		return
+	}
+	if len(keyValues) < 2 {
+		kataTraceLogger.WithField("type", "bug").Error("not enough inputs for attributes")
+		return
+	} else if len(keyValues)%2 != 0 {
+		kataTraceLogger.WithField("type", "bug").Error("number of attribute keyValues is not even")
+		return
+	}
+	for i := 0; i < len(keyValues); i++ {
+		if key, ok := keyValues[i].(string); ok {
+			addTag(span, key, keyValues[i+1])
+		} else {
+			kataTraceLogger.WithField("type", "bug").Error("key in attributes is not a string")
+		}
+		i++
+	}
 }

src/runtime/pkg/katautils/logger.go

@@ -20,6 +20,9 @@ import (
 var originalLoggerLevel = logrus.WarnLevel
 var kataUtilsLogger = logrus.NewEntry(logrus.New())
 
+// SYSLOGTAG is for a consistently named syslog identifier
+const SYSLOGTAG = "kata"
+
 // SetLogger sets the logger for the factory.
 func SetLogger(ctx context.Context, logger *logrus.Entry, level logrus.Level) {
 	fields := logrus.Fields{
@@ -61,7 +64,7 @@ func (h *sysLogHook) Fire(e *logrus.Entry) (err error) {
 }
 
 func newSystemLogHook(network, raddr string) (*sysLogHook, error) {
-	hook, err := lSyslog.NewSyslogHook(network, raddr, syslog.LOG_INFO, NAME)
+	hook, err := lSyslog.NewSyslogHook(network, raddr, syslog.LOG_INFO, SYSLOGTAG)
 	if err != nil {
 		return nil, err
 	}

src/runtime/pkg/katautils/utils_test.go

@@ -7,20 +7,15 @@
 package katautils
 
 import (
-	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path"
 	"path/filepath"
-	"strings"
 	"syscall"
 	"testing"
 
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -28,173 +23,14 @@ const (
 	testDirMode  = os.FileMode(0750)
 	testFileMode = os.FileMode(0640)
 
-	// small docker image used to create root filesystems from
-	testDockerImage = "busybox"
-
 	testSandboxID   = "99999999-9999-9999-99999999999999999"
 	testContainerID = "1"
-	testBundle      = "bundle"
-	specConfig      = "config.json"
 )
 
 var (
 	testDir = ""
 )
 
-func init() {
-	var err error
-
-	fmt.Printf("INFO: creating test directory\n")
-	testDir, err = ioutil.TempDir("", fmt.Sprintf("%s-", NAME))
-	if err != nil {
-		panic(fmt.Sprintf("ERROR: failed to create test directory: %v", err))
-	}
-
-	fmt.Printf("INFO: test directory is %v\n", testDir)
-
-	testBundleDir = filepath.Join(testDir, testBundle)
-	err = os.MkdirAll(testBundleDir, testDirMode)
-	if err != nil {
-		panic(fmt.Sprintf("ERROR: failed to create bundle directory %v: %v", testBundleDir, err))
-	}
-
-	fmt.Printf("INFO: creating OCI bundle in %v for tests to use\n", testBundleDir)
-	err = realMakeOCIBundle(testBundleDir)
-	if err != nil {
-		panic(fmt.Sprintf("ERROR: failed to create OCI bundle: %v", err))
-	}
-}
-
-// createOCIConfig creates an OCI configuration (spec) file in
-// the bundle directory specified (which must exist).
-func createOCIConfig(bundleDir string) error {
-	if bundleDir == "" {
-		return errors.New("BUG: Need bundle directory")
-	}
-
-	if !FileExists(bundleDir) {
-		return fmt.Errorf("BUG: Bundle directory %s does not exist", bundleDir)
-	}
-
-	var configCmd string
-
-	// Search for a suitable version of runc to use to generate
-	// the OCI config file.
-	for _, cmd := range []string{"docker-runc", "runc"} {
-		fullPath, err := exec.LookPath(cmd)
-		if err == nil {
-			configCmd = fullPath
-			break
-		}
-	}
-
-	if configCmd == "" {
-		return errors.New("Cannot find command to generate OCI config file")
-	}
-
-	_, err := utils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
-	if err != nil {
-		return err
-	}
-
-	specFile := filepath.Join(bundleDir, specConfig)
-	if !FileExists(specFile) {
-		return fmt.Errorf("generated OCI config file does not exist: %v", specFile)
-	}
-
-	return nil
-}
-
-// realMakeOCIBundle will create an OCI bundle (including the "config.json"
-// config file) in the directory specified (which must already exist).
-//
-// XXX: Note that tests should *NOT* call this function - they should
-// XXX: instead call makeOCIBundle().
-func realMakeOCIBundle(bundleDir string) error {
-	if bundleDir == "" {
-		return errors.New("BUG: Need bundle directory")
-	}
-
-	if !FileExists(bundleDir) {
-		return fmt.Errorf("BUG: Bundle directory %v does not exist", bundleDir)
-	}
-
-	err := createOCIConfig(bundleDir)
-	if err != nil {
-		return err
-	}
-
-	// Note the unusual parameter (a directory, not the config
-	// file to parse!)
-	spec, err := compatoci.ParseConfigJSON(bundleDir)
-	if err != nil {
-		return err
-	}
-
-	// Determine the rootfs directory name the OCI config refers to
-	ociRootPath := spec.Root.Path
-
-	rootfsDir := filepath.Join(bundleDir, ociRootPath)
-
-	if strings.HasPrefix(ociRootPath, "/") {
-		return fmt.Errorf("Cannot handle absolute rootfs as bundle must be unique to each test")
-	}
-
-	err = createRootfs(rootfsDir)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// createRootfs creates a minimal root filesystem below the specified
-// directory.
-func createRootfs(dir string) error {
-	var (
-		output string
-		err    error
-	)
-
-	ctrEngine := CtrEngine{}
-	for _, name := range DockerLikeCtrEngines {
-		fmt.Printf("INFO: checking for container engine: %s\n", name)
-
-		output, err = ctrEngine.Init(name)
-		if err == nil {
-			break
-		}
-	}
-
-	if ctrEngine.Name == "" {
-		panic(fmt.Sprintf("ERROR: Docker-like container engine not accessible to current user: %v (error %v)",
-			output, err))
-	}
-
-	err = os.MkdirAll(dir, testDirMode)
-	if err != nil {
-		return err
-	}
-
-	container, err := ctrEngine.Create(testDockerImage)
-	if err != nil {
-		return err
-	}
-
-	err = ctrEngine.GetRootfs(container, dir)
-	if err != nil {
-		return err
-	}
-
-	// Clean up
-	_, err = ctrEngine.Rm(container)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 func createFile(file, contents string) error {
 	return ioutil.WriteFile(file, []byte(contents), testFileMode)
 }

src/runtime/pkg/utils/utils.go

@@ -13,6 +13,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/sirupsen/logrus"
 )
@@ -124,6 +125,7 @@ func CreateVmmUser() (string, error) {
 	// Add retries to mitigate temporary errors and race conditions. For example, the user already exists
 	// or another instance of the runtime is also creating a user.
 	maxAttempt := 5
+	rand.Seed(time.Now().UnixNano())
 	for i := 0; i < maxAttempt; i++ {
 		userName = fmt.Sprintf("kata-%v", rand.Intn(100000))
 		_, err = RunCommand([]string{useraddPath, "-M", "-s", nologinPath, userName, "-c", "\"Kata Containers temporary hypervisor user\""})

src/runtime/vendor/github.com/Microsoft/hcsshim/go.mod

@@ -4,21 +4,25 @@ go 1.13
 
 require (
 	github.com/Microsoft/go-winio v0.4.17
+	github.com/cenkalti/backoff/v4 v4.1.1
 	github.com/containerd/cgroups v1.0.1
 	github.com/containerd/console v1.0.2
-	github.com/containerd/containerd v1.5.1
+	github.com/containerd/containerd v1.4.9
+	github.com/containerd/continuity v0.1.0 // indirect
+	github.com/containerd/fifo v1.0.0 // indirect
 	github.com/containerd/go-runc v1.0.0
-	github.com/containerd/ttrpc v1.0.2
+	github.com/containerd/ttrpc v1.1.0
 	github.com/containerd/typeurl v1.0.2
 	github.com/gogo/protobuf v1.3.2
 	github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d
 	github.com/pkg/errors v0.9.1
-	github.com/sirupsen/logrus v1.7.0
+	github.com/sirupsen/logrus v1.8.1
 	github.com/urfave/cli v1.22.2
 	go.opencensus.io v0.22.3
 	golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
 	golang.org/x/sys v0.0.0-20210324051608-47abb6519492
 	google.golang.org/grpc v1.33.2
+	gotest.tools/v3 v3.0.3 // indirect
 )
 
 replace (

src/runtime/vendor/github.com/Microsoft/hcsshim/go.sum

@@ -1,779 +1,218 @@
 bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
-cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
-cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
-cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
-cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
-cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
-cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
-cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
-cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
-github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
-github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
-github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
-github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=
-github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg=
-github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
-github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
-github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
-github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
-github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
-github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
-github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
-github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
-github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
-github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
-github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
-github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.4.17 h1:iT12IBVClFevaf8PuVyi3UmZOVh4OqnaLxDTW2O6j3w=
 github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
-github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
-github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
-github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ=
-github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8=
-github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg=
-github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00=
-github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600=
-github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU=
-github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY=
-github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
-github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
-github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
-github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
-github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
-github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
-github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
-github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
-github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
-github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50=
-github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
+github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ=
+github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
-github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw=
-github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg=
-github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc=
-github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs=
 github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
-github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
-github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE=
-github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU=
-github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=
-github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=
-github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E=
-github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss=
-github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss=
-github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI=
-github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko=
-github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM=
-github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo=
-github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo=
-github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE=
 github.com/containerd/cgroups v1.0.1 h1:iJnMvco9XGvKUvNQkv88bE4uJXxRQH18efbKo9w5vHQ=
 github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU=
-github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
-github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
-github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE=
 github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw=
 github.com/containerd/console v1.0.2 h1:Pi6D+aZXM+oUw1czuKgH5IJ+y0jhYcwBJfx5/Ghn9dE=
 github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ=
-github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ=
-github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU=
-github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI=
-github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s=
-github.com/containerd/containerd v1.5.1 h1:xWHPAoe6VkUiI9GAvndJM7s/0MTrmwX3AQiYTr3olf0=
-github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g=
-github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
-github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
-github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
-github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo=
-github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y=
-github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ=
+github.com/containerd/containerd v1.4.9 h1:JIw9mjVw4LsGmnA/Bqg9j9e+XB7soOJufrKUpA6n2Ns=
+github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/continuity v0.1.0 h1:UFRRY5JemiAhPZrr/uE0n8fMTLcZsUvySPr1+D7pgr8=
 github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM=
-github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
-github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
-github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0=
-github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0=
-github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4=
 github.com/containerd/fifo v1.0.0 h1:6PirWBr9/L7GDamKr+XM0IeUFXu5mf3M/BPpH9gaLBU=
 github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4=
-github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU=
-github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk=
-github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
-github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
-github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g=
-github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok=
 github.com/containerd/go-runc v1.0.0 h1:oU+lLv1ULm5taqgV/CJivypVODI4SUz1znWjv3nNYS0=
 github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok=
-github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0=
-github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA=
-github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow=
-github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms=
-github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c=
-github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
-github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
-github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
-github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
-github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8=
-github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y=
-github.com/containerd/ttrpc v1.0.2 h1:2/O3oTZN36q2xRolk0a2WWGgh7/Vf/liElg5hFYLX9U=
-github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y=
-github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc=
-github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk=
-github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg=
+github.com/containerd/ttrpc v1.1.0 h1:GbtyLRxb0gOLR0TYQWt3O6B0NvT8tMdorEHqIQo/lWI=
+github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ=
 github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY=
 github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s=
-github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw=
-github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y=
-github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY=
-github.com/containerd/zfs v0.0.0-20210324211415-d5c4544f0433/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY=
-github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY=
-github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
-github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
-github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
-github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM=
-github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8=
-github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc=
-github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4=
-github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
-github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
-github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
 github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
-github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
-github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
-github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
-github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
-github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8=
-github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0=
-github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
-github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY=
-github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
-github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
-github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI=
-github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
-github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
-github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
-github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
-github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA=
-github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
-github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
-github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
-github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
-github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
-github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
-github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
-github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
-github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU=
-github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
-github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
-github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
-github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
-github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
-github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
-github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
-github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
-github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
-github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
-github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
-github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
-github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
-github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
-github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
-github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
-github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
-github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
-github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
-github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
-github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
-github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
-github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
-github.com/moby/sys/mountinfo v0.4.1 h1:1O+1cHA1aujwEwwVMa2Xm2l+gIpUHyd3+D+d7LZh1kM=
-github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
-github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ=
-github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
-github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
-github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
-github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
-github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
-github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
-github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
-github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
-github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
-github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
-github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
-github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
-github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
-github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
-github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
-github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
-github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0=
-github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d h1:pNa8metDkwZjb9g4T8s+krQ+HRgZAkqnXml+wNir/+s=
 github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
-github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE=
-github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
-github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
-github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
-github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
-github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
-github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
-github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
-github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
-github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
-github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
-github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
-github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
-github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
-github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
-github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
-github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
-github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
-github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8=
-github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
-github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
-github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
-github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
-github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
-github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
-github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
-github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo=
 github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
-github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
-github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
-github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
-github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
-github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
-github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
-github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
-github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
-github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
-github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
-github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA=
-github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
-go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg=
-go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
-golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
-golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210324051608-47abb6519492 h1:Paq34FxTluEPvVyayQqMPgHm+vTOrIifmcYxFBx9TLg=
 golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
-golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -781,105 +220,23 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk=
 google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63 h1:YzfoEYWbODU5Fbt37+h7X16BWQbad7Q4S6gclTKFXM8=
 google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk=
 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
-google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
-google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
-google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
-google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
-google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
-gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
-gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
-gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
-gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
-gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
-gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
-gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo=
-k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ=
-k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8=
-k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
-k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
-k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc=
-k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU=
-k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM=
-k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q=
-k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y=
-k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k=
-k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0=
-k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk=
-k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI=
-k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM=
-k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM=
-k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=
-k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=
-k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc=
-k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
-k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
-k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
-k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=
-k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
-k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
-rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
-rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
-sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
-sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
-sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

src/runtime/vendor/github.com/apache/thrift/lib/go/thrift/binary_protocol.go

@@ -1,505 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package thrift
-
-import (
-	"bytes"
-	"context"
-	"encoding/binary"
-	"errors"
-	"fmt"
-	"io"
-	"math"
-)
-
-type TBinaryProtocol struct {
-	trans         TRichTransport
-	origTransport TTransport
-	strictRead    bool
-	strictWrite   bool
-	buffer        [64]byte
-}
-
-type TBinaryProtocolFactory struct {
-	strictRead  bool
-	strictWrite bool
-}
-
-func NewTBinaryProtocolTransport(t TTransport) *TBinaryProtocol {
-	return NewTBinaryProtocol(t, false, true)
-}
-
-func NewTBinaryProtocol(t TTransport, strictRead, strictWrite bool) *TBinaryProtocol {
-	p := &TBinaryProtocol{origTransport: t, strictRead: strictRead, strictWrite: strictWrite}
-	if et, ok := t.(TRichTransport); ok {
-		p.trans = et
-	} else {
-		p.trans = NewTRichTransport(t)
-	}
-	return p
-}
-
-func NewTBinaryProtocolFactoryDefault() *TBinaryProtocolFactory {
-	return NewTBinaryProtocolFactory(false, true)
-}
-
-func NewTBinaryProtocolFactory(strictRead, strictWrite bool) *TBinaryProtocolFactory {
-	return &TBinaryProtocolFactory{strictRead: strictRead, strictWrite: strictWrite}
-}
-
-func (p *TBinaryProtocolFactory) GetProtocol(t TTransport) TProtocol {
-	return NewTBinaryProtocol(t, p.strictRead, p.strictWrite)
-}
-
-/**
- * Writing Methods
- */
-
-func (p *TBinaryProtocol) WriteMessageBegin(name string, typeId TMessageType, seqId int32) error {
-	if p.strictWrite {
-		version := uint32(VERSION_1) | uint32(typeId)
-		e := p.WriteI32(int32(version))
-		if e != nil {
-			return e
-		}
-		e = p.WriteString(name)
-		if e != nil {
-			return e
-		}
-		e = p.WriteI32(seqId)
-		return e
-	} else {
-		e := p.WriteString(name)
-		if e != nil {
-			return e
-		}
-		e = p.WriteByte(int8(typeId))
-		if e != nil {
-			return e
-		}
-		e = p.WriteI32(seqId)
-		return e
-	}
-	return nil
-}
-
-func (p *TBinaryProtocol) WriteMessageEnd() error {
-	return nil
-}
-
-func (p *TBinaryProtocol) WriteStructBegin(name string) error {
-	return nil
-}
-
-func (p *TBinaryProtocol) WriteStructEnd() error {
-	return nil
-}
-
-func (p *TBinaryProtocol) WriteFieldBegin(name string, typeId TType, id int16) error {
-	e := p.WriteByte(int8(typeId))
-	if e != nil {
-		return e
-	}
-	e = p.WriteI16(id)
-	return e
-}
-
-func (p *TBinaryProtocol) WriteFieldEnd() error {
-	return nil
-}
-
-func (p *TBinaryProtocol) WriteFieldStop() error {
-	e := p.WriteByte(STOP)
-	return e
-}
-
-func (p *TBinaryProtocol) WriteMapBegin(keyType TType, valueType TType, size int) error {
-	e := p.WriteByte(int8(keyType))
-	if e != nil {
-		return e
-	}
-	e = p.WriteByte(int8(valueType))
-	if e != nil {
-		return e
-	}
-	e = p.WriteI32(int32(size))
-	return e
-}
-
-func (p *TBinaryProtocol) WriteMapEnd() error {
-	return nil
-}
-
-func (p *TBinaryProtocol) WriteListBegin(elemType TType, size int) error {
-	e := p.WriteByte(int8(elemType))
-	if e != nil {
-		return e
-	}
-	e = p.WriteI32(int32(size))
-	return e
-}
-
-func (p *TBinaryProtocol) WriteListEnd() error {
-	return nil
-}
-
-func (p *TBinaryProtocol) WriteSetBegin(elemType TType, size int) error {
-	e := p.WriteByte(int8(elemType))
-	if e != nil {
-		return e
-	}
-	e = p.WriteI32(int32(size))
-	return e
-}
-
-func (p *TBinaryProtocol) WriteSetEnd() error {
-	return nil
-}
-
-func (p *TBinaryProtocol) WriteBool(value bool) error {
-	if value {
-		return p.WriteByte(1)
-	}
-	return p.WriteByte(0)
-}
-
-func (p *TBinaryProtocol) WriteByte(value int8) error {
-	e := p.trans.WriteByte(byte(value))
-	return NewTProtocolException(e)
-}
-
-func (p *TBinaryProtocol) WriteI16(value int16) error {
-	v := p.buffer[0:2]
-	binary.BigEndian.PutUint16(v, uint16(value))
-	_, e := p.trans.Write(v)
-	return NewTProtocolException(e)
-}
-
-func (p *TBinaryProtocol) WriteI32(value int32) error {
-	v := p.buffer[0:4]
-	binary.BigEndian.PutUint32(v, uint32(value))
-	_, e := p.trans.Write(v)
-	return NewTProtocolException(e)
-}
-
-func (p *TBinaryProtocol) WriteI64(value int64) error {
-	v := p.buffer[0:8]
-	binary.BigEndian.PutUint64(v, uint64(value))
-	_, err := p.trans.Write(v)
-	return NewTProtocolException(err)
-}
-
-func (p *TBinaryProtocol) WriteDouble(value float64) error {
-	return p.WriteI64(int64(math.Float64bits(value)))
-}
-
-func (p *TBinaryProtocol) WriteString(value string) error {
-	e := p.WriteI32(int32(len(value)))
-	if e != nil {
-		return e
-	}
-	_, err := p.trans.WriteString(value)
-	return NewTProtocolException(err)
-}
-
-func (p *TBinaryProtocol) WriteBinary(value []byte) error {
-	e := p.WriteI32(int32(len(value)))
-	if e != nil {
-		return e
-	}
-	_, err := p.trans.Write(value)
-	return NewTProtocolException(err)
-}
-
-/**
- * Reading methods
- */
-
-func (p *TBinaryProtocol) ReadMessageBegin() (name string, typeId TMessageType, seqId int32, err error) {
-	size, e := p.ReadI32()
-	if e != nil {
-		return "", typeId, 0, NewTProtocolException(e)
-	}
-	if size < 0 {
-		typeId = TMessageType(size & 0x0ff)
-		version := int64(int64(size) & VERSION_MASK)
-		if version != VERSION_1 {
-			return name, typeId, seqId, NewTProtocolExceptionWithType(BAD_VERSION, fmt.Errorf("Bad version in ReadMessageBegin"))
-		}
-		name, e = p.ReadString()
-		if e != nil {
-			return name, typeId, seqId, NewTProtocolException(e)
-		}
-		seqId, e = p.ReadI32()
-		if e != nil {
-			return name, typeId, seqId, NewTProtocolException(e)
-		}
-		return name, typeId, seqId, nil
-	}
-	if p.strictRead {
-		return name, typeId, seqId, NewTProtocolExceptionWithType(BAD_VERSION, fmt.Errorf("Missing version in ReadMessageBegin"))
-	}
-	name, e2 := p.readStringBody(size)
-	if e2 != nil {
-		return name, typeId, seqId, e2
-	}
-	b, e3 := p.ReadByte()
-	if e3 != nil {
-		return name, typeId, seqId, e3
-	}
-	typeId = TMessageType(b)
-	seqId, e4 := p.ReadI32()
-	if e4 != nil {
-		return name, typeId, seqId, e4
-	}
-	return name, typeId, seqId, nil
-}
-
-func (p *TBinaryProtocol) ReadMessageEnd() error {
-	return nil
-}
-
-func (p *TBinaryProtocol) ReadStructBegin() (name string, err error) {
-	return
-}
-
-func (p *TBinaryProtocol) ReadStructEnd() error {
-	return nil
-}
-
-func (p *TBinaryProtocol) ReadFieldBegin() (name string, typeId TType, seqId int16, err error) {
-	t, err := p.ReadByte()
-	typeId = TType(t)
-	if err != nil {
-		return name, typeId, seqId, err
-	}
-	if t != STOP {
-		seqId, err = p.ReadI16()
-	}
-	return name, typeId, seqId, err
-}
-
-func (p *TBinaryProtocol) ReadFieldEnd() error {
-	return nil
-}
-
-var invalidDataLength = NewTProtocolExceptionWithType(INVALID_DATA, errors.New("Invalid data length"))
-
-func (p *TBinaryProtocol) ReadMapBegin() (kType, vType TType, size int, err error) {
-	k, e := p.ReadByte()
-	if e != nil {
-		err = NewTProtocolException(e)
-		return
-	}
-	kType = TType(k)
-	v, e := p.ReadByte()
-	if e != nil {
-		err = NewTProtocolException(e)
-		return
-	}
-	vType = TType(v)
-	size32, e := p.ReadI32()
-	if e != nil {
-		err = NewTProtocolException(e)
-		return
-	}
-	if size32 < 0 {
-		err = invalidDataLength
-		return
-	}
-	size = int(size32)
-	return kType, vType, size, nil
-}
-
-func (p *TBinaryProtocol) ReadMapEnd() error {
-	return nil
-}
-
-func (p *TBinaryProtocol) ReadListBegin() (elemType TType, size int, err error) {
-	b, e := p.ReadByte()
-	if e != nil {
-		err = NewTProtocolException(e)
-		return
-	}
-	elemType = TType(b)
-	size32, e := p.ReadI32()
-	if e != nil {
-		err = NewTProtocolException(e)
-		return
-	}
-	if size32 < 0 {
-		err = invalidDataLength
-		return
-	}
-	size = int(size32)
-
-	return
-}
-
-func (p *TBinaryProtocol) ReadListEnd() error {
-	return nil
-}
-
-func (p *TBinaryProtocol) ReadSetBegin() (elemType TType, size int, err error) {
-	b, e := p.ReadByte()
-	if e != nil {
-		err = NewTProtocolException(e)
-		return
-	}
-	elemType = TType(b)
-	size32, e := p.ReadI32()
-	if e != nil {
-		err = NewTProtocolException(e)
-		return
-	}
-	if size32 < 0 {
-		err = invalidDataLength
-		return
-	}
-	size = int(size32)
-	return elemType, size, nil
-}
-
-func (p *TBinaryProtocol) ReadSetEnd() error {
-	return nil
-}
-
-func (p *TBinaryProtocol) ReadBool() (bool, error) {
-	b, e := p.ReadByte()
-	v := true
-	if b != 1 {
-		v = false
-	}
-	return v, e
-}
-
-func (p *TBinaryProtocol) ReadByte() (int8, error) {
-	v, err := p.trans.ReadByte()
-	return int8(v), err
-}
-
-func (p *TBinaryProtocol) ReadI16() (value int16, err error) {
-	buf := p.buffer[0:2]
-	err = p.readAll(buf)
-	value = int16(binary.BigEndian.Uint16(buf))
-	return value, err
-}
-
-func (p *TBinaryProtocol) ReadI32() (value int32, err error) {
-	buf := p.buffer[0:4]
-	err = p.readAll(buf)
-	value = int32(binary.BigEndian.Uint32(buf))
-	return value, err
-}
-
-func (p *TBinaryProtocol) ReadI64() (value int64, err error) {
-	buf := p.buffer[0:8]
-	err = p.readAll(buf)
-	value = int64(binary.BigEndian.Uint64(buf))
-	return value, err
-}
-
-func (p *TBinaryProtocol) ReadDouble() (value float64, err error) {
-	buf := p.buffer[0:8]
-	err = p.readAll(buf)
-	value = math.Float64frombits(binary.BigEndian.Uint64(buf))
-	return value, err
-}
-
-func (p *TBinaryProtocol) ReadString() (value string, err error) {
-	size, e := p.ReadI32()
-	if e != nil {
-		return "", e
-	}
-	if size < 0 {
-		err = invalidDataLength
-		return
-	}
-
-	return p.readStringBody(size)
-}
-
-func (p *TBinaryProtocol) ReadBinary() ([]byte, error) {
-	size, e := p.ReadI32()
-	if e != nil {
-		return nil, e
-	}
-	if size < 0 {
-		return nil, invalidDataLength
-	}
-
-	isize := int(size)
-	buf := make([]byte, isize)
-	_, err := io.ReadFull(p.trans, buf)
-	return buf, NewTProtocolException(err)
-}
-
-func (p *TBinaryProtocol) Flush(ctx context.Context) (err error) {
-	return NewTProtocolException(p.trans.Flush(ctx))
-}
-
-func (p *TBinaryProtocol) Skip(fieldType TType) (err error) {
-	return SkipDefaultDepth(p, fieldType)
-}
-
-func (p *TBinaryProtocol) Transport() TTransport {
-	return p.origTransport
-}
-
-func (p *TBinaryProtocol) readAll(buf []byte) error {
-	_, err := io.ReadFull(p.trans, buf)
-	return NewTProtocolException(err)
-}
-
-const readLimit = 32768
-
-func (p *TBinaryProtocol) readStringBody(size int32) (value string, err error) {
-	if size < 0 {
-		return "", nil
-	}
-
-	var (
-		buf bytes.Buffer
-		e   error
-		b   []byte
-	)
-
-	switch {
-	case int(size) <= len(p.buffer):
-		b = p.buffer[:size] // avoids allocation for small reads
-	case int(size) < readLimit:
-		b = make([]byte, size)
-	default:
-		b = make([]byte, readLimit)
-	}
-
-	for size > 0 {
-		_, e = io.ReadFull(p.trans, b)
-		buf.Write(b)
-		if e != nil {
-			break
-		}
-		size -= readLimit
-		if size < readLimit && size > 0 {
-			b = b[:size]
-		}
-	}
-	return buf.String(), NewTProtocolException(e)
-}

src/runtime/vendor/github.com/apache/thrift/lib/go/thrift/debug_protocol.go

@@ -1,270 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package thrift
-
-import (
-	"context"
-	"log"
-)
-
-type TDebugProtocol struct {
-	Delegate  TProtocol
-	LogPrefix string
-}
-
-type TDebugProtocolFactory struct {
-	Underlying TProtocolFactory
-	LogPrefix  string
-}
-
-func NewTDebugProtocolFactory(underlying TProtocolFactory, logPrefix string) *TDebugProtocolFactory {
-	return &TDebugProtocolFactory{
-		Underlying: underlying,
-		LogPrefix:  logPrefix,
-	}
-}
-
-func (t *TDebugProtocolFactory) GetProtocol(trans TTransport) TProtocol {
-	return &TDebugProtocol{
-		Delegate:  t.Underlying.GetProtocol(trans),
-		LogPrefix: t.LogPrefix,
-	}
-}
-
-func (tdp *TDebugProtocol) WriteMessageBegin(name string, typeId TMessageType, seqid int32) error {
-	err := tdp.Delegate.WriteMessageBegin(name, typeId, seqid)
-	log.Printf("%sWriteMessageBegin(name=%#v, typeId=%#v, seqid=%#v) => %#v", tdp.LogPrefix, name, typeId, seqid, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteMessageEnd() error {
-	err := tdp.Delegate.WriteMessageEnd()
-	log.Printf("%sWriteMessageEnd() => %#v", tdp.LogPrefix, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteStructBegin(name string) error {
-	err := tdp.Delegate.WriteStructBegin(name)
-	log.Printf("%sWriteStructBegin(name=%#v) => %#v", tdp.LogPrefix, name, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteStructEnd() error {
-	err := tdp.Delegate.WriteStructEnd()
-	log.Printf("%sWriteStructEnd() => %#v", tdp.LogPrefix, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteFieldBegin(name string, typeId TType, id int16) error {
-	err := tdp.Delegate.WriteFieldBegin(name, typeId, id)
-	log.Printf("%sWriteFieldBegin(name=%#v, typeId=%#v, id%#v) => %#v", tdp.LogPrefix, name, typeId, id, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteFieldEnd() error {
-	err := tdp.Delegate.WriteFieldEnd()
-	log.Printf("%sWriteFieldEnd() => %#v", tdp.LogPrefix, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteFieldStop() error {
-	err := tdp.Delegate.WriteFieldStop()
-	log.Printf("%sWriteFieldStop() => %#v", tdp.LogPrefix, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteMapBegin(keyType TType, valueType TType, size int) error {
-	err := tdp.Delegate.WriteMapBegin(keyType, valueType, size)
-	log.Printf("%sWriteMapBegin(keyType=%#v, valueType=%#v, size=%#v) => %#v", tdp.LogPrefix, keyType, valueType, size, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteMapEnd() error {
-	err := tdp.Delegate.WriteMapEnd()
-	log.Printf("%sWriteMapEnd() => %#v", tdp.LogPrefix, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteListBegin(elemType TType, size int) error {
-	err := tdp.Delegate.WriteListBegin(elemType, size)
-	log.Printf("%sWriteListBegin(elemType=%#v, size=%#v) => %#v", tdp.LogPrefix, elemType, size, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteListEnd() error {
-	err := tdp.Delegate.WriteListEnd()
-	log.Printf("%sWriteListEnd() => %#v", tdp.LogPrefix, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteSetBegin(elemType TType, size int) error {
-	err := tdp.Delegate.WriteSetBegin(elemType, size)
-	log.Printf("%sWriteSetBegin(elemType=%#v, size=%#v) => %#v", tdp.LogPrefix, elemType, size, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteSetEnd() error {
-	err := tdp.Delegate.WriteSetEnd()
-	log.Printf("%sWriteSetEnd() => %#v", tdp.LogPrefix, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteBool(value bool) error {
-	err := tdp.Delegate.WriteBool(value)
-	log.Printf("%sWriteBool(value=%#v) => %#v", tdp.LogPrefix, value, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteByte(value int8) error {
-	err := tdp.Delegate.WriteByte(value)
-	log.Printf("%sWriteByte(value=%#v) => %#v", tdp.LogPrefix, value, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteI16(value int16) error {
-	err := tdp.Delegate.WriteI16(value)
-	log.Printf("%sWriteI16(value=%#v) => %#v", tdp.LogPrefix, value, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteI32(value int32) error {
-	err := tdp.Delegate.WriteI32(value)
-	log.Printf("%sWriteI32(value=%#v) => %#v", tdp.LogPrefix, value, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteI64(value int64) error {
-	err := tdp.Delegate.WriteI64(value)
-	log.Printf("%sWriteI64(value=%#v) => %#v", tdp.LogPrefix, value, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteDouble(value float64) error {
-	err := tdp.Delegate.WriteDouble(value)
-	log.Printf("%sWriteDouble(value=%#v) => %#v", tdp.LogPrefix, value, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteString(value string) error {
-	err := tdp.Delegate.WriteString(value)
-	log.Printf("%sWriteString(value=%#v) => %#v", tdp.LogPrefix, value, err)
-	return err
-}
-func (tdp *TDebugProtocol) WriteBinary(value []byte) error {
-	err := tdp.Delegate.WriteBinary(value)
-	log.Printf("%sWriteBinary(value=%#v) => %#v", tdp.LogPrefix, value, err)
-	return err
-}
-
-func (tdp *TDebugProtocol) ReadMessageBegin() (name string, typeId TMessageType, seqid int32, err error) {
-	name, typeId, seqid, err = tdp.Delegate.ReadMessageBegin()
-	log.Printf("%sReadMessageBegin() (name=%#v, typeId=%#v, seqid=%#v, err=%#v)", tdp.LogPrefix, name, typeId, seqid, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadMessageEnd() (err error) {
-	err = tdp.Delegate.ReadMessageEnd()
-	log.Printf("%sReadMessageEnd() err=%#v", tdp.LogPrefix, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadStructBegin() (name string, err error) {
-	name, err = tdp.Delegate.ReadStructBegin()
-	log.Printf("%sReadStructBegin() (name%#v, err=%#v)", tdp.LogPrefix, name, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadStructEnd() (err error) {
-	err = tdp.Delegate.ReadStructEnd()
-	log.Printf("%sReadStructEnd() err=%#v", tdp.LogPrefix, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadFieldBegin() (name string, typeId TType, id int16, err error) {
-	name, typeId, id, err = tdp.Delegate.ReadFieldBegin()
-	log.Printf("%sReadFieldBegin() (name=%#v, typeId=%#v, id=%#v, err=%#v)", tdp.LogPrefix, name, typeId, id, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadFieldEnd() (err error) {
-	err = tdp.Delegate.ReadFieldEnd()
-	log.Printf("%sReadFieldEnd() err=%#v", tdp.LogPrefix, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadMapBegin() (keyType TType, valueType TType, size int, err error) {
-	keyType, valueType, size, err = tdp.Delegate.ReadMapBegin()
-	log.Printf("%sReadMapBegin() (keyType=%#v, valueType=%#v, size=%#v, err=%#v)", tdp.LogPrefix, keyType, valueType, size, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadMapEnd() (err error) {
-	err = tdp.Delegate.ReadMapEnd()
-	log.Printf("%sReadMapEnd() err=%#v", tdp.LogPrefix, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadListBegin() (elemType TType, size int, err error) {
-	elemType, size, err = tdp.Delegate.ReadListBegin()
-	log.Printf("%sReadListBegin() (elemType=%#v, size=%#v, err=%#v)", tdp.LogPrefix, elemType, size, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadListEnd() (err error) {
-	err = tdp.Delegate.ReadListEnd()
-	log.Printf("%sReadListEnd() err=%#v", tdp.LogPrefix, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadSetBegin() (elemType TType, size int, err error) {
-	elemType, size, err = tdp.Delegate.ReadSetBegin()
-	log.Printf("%sReadSetBegin() (elemType=%#v, size=%#v, err=%#v)", tdp.LogPrefix, elemType, size, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadSetEnd() (err error) {
-	err = tdp.Delegate.ReadSetEnd()
-	log.Printf("%sReadSetEnd() err=%#v", tdp.LogPrefix, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadBool() (value bool, err error) {
-	value, err = tdp.Delegate.ReadBool()
-	log.Printf("%sReadBool() (value=%#v, err=%#v)", tdp.LogPrefix, value, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadByte() (value int8, err error) {
-	value, err = tdp.Delegate.ReadByte()
-	log.Printf("%sReadByte() (value=%#v, err=%#v)", tdp.LogPrefix, value, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadI16() (value int16, err error) {
-	value, err = tdp.Delegate.ReadI16()
-	log.Printf("%sReadI16() (value=%#v, err=%#v)", tdp.LogPrefix, value, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadI32() (value int32, err error) {
-	value, err = tdp.Delegate.ReadI32()
-	log.Printf("%sReadI32() (value=%#v, err=%#v)", tdp.LogPrefix, value, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadI64() (value int64, err error) {
-	value, err = tdp.Delegate.ReadI64()
-	log.Printf("%sReadI64() (value=%#v, err=%#v)", tdp.LogPrefix, value, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadDouble() (value float64, err error) {
-	value, err = tdp.Delegate.ReadDouble()
-	log.Printf("%sReadDouble() (value=%#v, err=%#v)", tdp.LogPrefix, value, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadString() (value string, err error) {
-	value, err = tdp.Delegate.ReadString()
-	log.Printf("%sReadString() (value=%#v, err=%#v)", tdp.LogPrefix, value, err)
-	return
-}
-func (tdp *TDebugProtocol) ReadBinary() (value []byte, err error) {
-	value, err = tdp.Delegate.ReadBinary()
-	log.Printf("%sReadBinary() (value=%#v, err=%#v)", tdp.LogPrefix, value, err)
-	return
-}
-func (tdp *TDebugProtocol) Skip(fieldType TType) (err error) {
-	err = tdp.Delegate.Skip(fieldType)
-	log.Printf("%sSkip(fieldType=%#v) (err=%#v)", tdp.LogPrefix, fieldType, err)
-	return
-}
-func (tdp *TDebugProtocol) Flush(ctx context.Context) (err error) {
-	err = tdp.Delegate.Flush(ctx)
-	log.Printf("%sFlush() (err=%#v)", tdp.LogPrefix, err)
-	return
-}
-
-func (tdp *TDebugProtocol) Transport() TTransport {
-	return tdp.Delegate.Transport()
-}

src/runtime/vendor/github.com/apache/thrift/lib/go/thrift/deserializer.go

@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package thrift
-
-type TDeserializer struct {
-	Transport TTransport
-	Protocol  TProtocol
-}
-
-func NewTDeserializer() *TDeserializer {
-	var transport TTransport
-	transport = NewTMemoryBufferLen(1024)
-
-	protocol := NewTBinaryProtocolFactoryDefault().GetProtocol(transport)
-
-	return &TDeserializer{
-		transport,
-		protocol}
-}
-
-func (t *TDeserializer) ReadString(msg TStruct, s string) (err error) {
-	err = nil
-	if _, err = t.Transport.Write([]byte(s)); err != nil {
-		return
-	}
-	if err = msg.Read(t.Protocol); err != nil {
-		return
-	}
-	return
-}
-
-func (t *TDeserializer) Read(msg TStruct, b []byte) (err error) {
-	err = nil
-	if _, err = t.Transport.Write(b); err != nil {
-		return
-	}
-	if err = msg.Read(t.Protocol); err != nil {
-		return
-	}
-	return
-}

src/runtime/vendor/github.com/apache/thrift/lib/go/thrift/field.go

@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package thrift
-
-// Helper class that encapsulates field metadata.
-type field struct {
-	name   string
-	typeId TType
-	id     int
-}
-
-func newField(n string, t TType, i int) *field {
-	return &field{name: n, typeId: t, id: i}
-}
-
-func (p *field) Name() string {
-	if p == nil {
-		return ""
-	}
-	return p.name
-}
-
-func (p *field) TypeId() TType {
-	if p == nil {
-		return TType(VOID)
-	}
-	return p.typeId
-}
-
-func (p *field) Id() int {
-	if p == nil {
-		return -1
-	}
-	return p.id
-}
-
-func (p *field) String() string {
-	if p == nil {
-		return "<nil>"
-	}
-	return "<TField name:'" + p.name + "' type:" + string(p.typeId) + " field-id:" + string(p.id) + ">"
-}
-
-var ANONYMOUS_FIELD *field
-
-type fieldSlice []field
-
-func (p fieldSlice) Len() int {
-	return len(p)
-}
-
-func (p fieldSlice) Less(i, j int) bool {
-	return p[i].Id() < p[j].Id()
-}
-
-func (p fieldSlice) Swap(i, j int) {
-	p[i], p[j] = p[j], p[i]
-}
-
-func init() {
-	ANONYMOUS_FIELD = newField("", STOP, 0)
-}

src/runtime/vendor/github.com/apache/thrift/lib/go/thrift/framed_transport.go

@@ -1,187 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package thrift
-
-import (
-	"bufio"
-	"bytes"
-	"context"
-	"encoding/binary"
-	"fmt"
-	"io"
-)
-
-const DEFAULT_MAX_LENGTH = 16384000
-
-type TFramedTransport struct {
-	transport TTransport
-	buf       bytes.Buffer
-	reader    *bufio.Reader
-	frameSize uint32 //Current remaining size of the frame. if ==0 read next frame header
-	buffer    [4]byte
-	maxLength uint32
-}
-
-type tFramedTransportFactory struct {
-	factory   TTransportFactory
-	maxLength uint32
-}
-
-func NewTFramedTransportFactory(factory TTransportFactory) TTransportFactory {
-	return &tFramedTransportFactory{factory: factory, maxLength: DEFAULT_MAX_LENGTH}
-}
-
-func NewTFramedTransportFactoryMaxLength(factory TTransportFactory, maxLength uint32) TTransportFactory {
-	return &tFramedTransportFactory{factory: factory, maxLength: maxLength}
-}
-
-func (p *tFramedTransportFactory) GetTransport(base TTransport) (TTransport, error) {
-	tt, err := p.factory.GetTransport(base)
-	if err != nil {
-		return nil, err
-	}
-	return NewTFramedTransportMaxLength(tt, p.maxLength), nil
-}
-
-func NewTFramedTransport(transport TTransport) *TFramedTransport {
-	return &TFramedTransport{transport: transport, reader: bufio.NewReader(transport), maxLength: DEFAULT_MAX_LENGTH}
-}
-
-func NewTFramedTransportMaxLength(transport TTransport, maxLength uint32) *TFramedTransport {
-	return &TFramedTransport{transport: transport, reader: bufio.NewReader(transport), maxLength: maxLength}
-}
-
-func (p *TFramedTransport) Open() error {
-	return p.transport.Open()
-}
-
-func (p *TFramedTransport) IsOpen() bool {
-	return p.transport.IsOpen()
-}
-
-func (p *TFramedTransport) Close() error {
-	return p.transport.Close()
-}
-
-func (p *TFramedTransport) Read(buf []byte) (l int, err error) {
-	if p.frameSize == 0 {
-		p.frameSize, err = p.readFrameHeader()
-		if err != nil {
-			return
-		}
-	}
-	if p.frameSize < uint32(len(buf)) {
-		frameSize := p.frameSize
-		tmp := make([]byte, p.frameSize)
-		l, err = p.Read(tmp)
-		copy(buf, tmp)
-		if err == nil {
-			// Note: It's important to only return an error when l
-			// is zero.
-			// In io.Reader.Read interface, it's perfectly fine to
-			// return partial data and nil error, which means
-			// "This is all the data we have right now without
-			// blocking. If you need the full data, call Read again
-			// or use io.ReadFull instead".
-			// Returning partial data with an error actually means
-			// there's no more data after the partial data just
-			// returned, which is not true in this case
-			// (it might be that the other end just haven't written
-			// them yet).
-			if l == 0 {
-				err = NewTTransportExceptionFromError(fmt.Errorf("Not enough frame size %d to read %d bytes", frameSize, len(buf)))
-			}
-			return
-		}
-	}
-	got, err := p.reader.Read(buf)
-	p.frameSize = p.frameSize - uint32(got)
-	//sanity check
-	if p.frameSize < 0 {
-		return 0, NewTTransportException(UNKNOWN_TRANSPORT_EXCEPTION, "Negative frame size")
-	}
-	return got, NewTTransportExceptionFromError(err)
-}
-
-func (p *TFramedTransport) ReadByte() (c byte, err error) {
-	if p.frameSize == 0 {
-		p.frameSize, err = p.readFrameHeader()
-		if err != nil {
-			return
-		}
-	}
-	if p.frameSize < 1 {
-		return 0, NewTTransportExceptionFromError(fmt.Errorf("Not enough frame size %d to read %d bytes", p.frameSize, 1))
-	}
-	c, err = p.reader.ReadByte()
-	if err == nil {
-		p.frameSize--
-	}
-	return
-}
-
-func (p *TFramedTransport) Write(buf []byte) (int, error) {
-	n, err := p.buf.Write(buf)
-	return n, NewTTransportExceptionFromError(err)
-}
-
-func (p *TFramedTransport) WriteByte(c byte) error {
-	return p.buf.WriteByte(c)
-}
-
-func (p *TFramedTransport) WriteString(s string) (n int, err error) {
-	return p.buf.WriteString(s)
-}
-
-func (p *TFramedTransport) Flush(ctx context.Context) error {
-	size := p.buf.Len()
-	buf := p.buffer[:4]
-	binary.BigEndian.PutUint32(buf, uint32(size))
-	_, err := p.transport.Write(buf)
-	if err != nil {
-		p.buf.Truncate(0)
-		return NewTTransportExceptionFromError(err)
-	}
-	if size > 0 {
-		if n, err := p.buf.WriteTo(p.transport); err != nil {
-			print("Error while flushing write buffer of size ", size, " to transport, only wrote ", n, " bytes: ", err.Error(), "\n")
-			p.buf.Truncate(0)
-			return NewTTransportExceptionFromError(err)
-		}
-	}
-	err = p.transport.Flush(ctx)
-	return NewTTransportExceptionFromError(err)
-}
-
-func (p *TFramedTransport) readFrameHeader() (uint32, error) {
-	buf := p.buffer[:4]
-	if _, err := io.ReadFull(p.reader, buf); err != nil {
-		return 0, err
-	}
-	size := binary.BigEndian.Uint32(buf)
-	if size < 0 || size > p.maxLength {
-		return 0, NewTTransportException(UNKNOWN_TRANSPORT_EXCEPTION, fmt.Sprintf("Incorrect frame size (%d)", size))
-	}
-	return size, nil
-}
-
-func (p *TFramedTransport) RemainingBytes() (num_bytes uint64) {
-	return uint64(p.frameSize)
-}

src/runtime/vendor/github.com/apache/thrift/lib/go/thrift/header_protocol.go

@@ -1,305 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package thrift
-
-import (
-	"context"
-)
-
-// THeaderProtocol is a thrift protocol that implements THeader:
-// https://github.com/apache/thrift/blob/master/doc/specs/HeaderFormat.md
-//
-// It supports either binary or compact protocol as the wrapped protocol.
-//
-// Most of the THeader handlings are happening inside THeaderTransport.
-type THeaderProtocol struct {
-	transport *THeaderTransport
-
-	// Will be initialized on first read/write.
-	protocol TProtocol
-}
-
-// NewTHeaderProtocol creates a new THeaderProtocol from the underlying
-// transport. The passed in transport will be wrapped with THeaderTransport.
-//
-// Note that THeaderTransport handles frame and zlib by itself,
-// so the underlying transport should be a raw socket transports (TSocket or TSSLSocket),
-// instead of rich transports like TZlibTransport or TFramedTransport.
-func NewTHeaderProtocol(trans TTransport) *THeaderProtocol {
-	t := NewTHeaderTransport(trans)
-	p, _ := THeaderProtocolDefault.GetProtocol(t)
-	return &THeaderProtocol{
-		transport: t,
-		protocol:  p,
-	}
-}
-
-type tHeaderProtocolFactory struct{}
-
-func (tHeaderProtocolFactory) GetProtocol(trans TTransport) TProtocol {
-	return NewTHeaderProtocol(trans)
-}
-
-// NewTHeaderProtocolFactory creates a factory for THeader.
-//
-// It's a wrapper for NewTHeaderProtocol
-func NewTHeaderProtocolFactory() TProtocolFactory {
-	return tHeaderProtocolFactory{}
-}
-
-// Transport returns the underlying transport.
-//
-// It's guaranteed to be of type *THeaderTransport.
-func (p *THeaderProtocol) Transport() TTransport {
-	return p.transport
-}
-
-// GetReadHeaders returns the THeaderMap read from transport.
-func (p *THeaderProtocol) GetReadHeaders() THeaderMap {
-	return p.transport.GetReadHeaders()
-}
-
-// SetWriteHeader sets a header for write.
-func (p *THeaderProtocol) SetWriteHeader(key, value string) {
-	p.transport.SetWriteHeader(key, value)
-}
-
-// ClearWriteHeaders clears all write headers previously set.
-func (p *THeaderProtocol) ClearWriteHeaders() {
-	p.transport.ClearWriteHeaders()
-}
-
-// AddTransform add a transform for writing.
-func (p *THeaderProtocol) AddTransform(transform THeaderTransformID) error {
-	return p.transport.AddTransform(transform)
-}
-
-func (p *THeaderProtocol) Flush(ctx context.Context) error {
-	return p.transport.Flush(ctx)
-}
-
-func (p *THeaderProtocol) WriteMessageBegin(name string, typeID TMessageType, seqID int32) error {
-	newProto, err := p.transport.Protocol().GetProtocol(p.transport)
-	if err != nil {
-		return err
-	}
-	p.protocol = newProto
-	p.transport.SequenceID = seqID
-	return p.protocol.WriteMessageBegin(name, typeID, seqID)
-}
-
-func (p *THeaderProtocol) WriteMessageEnd() error {
-	if err := p.protocol.WriteMessageEnd(); err != nil {
-		return err
-	}
-	return p.transport.Flush(context.Background())
-}
-
-func (p *THeaderProtocol) WriteStructBegin(name string) error {
-	return p.protocol.WriteStructBegin(name)
-}
-
-func (p *THeaderProtocol) WriteStructEnd() error {
-	return p.protocol.WriteStructEnd()
-}
-
-func (p *THeaderProtocol) WriteFieldBegin(name string, typeID TType, id int16) error {
-	return p.protocol.WriteFieldBegin(name, typeID, id)
-}
-
-func (p *THeaderProtocol) WriteFieldEnd() error {
-	return p.protocol.WriteFieldEnd()
-}
-
-func (p *THeaderProtocol) WriteFieldStop() error {
-	return p.protocol.WriteFieldStop()
-}
-
-func (p *THeaderProtocol) WriteMapBegin(keyType TType, valueType TType, size int) error {
-	return p.protocol.WriteMapBegin(keyType, valueType, size)
-}
-
-func (p *THeaderProtocol) WriteMapEnd() error {
-	return p.protocol.WriteMapEnd()
-}
-
-func (p *THeaderProtocol) WriteListBegin(elemType TType, size int) error {
-	return p.protocol.WriteListBegin(elemType, size)
-}
-
-func (p *THeaderProtocol) WriteListEnd() error {
-	return p.protocol.WriteListEnd()
-}
-
-func (p *THeaderProtocol) WriteSetBegin(elemType TType, size int) error {
-	return p.protocol.WriteSetBegin(elemType, size)
-}
-
-func (p *THeaderProtocol) WriteSetEnd() error {
-	return p.protocol.WriteSetEnd()
-}
-
-func (p *THeaderProtocol) WriteBool(value bool) error {
-	return p.protocol.WriteBool(value)
-}
-
-func (p *THeaderProtocol) WriteByte(value int8) error {
-	return p.protocol.WriteByte(value)
-}
-
-func (p *THeaderProtocol) WriteI16(value int16) error {
-	return p.protocol.WriteI16(value)
-}
-
-func (p *THeaderProtocol) WriteI32(value int32) error {
-	return p.protocol.WriteI32(value)
-}
-
-func (p *THeaderProtocol) WriteI64(value int64) error {
-	return p.protocol.WriteI64(value)
-}
-
-func (p *THeaderProtocol) WriteDouble(value float64) error {
-	return p.protocol.WriteDouble(value)
-}
-
-func (p *THeaderProtocol) WriteString(value string) error {
-	return p.protocol.WriteString(value)
-}
-
-func (p *THeaderProtocol) WriteBinary(value []byte) error {
-	return p.protocol.WriteBinary(value)
-}
-
-// ReadFrame calls underlying THeaderTransport's ReadFrame function.
-func (p *THeaderProtocol) ReadFrame() error {
-	return p.transport.ReadFrame()
-}
-
-func (p *THeaderProtocol) ReadMessageBegin() (name string, typeID TMessageType, seqID int32, err error) {
-	if err = p.transport.ReadFrame(); err != nil {
-		return
-	}
-
-	var newProto TProtocol
-	newProto, err = p.transport.Protocol().GetProtocol(p.transport)
-	if err != nil {
-		tAppExc, ok := err.(TApplicationException)
-		if !ok {
-			return
-		}
-		if e := p.protocol.WriteMessageBegin("", EXCEPTION, seqID); e != nil {
-			return
-		}
-		if e := tAppExc.Write(p.protocol); e != nil {
-			return
-		}
-		if e := p.protocol.WriteMessageEnd(); e != nil {
-			return
-		}
-		if e := p.transport.Flush(context.Background()); e != nil {
-			return
-		}
-		return
-	}
-	p.protocol = newProto
-
-	return p.protocol.ReadMessageBegin()
-}
-
-func (p *THeaderProtocol) ReadMessageEnd() error {
-	return p.protocol.ReadMessageEnd()
-}
-
-func (p *THeaderProtocol) ReadStructBegin() (name string, err error) {
-	return p.protocol.ReadStructBegin()
-}
-
-func (p *THeaderProtocol) ReadStructEnd() error {
-	return p.protocol.ReadStructEnd()
-}
-
-func (p *THeaderProtocol) ReadFieldBegin() (name string, typeID TType, id int16, err error) {
-	return p.protocol.ReadFieldBegin()
-}
-
-func (p *THeaderProtocol) ReadFieldEnd() error {
-	return p.protocol.ReadFieldEnd()
-}
-
-func (p *THeaderProtocol) ReadMapBegin() (keyType TType, valueType TType, size int, err error) {
-	return p.protocol.ReadMapBegin()
-}
-
-func (p *THeaderProtocol) ReadMapEnd() error {
-	return p.protocol.ReadMapEnd()
-}
-
-func (p *THeaderProtocol) ReadListBegin() (elemType TType, size int, err error) {
-	return p.protocol.ReadListBegin()
-}
-
-func (p *THeaderProtocol) ReadListEnd() error {
-	return p.protocol.ReadListEnd()
-}
-
-func (p *THeaderProtocol) ReadSetBegin() (elemType TType, size int, err error) {
-	return p.protocol.ReadSetBegin()
-}
-
-func (p *THeaderProtocol) ReadSetEnd() error {
-	return p.protocol.ReadSetEnd()
-}
-
-func (p *THeaderProtocol) ReadBool() (value bool, err error) {
-	return p.protocol.ReadBool()
-}
-
-func (p *THeaderProtocol) ReadByte() (value int8, err error) {
-	return p.protocol.ReadByte()
-}
-
-func (p *THeaderProtocol) ReadI16() (value int16, err error) {
-	return p.protocol.ReadI16()
-}
-
-func (p *THeaderProtocol) ReadI32() (value int32, err error) {
-	return p.protocol.ReadI32()
-}
-
-func (p *THeaderProtocol) ReadI64() (value int64, err error) {
-	return p.protocol.ReadI64()
-}
-
-func (p *THeaderProtocol) ReadDouble() (value float64, err error) {
-	return p.protocol.ReadDouble()
-}
-
-func (p *THeaderProtocol) ReadString() (value string, err error) {
-	return p.protocol.ReadString()
-}
-
-func (p *THeaderProtocol) ReadBinary() (value []byte, err error) {
-	return p.protocol.ReadBinary()
-}
-
-func (p *THeaderProtocol) Skip(fieldType TType) error {
-	return p.protocol.Skip(fieldType)
-}

src/runtime/vendor/github.com/apache/thrift/lib/go/thrift/protocol.go

@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package thrift
-
-import (
-	"context"
-	"errors"
-	"fmt"
-)
-
-const (
-	VERSION_MASK = 0xffff0000
-	VERSION_1    = 0x80010000
-)
-
-type TProtocol interface {
-	WriteMessageBegin(name string, typeId TMessageType, seqid int32) error
-	WriteMessageEnd() error
-	WriteStructBegin(name string) error
-	WriteStructEnd() error
-	WriteFieldBegin(name string, typeId TType, id int16) error
-	WriteFieldEnd() error
-	WriteFieldStop() error
-	WriteMapBegin(keyType TType, valueType TType, size int) error
-	WriteMapEnd() error
-	WriteListBegin(elemType TType, size int) error
-	WriteListEnd() error
-	WriteSetBegin(elemType TType, size int) error
-	WriteSetEnd() error
-	WriteBool(value bool) error
-	WriteByte(value int8) error
-	WriteI16(value int16) error
-	WriteI32(value int32) error
-	WriteI64(value int64) error
-	WriteDouble(value float64) error
-	WriteString(value string) error
-	WriteBinary(value []byte) error
-
-	ReadMessageBegin() (name string, typeId TMessageType, seqid int32, err error)
-	ReadMessageEnd() error
-	ReadStructBegin() (name string, err error)
-	ReadStructEnd() error
-	ReadFieldBegin() (name string, typeId TType, id int16, err error)
-	ReadFieldEnd() error
-	ReadMapBegin() (keyType TType, valueType TType, size int, err error)
-	ReadMapEnd() error
-	ReadListBegin() (elemType TType, size int, err error)
-	ReadListEnd() error
-	ReadSetBegin() (elemType TType, size int, err error)
-	ReadSetEnd() error
-	ReadBool() (value bool, err error)
-	ReadByte() (value int8, err error)
-	ReadI16() (value int16, err error)
-	ReadI32() (value int32, err error)
-	ReadI64() (value int64, err error)
-	ReadDouble() (value float64, err error)
-	ReadString() (value string, err error)
-	ReadBinary() (value []byte, err error)
-
-	Skip(fieldType TType) (err error)
-	Flush(ctx context.Context) (err error)
-
-	Transport() TTransport
-}
-
-// The maximum recursive depth the skip() function will traverse
-const DEFAULT_RECURSION_DEPTH = 64
-
-// Skips over the next data element from the provided input TProtocol object.
-func SkipDefaultDepth(prot TProtocol, typeId TType) (err error) {
-	return Skip(prot, typeId, DEFAULT_RECURSION_DEPTH)
-}
-
-// Skips over the next data element from the provided input TProtocol object.
-func Skip(self TProtocol, fieldType TType, maxDepth int) (err error) {
-
-	if maxDepth <= 0 {
-		return NewTProtocolExceptionWithType(DEPTH_LIMIT, errors.New("Depth limit exceeded"))
-	}
-
-	switch fieldType {
-	case BOOL:
-		_, err = self.ReadBool()
-		return
-	case BYTE:
-		_, err = self.ReadByte()
-		return
-	case I16:
-		_, err = self.ReadI16()
-		return
-	case I32:
-		_, err = self.ReadI32()
-		return
-	case I64:
-		_, err = self.ReadI64()
-		return
-	case DOUBLE:
-		_, err = self.ReadDouble()
-		return
-	case STRING:
-		_, err = self.ReadString()
-		return
-	case STRUCT:
-		if _, err = self.ReadStructBegin(); err != nil {
-			return err
-		}
-		for {
-			_, typeId, _, _ := self.ReadFieldBegin()
-			if typeId == STOP {
-				break
-			}
-			err := Skip(self, typeId, maxDepth-1)
-			if err != nil {
-				return err
-			}
-			self.ReadFieldEnd()
-		}
-		return self.ReadStructEnd()
-	case MAP:
-		keyType, valueType, size, err := self.ReadMapBegin()
-		if err != nil {
-			return err
-		}
-		for i := 0; i < size; i++ {
-			err := Skip(self, keyType, maxDepth-1)
-			if err != nil {
-				return err
-			}
-			self.Skip(valueType)
-		}
-		return self.ReadMapEnd()
-	case SET:
-		elemType, size, err := self.ReadSetBegin()
-		if err != nil {
-			return err
-		}
-		for i := 0; i < size; i++ {
-			err := Skip(self, elemType, maxDepth-1)
-			if err != nil {
-				return err
-			}
-		}
-		return self.ReadSetEnd()
-	case LIST:
-		elemType, size, err := self.ReadListBegin()
-		if err != nil {
-			return err
-		}
-		for i := 0; i < size; i++ {
-			err := Skip(self, elemType, maxDepth-1)
-			if err != nil {
-				return err
-			}
-		}
-		return self.ReadListEnd()
-	default:
-		return NewTProtocolExceptionWithType(INVALID_DATA, errors.New(fmt.Sprintf("Unknown data type %d", fieldType)))
-	}
-	return nil
-}

src/runtime/vendor/github.com/apache/thrift/lib/go/thrift/serializer.go

@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package thrift
-
-import (
-	"context"
-)
-
-type TSerializer struct {
-	Transport *TMemoryBuffer
-	Protocol  TProtocol
-}
-
-type TStruct interface {
-	Write(p TProtocol) error
-	Read(p TProtocol) error
-}
-
-func NewTSerializer() *TSerializer {
-	transport := NewTMemoryBufferLen(1024)
-	protocol := NewTBinaryProtocolFactoryDefault().GetProtocol(transport)
-
-	return &TSerializer{
-		transport,
-		protocol}
-}
-
-func (t *TSerializer) WriteString(ctx context.Context, msg TStruct) (s string, err error) {
-	t.Transport.Reset()
-
-	if err = msg.Write(t.Protocol); err != nil {
-		return
-	}
-
-	if err = t.Protocol.Flush(ctx); err != nil {
-		return
-	}
-	if err = t.Transport.Flush(ctx); err != nil {
-		return
-	}
-
-	return t.Transport.String(), nil
-}
-
-func (t *TSerializer) Write(ctx context.Context, msg TStruct) (b []byte, err error) {
-	t.Transport.Reset()
-
-	if err = msg.Write(t.Protocol); err != nil {
-		return
-	}
-
-	if err = t.Protocol.Flush(ctx); err != nil {
-		return
-	}
-
-	if err = t.Transport.Flush(ctx); err != nil {
-		return
-	}
-
-	b = append(b, t.Transport.Bytes()...)
-	return
-}

src/runtime/vendor/github.com/apache/thrift/lib/go/thrift/socket.go

@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package thrift
-
-import (
-	"context"
-	"net"
-	"time"
-)
-
-type TSocket struct {
-	conn    net.Conn
-	addr    net.Addr
-	timeout time.Duration
-}
-
-// NewTSocket creates a net.Conn-backed TTransport, given a host and port
-//
-// Example:
-// 	trans, err := thrift.NewTSocket("localhost:9090")
-func NewTSocket(hostPort string) (*TSocket, error) {
-	return NewTSocketTimeout(hostPort, 0)
-}
-
-// NewTSocketTimeout creates a net.Conn-backed TTransport, given a host and port
-// it also accepts a timeout as a time.Duration
-func NewTSocketTimeout(hostPort string, timeout time.Duration) (*TSocket, error) {
-	//conn, err := net.DialTimeout(network, address, timeout)
-	addr, err := net.ResolveTCPAddr("tcp", hostPort)
-	if err != nil {
-		return nil, err
-	}
-	return NewTSocketFromAddrTimeout(addr, timeout), nil
-}
-
-// Creates a TSocket from a net.Addr
-func NewTSocketFromAddrTimeout(addr net.Addr, timeout time.Duration) *TSocket {
-	return &TSocket{addr: addr, timeout: timeout}
-}
-
-// Creates a TSocket from an existing net.Conn
-func NewTSocketFromConnTimeout(conn net.Conn, timeout time.Duration) *TSocket {
-	return &TSocket{conn: conn, addr: conn.RemoteAddr(), timeout: timeout}
-}
-
-// Sets the socket timeout
-func (p *TSocket) SetTimeout(timeout time.Duration) error {
-	p.timeout = timeout
-	return nil
-}
-
-func (p *TSocket) pushDeadline(read, write bool) {
-	var t time.Time
-	if p.timeout > 0 {
-		t = time.Now().Add(time.Duration(p.timeout))
-	}
-	if read && write {
-		p.conn.SetDeadline(t)
-	} else if read {
-		p.conn.SetReadDeadline(t)
-	} else if write {
-		p.conn.SetWriteDeadline(t)
-	}
-}
-
-// Connects the socket, creating a new socket object if necessary.
-func (p *TSocket) Open() error {
-	if p.IsOpen() {
-		return NewTTransportException(ALREADY_OPEN, "Socket already connected.")
-	}
-	if p.addr == nil {
-		return NewTTransportException(NOT_OPEN, "Cannot open nil address.")
-	}
-	if len(p.addr.Network()) == 0 {
-		return NewTTransportException(NOT_OPEN, "Cannot open bad network name.")
-	}
-	if len(p.addr.String()) == 0 {
-		return NewTTransportException(NOT_OPEN, "Cannot open bad address.")
-	}
-	var err error
-	if p.conn, err = net.DialTimeout(p.addr.Network(), p.addr.String(), p.timeout); err != nil {
-		return NewTTransportException(NOT_OPEN, err.Error())
-	}
-	return nil
-}
-
-// Retrieve the underlying net.Conn
-func (p *TSocket) Conn() net.Conn {
-	return p.conn
-}
-
-// Returns true if the connection is open
-func (p *TSocket) IsOpen() bool {
-	if p.conn == nil {
-		return false
-	}
-	return true
-}
-
-// Closes the socket.
-func (p *TSocket) Close() error {
-	// Close the socket
-	if p.conn != nil {
-		err := p.conn.Close()
-		if err != nil {
-			return err
-		}
-		p.conn = nil
-	}
-	return nil
-}
-
-//Returns the remote address of the socket.
-func (p *TSocket) Addr() net.Addr {
-	return p.addr
-}
-
-func (p *TSocket) Read(buf []byte) (int, error) {
-	if !p.IsOpen() {
-		return 0, NewTTransportException(NOT_OPEN, "Connection not open")
-	}
-	p.pushDeadline(true, false)
-	n, err := p.conn.Read(buf)
-	return n, NewTTransportExceptionFromError(err)
-}
-
-func (p *TSocket) Write(buf []byte) (int, error) {
-	if !p.IsOpen() {
-		return 0, NewTTransportException(NOT_OPEN, "Connection not open")
-	}
-	p.pushDeadline(false, true)
-	return p.conn.Write(buf)
-}
-
-func (p *TSocket) Flush(ctx context.Context) error {
-	return nil
-}
-
-func (p *TSocket) Interrupt() error {
-	if !p.IsOpen() {
-		return nil
-	}
-	return p.conn.Close()
-}
-
-func (p *TSocket) RemainingBytes() (num_bytes uint64) {
-	const maxSize = ^uint64(0)
-	return maxSize // the truth is, we just don't know unless framed is used
-}

src/runtime/vendor/github.com/apache/thrift/lib/go/thrift/ssl_socket.go

@@ -1,176 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package thrift
-
-import (
-	"context"
-	"crypto/tls"
-	"net"
-	"time"
-)
-
-type TSSLSocket struct {
-	conn net.Conn
-	// hostPort contains host:port (e.g. "asdf.com:12345"). The field is
-	// only valid if addr is nil.
-	hostPort string
-	// addr is nil when hostPort is not "", and is only used when the
-	// TSSLSocket is constructed from a net.Addr.
-	addr    net.Addr
-	timeout time.Duration
-	cfg     *tls.Config
-}
-
-// NewTSSLSocket creates a net.Conn-backed TTransport, given a host and port and tls Configuration
-//
-// Example:
-// 	trans, err := thrift.NewTSSLSocket("localhost:9090", nil)
-func NewTSSLSocket(hostPort string, cfg *tls.Config) (*TSSLSocket, error) {
-	return NewTSSLSocketTimeout(hostPort, cfg, 0)
-}
-
-// NewTSSLSocketTimeout creates a net.Conn-backed TTransport, given a host and port
-// it also accepts a tls Configuration and a timeout as a time.Duration
-func NewTSSLSocketTimeout(hostPort string, cfg *tls.Config, timeout time.Duration) (*TSSLSocket, error) {
-	if cfg.MinVersion == 0 {
-		cfg.MinVersion = tls.VersionTLS10
-	}
-	return &TSSLSocket{hostPort: hostPort, timeout: timeout, cfg: cfg}, nil
-}
-
-// Creates a TSSLSocket from a net.Addr
-func NewTSSLSocketFromAddrTimeout(addr net.Addr, cfg *tls.Config, timeout time.Duration) *TSSLSocket {
-	return &TSSLSocket{addr: addr, timeout: timeout, cfg: cfg}
-}
-
-// Creates a TSSLSocket from an existing net.Conn
-func NewTSSLSocketFromConnTimeout(conn net.Conn, cfg *tls.Config, timeout time.Duration) *TSSLSocket {
-	return &TSSLSocket{conn: conn, addr: conn.RemoteAddr(), timeout: timeout, cfg: cfg}
-}
-
-// Sets the socket timeout
-func (p *TSSLSocket) SetTimeout(timeout time.Duration) error {
-	p.timeout = timeout
-	return nil
-}
-
-func (p *TSSLSocket) pushDeadline(read, write bool) {
-	var t time.Time
-	if p.timeout > 0 {
-		t = time.Now().Add(time.Duration(p.timeout))
-	}
-	if read && write {
-		p.conn.SetDeadline(t)
-	} else if read {
-		p.conn.SetReadDeadline(t)
-	} else if write {
-		p.conn.SetWriteDeadline(t)
-	}
-}
-
-// Connects the socket, creating a new socket object if necessary.
-func (p *TSSLSocket) Open() error {
-	var err error
-	// If we have a hostname, we need to pass the hostname to tls.Dial for
-	// certificate hostname checks.
-	if p.hostPort != "" {
-		if p.conn, err = tls.DialWithDialer(&net.Dialer{
-			Timeout: p.timeout}, "tcp", p.hostPort, p.cfg); err != nil {
-			return NewTTransportException(NOT_OPEN, err.Error())
-		}
-	} else {
-		if p.IsOpen() {
-			return NewTTransportException(ALREADY_OPEN, "Socket already connected.")
-		}
-		if p.addr == nil {
-			return NewTTransportException(NOT_OPEN, "Cannot open nil address.")
-		}
-		if len(p.addr.Network()) == 0 {
-			return NewTTransportException(NOT_OPEN, "Cannot open bad network name.")
-		}
-		if len(p.addr.String()) == 0 {
-			return NewTTransportException(NOT_OPEN, "Cannot open bad address.")
-		}
-		if p.conn, err = tls.DialWithDialer(&net.Dialer{
-			Timeout: p.timeout}, p.addr.Network(), p.addr.String(), p.cfg); err != nil {
-			return NewTTransportException(NOT_OPEN, err.Error())
-		}
-	}
-	return nil
-}
-
-// Retrieve the underlying net.Conn
-func (p *TSSLSocket) Conn() net.Conn {
-	return p.conn
-}
-
-// Returns true if the connection is open
-func (p *TSSLSocket) IsOpen() bool {
-	if p.conn == nil {
-		return false
-	}
-	return true
-}
-
-// Closes the socket.
-func (p *TSSLSocket) Close() error {
-	// Close the socket
-	if p.conn != nil {
-		err := p.conn.Close()
-		if err != nil {
-			return err
-		}
-		p.conn = nil
-	}
-	return nil
-}
-
-func (p *TSSLSocket) Read(buf []byte) (int, error) {
-	if !p.IsOpen() {
-		return 0, NewTTransportException(NOT_OPEN, "Connection not open")
-	}
-	p.pushDeadline(true, false)
-	n, err := p.conn.Read(buf)
-	return n, NewTTransportExceptionFromError(err)
-}
-
-func (p *TSSLSocket) Write(buf []byte) (int, error) {
-	if !p.IsOpen() {
-		return 0, NewTTransportException(NOT_OPEN, "Connection not open")
-	}
-	p.pushDeadline(false, true)
-	return p.conn.Write(buf)
-}
-
-func (p *TSSLSocket) Flush(ctx context.Context) error {
-	return nil
-}
-
-func (p *TSSLSocket) Interrupt() error {
-	if !p.IsOpen() {
-		return nil
-	}
-	return p.conn.Close()
-}
-
-func (p *TSSLSocket) RemainingBytes() (num_bytes uint64) {
-	const maxSize = ^uint64(0)
-	return maxSize // the thruth is, we just don't know unless framed is used
-}

src/runtime/vendor/github.com/cilium/ebpf/.clang-format

@@ -0,0 +1,17 @@
+---
+Language:        Cpp
+BasedOnStyle:    LLVM
+AlignAfterOpenBracket: DontAlign
+AlignConsecutiveAssignments: true
+AlignEscapedNewlines: DontAlign
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: false
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortFunctionsOnASingleLine: false
+BreakBeforeBraces: Attach
+IndentWidth:     4
+KeepEmptyLinesAtTheStartOfBlocks: false
+TabWidth:        4
+UseTab:          ForContinuationAndIndentation
+ColumnLimit:     1000
+...