Merge pull request #3150 from fidencio/2.3.0-branch-bump

# Kata Containers 2.3.0

.github/workflows/kata-deploy-push.yaml

@@ -9,6 +9,7 @@ jobs:
       matrix:
         asset:
           - kernel
+          - kernel-experimental
           - shim-v2
           - qemu
           - cloud-hypervisor

.github/workflows/kata-deploy-test.yaml

@@ -5,60 +5,121 @@ on:
 name: test-kata-deploy
 
 jobs:
-  check_comments:
-    if: ${{ github.event.issue.pull_request }}
+  check-comment-and-membership:
     runs-on: ubuntu-latest
+    if: |
+      github.event.issue.pull_request
+      && github.event_name == 'issue_comment'
+      && github.event.action == 'created'
+      && startsWith(github.event.comment.body, '/test_kata_deploy')
     steps:
-      - name: Check for Command
-        id: command
-        uses: kata-containers/slash-command-action@v1
+      - name: Check membership
+        uses: kata-containers/is-organization-member@1.0.1
+        id: is_organization_member
         with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          command: "test_kata_deploy"
-          reaction: "true"
-          reaction-type: "eyes"
-          allow-edits: "false"
-          permission-level: admin
-      - name: verify command arg is kata-deploy
+          organization: kata-containers
+          username: ${{ github.event.comment.user.login }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Fail if not member
         run: |
-           echo "The command was '${{ steps.command.outputs.command-name }}' with arguments '${{ steps.command.outputs.command-arguments }}'"
+          result=${{ steps.is_organization_member.outputs.result }}
+          if [ $result == false ]; then
+              user=${{ github.event.comment.user.login }}
+              echo Either ${user} is not part of the kata-containers organization
+              echo or ${user} has its Organization Visibility set to Private at
+              echo https://github.com/orgs/kata-containers/people?query=${user}
+              echo 
+              echo Ensure you change your Organization Visibility to Public and
+              echo trigger the test again.
+              exit 1
+          fi
 
-  create-and-test-container:
-    needs: check_comments
+  build-asset:
     runs-on: ubuntu-latest
+    needs: check-comment-and-membership
+    strategy:
+      matrix:
+        asset:
+          - cloud-hypervisor
+          - firecracker
+          - kernel
+          - qemu
+          - rootfs-image
+          - rootfs-initrd
+          - shim-v2
     steps:
-      - name: get-PR-ref
-        id: get-PR-ref
+      - uses: actions/checkout@v2
+      - name: Install docker
         run: |
-            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
-            echo "reference for PR: " ${ref}
-            echo "##[set-output name=pr-ref;]${ref}"
+          curl -fsSL https://test.docker.com -o test-docker.sh
+          sh test-docker.sh
 
-      - name: check out
-        uses: actions/checkout@v2
-        with:
-           ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
-
-      - name: build-container-image
-        id: build-container-image
+      - name: Build ${{ matrix.asset }}
         run: |
-            PR_SHA=$(git log --format=format:%H -n1)
-            VERSION="2.0.0"
-            ARTIFACT_URL="https://github.com/kata-containers/kata-containers/releases/download/${VERSION}/kata-static-${VERSION}-x86_64.tar.xz"
-            wget "${ARTIFACT_URL}" -O tools/packaging/kata-deploy/kata-static.tar.xz
-            docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:${PR_SHA} -t quay.io/kata-containers/kata-deploy-ci:${PR_SHA} ./tools/packaging/kata-deploy
-            docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
-            docker push katadocker/kata-deploy-ci:$PR_SHA
-            docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
-            docker push quay.io/kata-containers/kata-deploy-ci:$PR_SHA
-            echo "##[set-output name=pr-sha;]${PR_SHA}"
-
-      - name: test-kata-deploy-ci-in-aks
-        uses: ./tools/packaging/kata-deploy/action
-        with:
-          packaging-sha: ${{ steps.build-container-image.outputs.pr-sha }}
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
         env:
-          PKG_SHA: ${{ steps.build-container-image.outputs.pr-sha }}
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: build-asset
+    steps:
+      - uses: actions/checkout@v2
+      - name: get-artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: kata-artifacts
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: kata-static-tarball
+          path: kata-static.tar.xz
+
+  kata-deploy:
+    needs: create-kata-tarball
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v2
+        with:
+          name: kata-static-tarball
+      - name: build-and-push-kata-deploy-ci
+        id: build-and-push-kata-deploy-ci
+        run: |
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          pushd $GITHUB_WORKSPACE
+          git checkout $tag
+          pkg_sha=$(git rev-parse HEAD)
+          popd
+          mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
+          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t quay.io/kata-containers/kata-deploy-ci:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
+          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
+          docker push quay.io/kata-containers/kata-deploy-ci:$pkg_sha
+          mkdir -p packaging/kata-deploy
+          ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
+          echo "::set-output name=PKG_SHA::${pkg_sha}"
+      - name: test-kata-deploy-ci-in-aks
+        uses: ./packaging/kata-deploy/action
+        with:
+          packaging-sha: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
+        env:
+          PKG_SHA: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
           AZ_APPID: ${{ secrets.AZ_APPID }}
           AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
           AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}

.github/workflows/main.yaml

@@ -1,295 +0,0 @@
-name: Publish release tarball
-on: 
-  push: 
-    tags:
-     - '1.*'
-
-jobs:
-  get-artifact-list:
-    runs-on: ubuntu-latest
-    steps:
-      - name: get the list
-        run: |
-         pushd $GITHUB_WORKSPACE
-         tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-         git checkout $tag
-         popd
-         $GITHUB_WORKSPACE/tools/packaging/artifact-list.sh > artifact-list.txt
-      - name: save-artifact-list
-        uses: actions/upload-artifact@master
-        with:
-          name: artifact-list
-          path: artifact-list.txt
-
-  build-kernel:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_kernel"
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifact-list
-        uses: actions/download-artifact@master
-        with:
-          name: artifact-list
-      - run: |
-         sudo apt-get update && sudo apt install -y flex bison libelf-dev bc iptables
-      - name: build-kernel
-        run: |
-         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@master
-        with:
-          name: kata-artifacts
-          path: kata-static-kernel.tar.gz
-
-  build-experimental-kernel:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_experimental_kernel"
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifact-list
-        uses: actions/download-artifact@master
-        with:
-          name: artifact-list
-      - run: |
-         sudo apt-get update && sudo apt install -y flex bison libelf-dev bc iptables
-      - name: build-experimental-kernel
-        run: |
-         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@master
-        with:
-          name: kata-artifacts
-          path: kata-static-experimental-kernel.tar.gz
-
-  build-qemu:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_qemu"
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifact-list
-        uses: actions/download-artifact@master
-        with:
-          name: artifact-list
-      - name: build-qemu
-        run: |
-         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@master
-        with:
-          name: kata-artifacts
-          path: kata-static-qemu.tar.gz
-
-  # Job for building the image
-  build-image:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_image"
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifact-list
-        uses: actions/download-artifact@master
-        with:
-          name: artifact-list
-      - name: build-image
-        run: |
-         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@master
-        with:
-          name: kata-artifacts
-          path: kata-static-image.tar.gz
-
-  # Job for building firecracker hypervisor
-  build-firecracker:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_firecracker"
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifact-list
-        uses: actions/download-artifact@master
-        with:
-          name: artifact-list
-      - name: build-firecracker
-        run: |
-         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@master
-        with:
-          name: kata-artifacts
-          path: kata-static-firecracker.tar.gz
-
-  # Job for building cloud-hypervisor
-  build-clh:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_clh"
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifact-list
-        uses: actions/download-artifact@master
-        with:
-          name: artifact-list
-      - name: build-clh
-        run: |
-         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@master
-        with:
-          name: kata-artifacts
-          path: kata-static-clh.tar.gz
-
-  # Job for building kata components
-  build-kata-components:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_kata_components"
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifact-list
-        uses: actions/download-artifact@master
-        with:
-          name: artifact-list
-      - name: build-kata-components
-        run: |
-         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@master
-        with:
-          name: kata-artifacts
-          path: kata-static-kata-components.tar.gz
-
-  gather-artifacts:
-    runs-on: ubuntu-16.04
-    needs: [build-experimental-kernel, build-kernel, build-qemu, build-image, build-firecracker, build-kata-components, build-clh]
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifacts
-        uses: actions/download-artifact@master
-        with:
-          name: kata-artifacts
-      - name: colate-artifacts
-        run: |
-          $GITHUB_WORKSPACE/.github/workflows/gather-artifacts.sh
-      - name: store-artifacts
-        uses: actions/upload-artifact@master
-        with:
-          name: release-candidate
-          path: kata-static.tar.xz
-
-  kata-deploy:
-    needs: gather-artifacts
-    runs-on: ubuntu-latest
-    steps:
-      - name: get-artifacts
-        uses: actions/download-artifact@master
-        with:
-          name: release-candidate
-      - name: build-and-push-kata-deploy-ci
-        id: build-and-push-kata-deploy-ci
-        run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          git clone https://github.com/kata-containers/packaging
-          pushd packaging
-          git checkout $tag
-          pkg_sha=$(git rev-parse HEAD)
-          popd
-          mv release-candidate/kata-static.tar.xz ./packaging/kata-deploy/kata-static.tar.xz
-          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:$pkg_sha -t quay.io/kata-containers/kata-deploy-ci:$pkg_sha ./packaging/kata-deploy
-          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
-          docker push katadocker/kata-deploy-ci:$pkg_sha
-          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
-          docker push quay.io/kata-containers/kata-deploy-ci:$pkg_sha
-          echo "::set-output name=PKG_SHA::${pkg_sha}"
-      - name: test-kata-deploy-ci-in-aks
-        uses: ./packaging/kata-deploy/action
-        with:
-          packaging-sha: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
-        env:
-          PKG_SHA: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
-          AZ_APPID: ${{ secrets.AZ_APPID }}
-          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
-          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
-          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
-      - name: push-tarball
-        run: |
-          # tag the container image we created and push to DockerHub
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          docker tag katadocker/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} katadocker/kata-deploy:${tag}
-          docker push katadocker/kata-deploy:${tag}
-
-  upload-static-tarball:
-    needs: kata-deploy
-    runs-on: ubuntu-latest
-    steps:
-      - name: download-artifacts
-        uses: actions/download-artifact@master
-        with:
-          name: release-candidate
-      - name: install hub
-        run: |
-          HUB_VER=$(curl -s "https://api.github.com/repos/github/hub/releases/latest" | jq -r .tag_name | sed 's/^v//')
-          wget -q -O- https://github.com/github/hub/releases/download/v$HUB_VER/hub-linux-amd64-$HUB_VER.tgz | \
-          tar xz --strip-components=2 --wildcards '*/bin/hub' && sudo mv hub /usr/local/bin/hub
-      - name: push static tarball to github
-        run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          tarball="kata-static-$tag-x86_64.tar.xz"
-          repo="https://github.com/kata-containers/runtime.git"
-          mv release-candidate/kata-static.tar.xz "release-candidate/${tarball}"
-          git clone "${repo}"
-          cd runtime
-          echo "uploading asset '${tarball}' to '${repo}' tag: ${tag}"
-          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "../release-candidate/${tarball}" "${tag}"

.github/workflows/release.yaml

@@ -149,3 +149,31 @@ jobs:
           tar -cvzf "${tarball}" src/agent/.cargo/config src/agent/vendor
           GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}" 
           popd
+
+  upload-libseccomp-tarball:
+    needs: upload-cargo-vendored-tarball
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: download-and-upload-tarball
+        env:
+          GITHUB_TOKEN: ${{ secrets.GIT_UPLOAD_TOKEN }}
+          GOPATH: ${HOME}/go
+        run: |
+          pushd $GITHUB_WORKSPACE
+          ./ci/install_yq.sh
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          versions_yaml="versions.yaml"
+          version=$(${GOPATH}/bin/yq read ${versions_yaml} "externals.libseccomp.version")
+          repo_url=$(${GOPATH}/bin/yq read ${versions_yaml} "externals.libseccomp.url")
+          download_url="${repo_url}/releases/download/v${version}"
+          tarball="libseccomp-${version}.tar.gz"
+          asc="${tarball}.asc"
+          curl -sSLO "${download_url}/${tarball}"
+          curl -sSLO "${download_url}/${asc}"
+          # "-m" option should be empty to re-use the existing release title
+          # without opening a text editor.
+          # For the details, check https://hub.github.com/hub-release.1.html.
+          hub release edit -m "" -a "${tarball}" "${tag}"
+          hub release edit -m "" -a "${asc}" "${tag}"
+          popd

.github/workflows/require-pr-porting-labels.yaml

@@ -12,8 +12,7 @@ on:
       - reopened
       - labeled
       - unlabeled
-   pull_request:
-     branches:
+    branches:
       - main
 
 jobs:
@@ -32,8 +31,6 @@ jobs:
 
       - name: Checkout code to allow hub to communicate with the project
         uses: actions/checkout@v2
-        with:
-          token: ${{ secrets.KATA_GITHUB_ACTIONS_TOKEN }}
 
       - name: Install porting checker script
         run: |

.github/workflows/static-checks.yaml

@@ -13,7 +13,7 @@ jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.15.x, 1.16.x]
+        go-version: [1.16.x, 1.17.x]
         os: [ubuntu-20.04]
     runs-on: ${{ matrix.os }}
     env:
@@ -67,6 +67,14 @@ jobs:
         PATH=$PATH:"$HOME/.cargo/bin"
         rustup target add x86_64-unknown-linux-musl
         rustup component add rustfmt clippy
+    - name: Setup seccomp
+      run: |
+        libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
+        gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
+        echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
+        echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
+        echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
     # Check whether the vendored code is up-to-date & working as the first thing
     - name: Check vendored code
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}

VERSION

@@ -1 +1 @@
-2.3.0-alpha1
+2.3.0

ci/go-no-os-exit.sh

@@ -1,30 +0,0 @@
-#!/bin/bash
-# Copyright (c) 2018 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-# Check there are no os.Exit() calls creeping into the code
-# We don't use that exit path in the Kata codebase.
-
-# Allow the path to check to be over-ridden.
-# Default to the current directory.
-go_packages=${1:-.}
-
-echo "Checking for no os.Exit() calls for package [${go_packages}]"
-
-candidates=`go list -f '{{.Dir}}/*.go' $go_packages`
-for f in $candidates; do
-	filename=`basename $f`
-	# skip all go test files
-	[[ $filename == *_test.go ]] && continue
-	# skip exit.go where, the only file we should call os.Exit() from.
-	[[ $filename == "exit.go" ]] && continue
-	files="$f $files"
-done
-
-[ -z "$files" ] && echo "No files to check, skipping" && exit 0
-
-if egrep -n '\<os\.Exit\>' $files; then
-	echo "Direct calls to os.Exit() are forbidden, please use exit() so atexit() works"
-	exit 1
-fi

ci/install_libseccomp.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+#
+# Copyright 2021 Sony Group Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -o errexit
+
+cidir=$(dirname "$0")
+source "${cidir}/lib.sh"
+
+clone_tests_repo
+
+source "${tests_repo_dir}/.ci/lib.sh"
+
+# The following variables if set on the environment will change the behavior
+# of gperf and libseccomp configure scripts, that may lead this script to
+# fail. So let's ensure they are unset here.
+unset PREFIX DESTDIR
+
+arch=$(uname -m)
+workdir="$(mktemp -d --tmpdir build-libseccomp.XXXXX)"
+
+# Variables for libseccomp
+# Currently, specify the libseccomp version directly without using `versions.yaml`
+# because the current Snap workflow is incomplete.
+# After solving the issue, replace this code by using the `versions.yaml`.
+# libseccomp_version=$(get_version "externals.libseccomp.version")
+# libseccomp_url=$(get_version "externals.libseccomp.url")
+libseccomp_version="2.5.1"
+libseccomp_url="https://github.com/seccomp/libseccomp"
+libseccomp_tarball="libseccomp-${libseccomp_version}.tar.gz"
+libseccomp_tarball_url="${libseccomp_url}/releases/download/v${libseccomp_version}/${libseccomp_tarball}"
+cflags="-O2"
+
+# Variables for gperf
+# Currently, specify the gperf version directly without using `versions.yaml`
+# because the current Snap workflow is incomplete.
+# After solving the issue, replace this code by using the `versions.yaml`.
+# gperf_version=$(get_version "externals.gperf.version")
+# gperf_url=$(get_version "externals.gperf.url")
+gperf_version="3.1"
+gperf_url="https://ftp.gnu.org/gnu/gperf"
+gperf_tarball="gperf-${gperf_version}.tar.gz"
+gperf_tarball_url="${gperf_url}/${gperf_tarball}"
+
+# We need to build the libseccomp library from sources to create a static library for the musl libc.
+# However, ppc64le and s390x have no musl targets in Rust. Hence, we do not set cflags for the musl libc.
+if ([ "${arch}" != "ppc64le" ] && [ "${arch}" != "s390x" ]); then
+    # Set FORTIFY_SOURCE=1 because the musl-libc does not have some functions about FORTIFY_SOURCE=2
+    cflags="-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=1 -O2"
+fi
+
+die() {
+    msg="$*"
+    echo "[Error] ${msg}" >&2
+    exit 1
+}
+
+finish() {
+    rm -rf "${workdir}"
+}
+
+trap finish EXIT
+
+build_and_install_gperf() {
+    echo "Build and install gperf version ${gperf_version}"
+    mkdir -p "${gperf_install_dir}"
+    curl -sLO "${gperf_tarball_url}"
+    tar -xf "${gperf_tarball}"
+    pushd "gperf-${gperf_version}"
+    ./configure --prefix="${gperf_install_dir}"
+    make
+    make install
+    export PATH=$PATH:"${gperf_install_dir}"/bin
+    popd
+    echo "Gperf installed successfully"
+}
+
+build_and_install_libseccomp() {
+    echo "Build and install libseccomp version ${libseccomp_version}"
+    mkdir -p "${libseccomp_install_dir}"
+    curl -sLO "${libseccomp_tarball_url}"
+    tar -xf "${libseccomp_tarball}"
+    pushd "libseccomp-${libseccomp_version}"
+    ./configure --prefix="${libseccomp_install_dir}" CFLAGS="${cflags}" --enable-static
+    make
+    make install
+    popd
+    echo "Libseccomp installed successfully"
+}
+
+main() {
+    local libseccomp_install_dir="${1:-}"
+    local gperf_install_dir="${2:-}"
+
+    if [ -z "${libseccomp_install_dir}" ] || [ -z "${gperf_install_dir}" ]; then
+        die "Usage: ${0} <libseccomp-install-dir> <gperf-install-dir>"
+    fi
+
+    pushd "$workdir"
+    # gperf is required for building the libseccomp.
+    build_and_install_gperf
+    build_and_install_libseccomp
+    popd
+}
+
+main "$@"

ci/install_rust.sh

@@ -12,5 +12,5 @@ source "${cidir}/lib.sh"
 clone_tests_repo
 
 pushd ${tests_repo_dir}
-.ci/install_rust.sh
+.ci/install_rust.sh ${1:-}
 popd

docs/Developer-Guide.md

@@ -86,6 +86,16 @@ One of the `initrd` and `image` options in Kata runtime config file **MUST** be
 The main difference between the options is that the size of `initrd`(10MB+) is significantly smaller than
 rootfs `image`(100MB+).
 
+## Enable seccomp
+
+Enable seccomp as follows:
+
+```
+$ sudo sed -i '/^disable_guest_seccomp/ s/true/false/' /etc/kata-containers/configuration.toml
+```
+
+This will pass container seccomp profiles to the kata agent.
+
 ## Enable full debug
 
 Enable full debug as follows:
@@ -216,6 +226,18 @@ $ go get -d -u github.com/kata-containers/kata-containers
 $ cd $GOPATH/src/github.com/kata-containers/kata-containers/src/agent && make
 ```
 
+The agent is built with seccomp capability by default.
+If you want to build the agent without the seccomp capability, you need to run `make` with `SECCOMP=no` as follows.
+
+```
+$ make -C $GOPATH/src/github.com/kata-containers/kata-containers/src/agent SECCOMP=no
+```
+
+> **Note:**
+>
+> - If you enable seccomp in the main configuration file but build the agent without seccomp capability,
+>   the runtime exits conservatively with an error message.
+
 ## Get the osbuilder
 
 ```
@@ -234,9 +256,21 @@ the following example.
 $ export ROOTFS_DIR=${GOPATH}/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder/rootfs
 $ sudo rm -rf ${ROOTFS_DIR}
 $ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder
-$ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
+$ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true ./rootfs.sh ${distro}'
+```
+
+You MUST choose a distribution (e.g., `ubuntu`) for `${distro}`.
+You can get a supported distributions list in the Kata Containers by running the following.
+
+```
+$ ./rootfs.sh -l
+```
+
+If you want to build the agent without seccomp capability, you need to run the `rootfs.sh` script with `SECCOMP=no` as follows.
+
+```
+$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
 ```
-You MUST choose one of `alpine`, `centos`, `clearlinux`, `debian`, `euleros`, `fedora`, `suse`, and `ubuntu` for `${distro}`. By default `seccomp` packages are not included in the rootfs image. Set `SECCOMP` to `yes` to include them.
 
 > **Note:**
 >
@@ -272,6 +306,7 @@ $ script -fec 'sudo -E USE_DOCKER=true ./image_builder.sh ${ROOTFS_DIR}'
 > - If you do *not* wish to build under Docker, remove the `USE_DOCKER`
 >   variable in the previous command and ensure the `qemu-img` command is
 >   available on your system.
+>   - If `qemu-img` is not installed, you will likely see errors such as `ERROR: File /dev/loop19p1 is not a block device` and `losetup: /tmp/tmp.bHz11oY851: Warning: file is smaller than 512 bytes; the loop device may be useless or invisible for system tools`. These can be mitigated by installing the `qemu-img` command (available in the `qemu-img` package on Fedora or the `qemu-utils` package on Debian).
 
 
 ### Install the rootfs image
@@ -290,12 +325,23 @@ $ (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
 $ export ROOTFS_DIR="${GOPATH}/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder/rootfs"
 $ sudo rm -rf ${ROOTFS_DIR}
 $ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder
-$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
+$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true ./rootfs.sh ${distro}'
 ```
 `AGENT_INIT` controls if the guest image uses the Kata agent as the guest `init` process. When you create an initrd image,
-always set `AGENT_INIT` to `yes`. By default `seccomp` packages are not included in the initrd image. Set `SECCOMP` to `yes` to include them.
+always set `AGENT_INIT` to `yes`.
 
-You MUST choose one of `alpine`, `centos`, `clearlinux`, `euleros`, and `fedora` for `${distro}`.
+You MUST choose a distribution (e.g., `ubuntu`) for `${distro}`.
+You can get a supported distributions list in the Kata Containers by running the following.
+
+```
+$ ./rootfs.sh -l
+```
+
+If you want to build the agent without seccomp capability, you need to run the `rootfs.sh` script with `SECCOMP=no` as follows.
+
+```
+$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
+```
 
 > **Note:**
 >

docs/README.md

@@ -11,6 +11,10 @@ For details of the other Kata Containers repositories, see the
 
 * [Installation guides](./install/README.md): Install and run Kata Containers with Docker or Kubernetes
 
+## Tracing
+
+See the [tracing documentation](tracing.md).
+
 ## More User Guides
 
 * [Upgrading](Upgrading.md): how to upgrade from [Clear Containers](https://github.com/clearcontainers) and [runV](https://github.com/hyperhq/runv) to [Kata Containers](https://github.com/kata-containers) and how to upgrade an existing Kata Containers system to the latest version.

docs/Release-Process.md

@@ -64,7 +64,7 @@
 
 ### Check Git-hub Actions
 
-  We make use of [GitHub actions](https://github.com/features/actions) in this [file](https://github.com/kata-containers/kata-containers/blob/main/.github/workflows/main.yaml) in the `kata-containers/kata-containers` repository to build and upload release artifacts. This action is auto triggered with the above step when a new tag is pushed to the `kata-containers/kata-containers` repository.
+  We make use of [GitHub actions](https://github.com/features/actions) in this [file](https://github.com/kata-containers/kata-containers/blob/main/.github/workflows/release.yaml) in the `kata-containers/kata-containers` repository to build and upload release artifacts. This action is auto triggered with the above step when a new tag is pushed to the `kata-containers/kata-containers` repository.
 
   Check the [actions status page](https://github.com/kata-containers/kata-containers/actions) to verify all steps in the actions workflow have completed successfully. On success, a static tarball containing Kata release artifacts will be uploaded to the [Release page](https://github.com/kata-containers/kata-containers/releases).
 

docs/design/kata-2-0-metrics.md

@@ -207,7 +207,7 @@ Metrics for Firecracker vmm.
 | `kata_firecracker_uart`: <br> Metrics specific to the UART device. | `GAUGE` |  | <ul><li>`item`<ul><li>`error_count`</li><li>`flush_count`</li><li>`missed_read_count`</li><li>`missed_write_count`</li><li>`read_count`</li><li>`write_count`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
 | `kata_firecracker_vcpu`: <br> Metrics specific to VCPUs' mode of functioning. | `GAUGE` |  | <ul><li>`item`<ul><li>`exit_io_in`</li><li>`exit_io_out`</li><li>`exit_mmio_read`</li><li>`exit_mmio_write`</li><li>`failures`</li><li>`filter_cpuid`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
 | `kata_firecracker_vmm`: <br> Metrics specific to the machine manager as a whole. | `GAUGE` |  | <ul><li>`item`<ul><li>`device_events`</li><li>`panic_count`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
-| `kata_firecracker_vsock`: <br> Vsock-related metrics. | `GAUGE` |  | <ul><li>`item`<ul><li>`activate_fails`</li><li>`cfg_fails`</li><li>`conn_event_fails`</li><li>`conns_added`</li><li>`conns_killed`</li><li>`conns_removed`</li><li>`ev_queue_event_fails`</li><li>`killq_resync`</li><li>`muxer_event_fails`</li><li>`rx_bytes_count`</li><li>`rx_packets_count`</li><li>`rx_queue_event_count`</li><li>`rx_queue_event_fails`</li><li>`rx_read_fails`</li><li>`tx_bytes_count`</li><li>`tx_flush_fails`</li><li>`tx_packets_count`</li><li>`tx_queue_event_count`</li><li>`tx_queue_event_fails`</li><li>`tx_write_fails`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
+| `kata_firecracker_vsock`: <br> VSOCK-related metrics. | `GAUGE` |  | <ul><li>`item`<ul><li>`activate_fails`</li><li>`cfg_fails`</li><li>`conn_event_fails`</li><li>`conns_added`</li><li>`conns_killed`</li><li>`conns_removed`</li><li>`ev_queue_event_fails`</li><li>`killq_resync`</li><li>`muxer_event_fails`</li><li>`rx_bytes_count`</li><li>`rx_packets_count`</li><li>`rx_queue_event_count`</li><li>`rx_queue_event_fails`</li><li>`rx_read_fails`</li><li>`tx_bytes_count`</li><li>`tx_flush_fails`</li><li>`tx_packets_count`</li><li>`tx_queue_event_count`</li><li>`tx_queue_event_fails`</li><li>`tx_write_fails`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
 
 ### Kata guest OS metrics
 

docs/design/kata-design-requirements.md

@@ -30,7 +30,7 @@ The Kata Containers runtime **MUST** implement the following command line option
 The Kata Containers project **MUST** provide two interfaces for CRI shims to manage hardware
 virtualization based Kubernetes pods and containers:
 -  An OCI and `runc` compatible command line interface, as described in the previous section.
-This interface is used by implementations such as [`CRI-O`](http://cri-o.io) and [`cri-containerd`](https://github.com/containerd/cri-containerd), for example.
+This interface is used by implementations such as [`CRI-O`](http://cri-o.io) and [`containerd`](https://github.com/containerd/containerd), for example.
 - A hardware virtualization runtime library API for CRI shims to consume and provide a more
 CRI native implementation. The [`frakti`](https://github.com/kubernetes/frakti) CRI shim is an example of such a consumer.
 

docs/how-to/README.md

@@ -5,7 +5,7 @@
 - [Run Kata containers with `crictl`](run-kata-with-crictl.md)
 - [Run Kata Containers with Kubernetes](run-kata-with-k8s.md)
 - [How to use Kata Containers and Containerd](containerd-kata.md)
-- [How to use Kata Containers and CRI (containerd plugin) with Kubernetes](how-to-use-k8s-with-cri-containerd-and-kata.md)
+- [How to use Kata Containers and CRI (containerd) with Kubernetes](how-to-use-k8s-with-cri-containerd-and-kata.md)
 - [Kata Containers and service mesh for Kubernetes](service-mesh.md)
 - [How to import Kata Containers logs into Fluentd](how-to-import-kata-logs-with-fluentd.md)
 

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -34,8 +34,6 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.agent.enable_tracing` | `boolean` | enable tracing for the agent |
 | `io.katacontainers.config.agent.container_pipe_size` | uint32 | specify the size of the std(in/out) pipes created for containers |
 | `io.katacontainers.config.agent.kernel_modules` | string | the list of kernel modules and their parameters that will be loaded in the guest kernel. Semicolon separated list of kernel modules and their parameters. These modules will be loaded in the guest kernel using `modprobe`(8). E.g., `e1000e InterruptThrottleRate=3000,3000,3000 EEE=1; i915 enable_ppgtt=0` |
-| `io.katacontainers.config.agent.trace_mode` | string | the trace mode for the agent |
-| `io.katacontainers.config.agent.trace_type` | string | the trace type for the agent |
 
 ## Hypervisor Options
 | Key | Value Type | Comments |

docs/how-to/how-to-use-k8s-with-cri-containerd-and-kata.md

@@ -3,7 +3,7 @@
 This document describes how to set up a single-machine Kubernetes (k8s) cluster.
 
 The Kubernetes cluster will use the
-[CRI containerd plugin](https://github.com/containerd/containerd/tree/main/pkg/cri) and
+[CRI containerd](https://github.com/containerd/containerd/) and
 [Kata Containers](https://katacontainers.io) to launch untrusted workloads.
 
 ## Requirements
@@ -71,12 +71,12 @@ $ for service in ${services}; do
     service_dir="/etc/systemd/system/${service}.service.d/"
     sudo mkdir -p ${service_dir}
 
-    cat << EOT | sudo tee "${service_dir}/proxy.conf"
+    cat << EOF | sudo tee "${service_dir}/proxy.conf"
 [Service]
 Environment="HTTP_PROXY=${http_proxy}"
 Environment="HTTPS_PROXY=${https_proxy}"
 Environment="NO_PROXY=${no_proxy}"
-EOT
+EOF
 done
 
 $ sudo systemctl daemon-reload
@@ -172,7 +172,7 @@ If a pod has the `runtimeClassName` set to `kata`, the CRI plugin runs the pod w
 - Create an pod configuration that using Kata Containers runtime
 
   ```bash
-  $ cat << EOT | tee nginx-kata.yaml
+  $ cat << EOF | tee nginx-kata.yaml
   apiVersion: v1
   kind: Pod
   metadata:
@@ -183,7 +183,7 @@ If a pod has the `runtimeClassName` set to `kata`, the CRI plugin runs the pod w
     - name: nginx
       image: nginx
       
-  EOT
+  EOF
   ```
 
 - Create the pod

docs/how-to/how-to-use-kata-containers-with-acrn.md

@@ -22,7 +22,7 @@ This document requires the presence of the ACRN hypervisor and Kata Containers o
 
 - ACRN supported [Hardware](https://projectacrn.github.io/latest/hardware.html#supported-hardware).
   > **Note:** Please make sure to have a minimum of 4 logical processors (HT) or cores.
-- ACRN [software](https://projectacrn.github.io/latest/tutorials/kbl-nuc-sdc.html#use-the-script-to-set-up-acrn-automatically) setup.
+- ACRN [software](https://projectacrn.github.io/latest/tutorials/run_kata_containers.html) setup.
 - For networking, ACRN supports either MACVTAP or TAP. If MACVTAP is not enabled in the Service OS, please follow the below steps to update the kernel:
 
   ```sh

docs/how-to/privileged.md

@@ -16,9 +16,9 @@ from the host, a potentially undesirable side-effect that decreases the security
 
 The following sections document how to configure this behavior in different container runtimes.
 
-#### Containerd and CRI
+#### Containerd
 
-The Containerd CRI allows configuring the privileged host devices behavior for each runtime in the CRI config. This is
+The Containerd allows configuring the privileged host devices behavior for each runtime in the containerd config. This is
 done with the `privileged_without_host_devices` option. Setting this to `true` will disable hot plugging of the host 
 devices into the guest, even when privileged is enabled.
 
@@ -41,7 +41,7 @@ See below example config:
 ```
 
  - [Kata Containers with Containerd and CRI documentation](how-to-use-k8s-with-cri-containerd-and-kata.md)
- - [Containerd CRI config documentation](https://github.com/containerd/cri/blob/master/docs/config.md)
+ - [Containerd CRI config documentation](https://github.com/containerd/containerd/blob/main/docs/cri/config.md)
 
 #### CRI-O
 

docs/how-to/run-kata-with-k8s.md

@@ -9,7 +9,7 @@ Kubernetes CRI (Container Runtime Interface) implementations allow using any
 OCI-compatible runtime with Kubernetes, such as the Kata Containers runtime.
 
 Kata Containers support both the [CRI-O](https://github.com/kubernetes-incubator/cri-o) and
-[CRI-containerd](https://github.com/containerd/cri) CRI implementations.
+[containerd](https://github.com/containerd/containerd) CRI implementations.
 
 After choosing one CRI implementation, you must make the appropriate configuration
 to ensure it integrates with Kata Containers.
@@ -111,11 +111,7 @@ manage_ns_lifecycle = true
 ```
 
 
-### containerd with CRI plugin
-
-If you select containerd with `cri` plugin, follow the "Getting Started for Developers"
-instructions [here](https://github.com/containerd/cri#getting-started-for-developers)
-to properly install it.
+### containerd
 
 To customize containerd to select Kata Containers runtime, follow our
 "Configure containerd to use Kata Containers" internal documentation
@@ -160,7 +156,7 @@ $ sudo systemctl restart kubelet
 # If using CRI-O
 $ sudo kubeadm init --ignore-preflight-errors=all --cri-socket /var/run/crio/crio.sock --pod-network-cidr=10.244.0.0/16
 
-# If using CRI-containerd
+# If using containerd
 $ sudo kubeadm init --ignore-preflight-errors=all --cri-socket /run/containerd/containerd.sock --pod-network-cidr=10.244.0.0/16
 
 $ export KUBECONFIG=/etc/kubernetes/admin.conf

docs/how-to/service-mesh.md

@@ -34,7 +34,7 @@ as the proxy starts.
 
 Follow the [instructions](../install/README.md)
 to get Kata Containers properly installed and configured with Kubernetes.
-You can choose between CRI-O and CRI-containerd, both are supported
+You can choose between CRI-O and containerd, both are supported
 through this document.
 
 For both cases, select the workloads as _trusted_ by default. This way,
@@ -159,7 +159,7 @@ containers with `privileged: true` to `privileged: false`.
 There is no difference between Istio and Linkerd in this section. It is
 about which CRI implementation you use.
 
-For both CRI-O and CRI-containerd, you have to add an annotation indicating
+For both CRI-O and containerd, you have to add an annotation indicating
 the workload for this deployment is not _trusted_, which will trigger
 `kata-runtime` to be called instead of `runc`.
 
@@ -193,9 +193,9 @@ spec:
 ...
 ```
 
-__CRI-containerd:__
+__containerd:__
 
-Add the following annotation for CRI-containerd
+Add the following annotation for containerd
 ```yaml
 io.kubernetes.cri.untrusted-workload: "true"
 ```

docs/tracing.md

@@ -0,0 +1,214 @@
+# Overview
+
+This document explains how to trace Kata Containers components.
+
+# Introduction
+
+The Kata Containers runtime and agent are able to generate
+[OpenTelemetry][opentelemetry] trace spans, which allow the administrator to
+observe what those components are doing and how much time they are spending on
+each operation.
+
+# OpenTelemetry summary
+
+An OpenTelemetry-enabled application creates a number of trace "spans". A span
+contains the following attributes:
+
+- A name
+- A pair of timestamps (recording the start time and end time of some operation)
+- A reference to the span's parent span
+
+All spans need to be *finished*, or *completed*, to allow the OpenTelemetry
+framework to generate the final trace information (by effectively closing the
+transaction encompassing the initial (root) span and all its children).
+
+For Kata, the root span represents the total amount of time taken to run a
+particular component from startup to its shutdown (the "run time").
+
+# Architecture
+
+## Runtime tracing architecture
+
+The runtime, which runs in the host environment, has been modified to
+optionally generate trace spans which are sent to a trace collector on the
+host.
+
+## Agent tracing architecture
+
+An OpenTelemetry system (such as [Jaeger][jaeger-tracing]) uses a collector to
+gather up trace spans from the application for viewing and processing. For an
+application to use the collector, it must run in the same context as
+the collector.
+
+This poses a problem for tracing the Kata Containers agent since it does not
+run in the same context as the collector: it runs inside a virtual machine (VM).
+
+To allow spans from the agent to be sent to the trace collector, Kata provides
+a [trace forwarder][trace-forwarder] component. This runs in the same context
+as the collector (generally on the host system) and listens on a
+[`VSOCK`][vsock] channel for traces generated by the agent, forwarding them on
+to the trace collector.
+
+> **Note:**
+>
+> This design supports agent tracing without having to make changes to the
+> image, but also means that [custom images][osbuilder] can also benefit from
+> agent tracing.
+
+The following diagram summarises the architecture used to trace the Kata
+Containers agent:
+
+```
++--------------------------------------------+
+| Host                                       |
+|                                            |
+| +---------------+                          |
+| | OpenTelemetry |                          |
+| | Trace         |                          |
+| | Collector     |                          |
+| +---------------+                          |
+|       ^                  +---------------+ |
+|       | spans            | Kata VM       | |
+| +-----+-----+            |               | |
+| | Kata      |    spans   o     +-------+ | |
+| | Trace     |<-----------------| Kata  | | |
+| | Forwarder |    VSOCK   o     | Agent | | |
+| +-----------+    Channel |     +-------+ | |
+|                          +---------------+ |
++--------------------------------------------+
+```
+
+# Agent tracing prerequisites
+
+- You must have a trace collector running.
+
+  Although the collector normally runs on the host, it can also be run from
+  inside a Docker image configured to expose the appropriate host ports to the
+  collector.
+
+  The [Jaeger "all-in-one" Docker image][jaeger-all-in-one] method
+  is the quickest and simplest way to run the collector for testing.
+
+- If you wish to trace the agent, you must start the
+  [trace forwarder][trace-forwarder].
+
+> **Notes:**
+>
+> - If agent tracing is enabled but the forwarder is not running,
+>   the agent will log an error (signalling that it cannot generate trace
+>   spans), but continue to work as normal.
+>
+> - The trace forwarder requires a trace collector (such as Jaeger) to be
+>   running before it is started. If a collector is not running, the trace
+>   forwarder will exit with an error.
+
+# Enable tracing
+
+By default, tracing is disabled for all components. To enable _any_ form of
+tracing an `enable_tracing` option must be enabled for at least one component.
+
+> **Note:** 
+>
+> Enabling this option will only allow tracing for subsequently
+> started containers.
+
+## Enable runtime tracing
+
+To enable runtime tracing, set the tracing option as shown:
+
+```toml
+[runtime]
+enable_tracing = true
+```
+
+## Enable agent tracing
+
+To enable agent tracing, set the tracing option as shown:
+
+```toml
+[agent.kata]
+enable_tracing = true
+```
+
+> **Note:**
+>
+> If both agent tracing and runtime tracing are enabled, the resulting trace
+> spans will be "collated": expanding individual runtime spans in the Jaeger
+> web UI will show the agent trace spans resulting from the runtime
+> operation.
+
+# Appendices
+
+## Agent tracing requirements
+
+### Host environment
+
+- The host kernel must support the VSOCK socket type.
+
+  This will be available if the kernel is built with the
+  `CONFIG_VHOST_VSOCK` configuration option.
+
+- The VSOCK kernel module must be loaded:
+
+   ```
+   $ sudo modprobe vhost_vsock
+   ```
+
+### Guest environment
+
+- The guest kernel must support the VSOCK socket type:
+
+  This will be available if the kernel is built with the
+  `CONFIG_VIRTIO_VSOCKETS` configuration option.
+
+  > **Note:** The default Kata Containers guest kernel provides this feature.
+
+## Agent tracing limitations
+
+- Agent tracing is only "completed" when the workload and the Kata agent
+  process have exited.
+
+  Although trace information *can* be inspected before the workload and agent
+  have exited, it is incomplete. This is shown as `<trace-without-root-span>`
+  in the Jaeger web UI.
+
+  If the workload is still running, the trace transaction -- which spans the entire
+  runtime of the Kata agent -- will not have been completed. To view the complete
+  trace details, wait for the workload to end, or stop the container.
+
+## Performance impact
+
+[OpenTelemetry][opentelemetry] is designed for high performance. It combines
+the best of two previous generation projects (OpenTracing and OpenCensus) and
+uses a very efficient mechanism to capture trace spans. Further, the trace
+points inserted into the agent are generated dynamically at compile time. This
+is advantageous since new versions of the agent will automatically benefit
+from improvements in the tracing infrastructure. Overall, the impact of
+enabling runtime and agent tracing should be extremely low.
+
+## Agent shutdown behaviour
+ 
+In normal operation, the Kata runtime manages the VM shutdown and performs
+certain optimisations to speed up this process. However, if agent tracing is
+enabled, the agent itself is responsible for shutting down the VM. This it to
+ensure all agent trace transactions are completed. This means there will be a
+small performance impact for container shutdown when agent tracing is enabled
+as the runtime must wait for the VM to shutdown fully.
+
+## Set up a tracing development environment
+
+If you want to debug, further develop, or test tracing,
+[enabling full debug][enable-full-debug]
+is highly recommended. For working with the agent, you may also wish to
+[enable a debug console][setup-debug-console]
+to allow you to access the VM environment.
+
+[agent-ctl]: https://github.com/kata-containers/kata-containers/blob/main/tools/agent-ctl
+[enable-full-debug]: https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md#enable-full-debug
+[jaeger-all-in-one]: https://www.jaegertracing.io/docs/getting-started/
+[jaeger-tracing]: https://www.jaegertracing.io
+[opentelemetry]: https://opentelemetry.io
+[osbuilder]: https://github.com/kata-containers/kata-containers/blob/main/tools/osbuilder
+[setup-debug-console]: https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md#set-up-a-debug-console
+[trace-forwarder]: https://github.com/kata-containers/kata-containers/blob/main/src/trace-forwarder
+[vsock]: https://wiki.qemu.org/Features/VirtioVsock

docs/use-cases/using-Intel-SGX-and-kata.md

@@ -1,107 +1,113 @@
 # Kata Containers with SGX
 
-Intel® Software Guard Extensions (SGX) is a set of instructions that increases the security
+Intel Software Guard Extensions (SGX) is a set of instructions that increases the security
 of applications code and data, giving them more protections from disclosure or modification.
 
-> **Note:** At the time of writing this document, SGX patches have not landed on the Linux kernel
-> project, so specific versions for guest and host kernels must be installed to enable SGX.
+This document guides you to run containers with SGX enclaves with Kata Containers in Kubernetes.
 
-## Check if SGX is enabled
+## Preconditions
 
-Run the following command to check if your host supports SGX.
+* Intel SGX capable bare metal nodes
+* Host kernel Linux 5.13 or later with SGX and SGX KVM enabled:
 
 ```sh
-$ grep -o sgx /proc/cpuinfo
+$ grep SGX /boot/config-`uname -r`
+CONFIG_X86_SGX=y
+CONFIG_X86_SGX_KVM=y
 ```
 
-Continue to the following section if the output of the above command is empty,
-otherwise continue to section [Install Guest kernel with SGX support](#install-guest-kernel-with-sgx-support)
+* Kubernetes cluster configured with:
+   * [`kata-deploy`](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy) based Kata Containers installation
+   * [Intel SGX Kubernetes device plugin](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/main/cmd/sgx_plugin#deploying-with-pre-built-images)
 
-## Install Host kernel with SGX support
+> Note: Kata Containers supports creating VM sandboxes with Intel® SGX enabled
+> using [cloud-hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor/) VMM only. QEMU support is waiting to get the
+> Intel SGX enabled QEMU upstream release.
 
-The following commands were tested on Fedora 32, they might work on other distros too.
+## Installation
+
+### Kata Containers Guest Kernel
+
+Follow the instructions to [setup](../../tools/packaging/kernel/README.md#setup-kernel-source-code) and [build](../../tools/packaging/kernel/README.md#build-the-kernel) the experimental guest kernel. Then, install as:
 
 ```sh
-$ git clone --depth=1 https://github.com/intel/kvm-sgx
-$ pushd kvm-sgx
-$ cp /boot/config-$(uname -r) .config
-$ yes "" | make oldconfig
-$ # In the following step, enable: INTEL_SGX and INTEL_SGX_VIRTUALIZATION
-$ make menuconfig
-$ make -j$(($(nproc)-1)) bzImage
-$ make -j$(($(nproc)-1)) modules
-$ sudo make modules_install
-$ sudo make install
-$ popd
-$ sudo reboot
+$ sudo cp kata-linux-experimental-*/vmlinux /opt/kata/share/kata-containers/vmlinux.sgx
+$ sudo sed -i 's|vmlinux.container|vmlinux.sgx|g' \
+  /opt/kata/share/defaults/kata-containers/configuration-clh.toml
 ```
 
-> **Notes:**
-> * Run: `mokutil --sb-state` to check whether secure boot is enabled, if so, you will need to sign the kernel.
-> * You'll lose SGX support when a new distro kernel is installed and the system rebooted.
-
-Once you have restarted your system with the new brand Linux Kernel with SGX support, run
-the following command to make sure it's enabled. If the output is empty, go to the BIOS
-setup and enable SGX manually.
-
-```sh
-$ grep -o sgx /proc/cpuinfo
-```
-
-## Install Guest kernel with SGX support
-
-Install the guest kernel in the Kata Containers directory, this way it can be used to run
-Kata Containers.
-
-```sh
-$ curl -LOk https://github.com/devimc/kvm-sgx/releases/download/v0.0.1/kata-virtiofs-sgx.tar.gz
-$ sudo tar -xf kata-virtiofs-sgx.tar.gz -C /usr/share/kata-containers/
-$ sudo sed -i 's|kernel =|kernel = "/usr/share/kata-containers/vmlinux-virtiofs-sgx.container"|g' \
-  /usr/share/defaults/kata-containers/configuration.toml
-```
-
-## Run Kata Containers with SGX enabled
+### Kata Containers Configuration
 
 Before running a Kata Container make sure that your version of `crio` or `containerd`
 supports annotations.
+
 For `containerd` check in `/etc/containerd/config.toml` that the list of `pod_annotations` passed
 to the `sandbox` are: `["io.katacontainers.*", "sgx.intel.com/epc"]`.
 
-> `sgx.yaml`
+## Usage
+
+With the following sample job deployed using `kubectl apply -f`:
+
 ```yaml
-apiVersion: v1
-kind: Pod
+apiVersion: batch/v1
+kind: Job
 metadata:
-  name: sgx
-  annotations:
-    sgx.intel.com/epc: "32Mi"
+  name: oesgx-demo-job
+  labels:
+    jobgroup: oesgx-demo
 spec:
-  terminationGracePeriodSeconds: 0
-  runtimeClassName: kata
-  containers:
-  - name: c1
-    image: busybox
-    command:
-        - sh
-    stdin: true
-    tty: true
-    volumeMounts:
-    - mountPath: /dev/sgx/
-      name: test-volume
-  volumes:
-  - name: test-volume
-    hostPath:
-      path: /dev/sgx/
-      type: Directory
+  template:
+    metadata:
+      labels:
+        jobgroup: oesgx-demo
+    spec:
+      runtimeClassName: kata-clh
+      initContainers:
+        - name: init-sgx
+          image: busybox
+          command: ['sh', '-c', 'mkdir /dev/sgx; ln -s /dev/sgx_enclave /dev/sgx/enclave; ln -s /dev/sgx_provision /dev/sgx/provision']
+          volumeMounts:
+          - mountPath: /dev
+            name: dev-mount
+      restartPolicy: Never
+      containers:
+        -
+          name: eosgx-demo-job-1
+          image: oeciteam/oe-helloworld:latest
+          imagePullPolicy: IfNotPresent
+          securityContext:
+            readOnlyRootFilesystem: true
+            capabilities:
+              add: ["IPC_LOCK"]
+          resources:
+            limits:
+              sgx.intel.com/epc: "512Ki"
+      volumes:
+        - name: dev-mount
+          hostPath:
+            path: /dev
 ```
 
+You'll see the enclave output:
+
 ```sh
-$ kubectl apply -f sgx.yaml
-$ kubectl exec -ti sgx ls /dev/sgx/
-enclave    provision
+$ kubectl logs oesgx-demo-job-wh42g
+Hello world from the enclave
+Enclave called into host to print: Hello World!
 ```
 
-The output of the latest command shouldn't be empty, otherwise check
-your system environment to make sure SGX is fully supported.
+### Notes
 
-[1]: github.com/cloud-hypervisor/cloud-hypervisor/
+* The Kata VM's SGX Encrypted Page Cache (EPC) memory size is based on the sum of `sgx.intel.com/epc`
+resource requests within the pod.
+* `init-sgx` can be removed from the YAML configuration file if the Kata rootfs is modified with the
+necessary udev rules.
+   See the [note on SGX backwards compatibility](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/main/cmd/sgx_plugin#backwards-compatibility-note).
+* Intel SGX DCAP attestation is known to work from Kata sandboxes but it comes with one limitation: If
+the Intel SGX `aesm` daemon runs on the bare metal node and DCAP `out-of-proc` attestation is used,
+containers within the Kata sandbox cannot get the access to the host's `/var/run/aesmd/aesm.sock`
+because socket passthrough is not supported. An alternative is to deploy the `aesm` daemon as a side-car
+container.
+* Projects like [Gramine Shielded Containers (GSC)](https://gramine-gsc.readthedocs.io/en/latest/) are
+also known to work. For GSC specifically, the Kata guest kernel needs to have the `CONFIG_NUMA=y`
+enabled and at least one CPU online when running the GSC container.

pkg/logging/Cargo.toml

@@ -12,7 +12,7 @@ serde_json = "1.0.39"
 # - Dynamic keys required to allow HashMap keys to be slog::Serialized.
 # - The 'max_*' features allow changing the log level at runtime
 #   (by stopping the compiler from removing log calls).
-slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_info"] }
+slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug"] }
 slog-json = "2.3.0"
 slog-async = "2.3.0"
 slog-scope = "4.1.2"

snap/snapcraft.yaml

@@ -59,7 +59,7 @@ parts:
 
       yq_version=3.4.1
       yq_url="https://${yq_pkg}/releases/download/${yq_version}/yq_${goos}_${goarch}"
-      curl -o "${yq_path}" -LSsf "${yq_url}"
+      curl -o "${yq_path}" -L "${yq_url}"
       chmod +x "${yq_path}"
 
       kata_dir=gopath/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
@@ -139,7 +139,7 @@ parts:
       cp kata-containers*.img ${kata_image_dir}
 
   runtime:
-    after: [godeps, image]
+    after: [godeps, image, cloud-hypervisor]
     plugin: nil
     build-attributes: [no-patchelf]
     override-build: |
@@ -185,6 +185,7 @@ parts:
       - flex
     override-build: |
       yq=${SNAPCRAFT_STAGE}/yq
+      export PATH="${PATH}:${SNAPCRAFT_STAGE}"
       export GOPATH=${SNAPCRAFT_STAGE}/gopath
       kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
       versions_file="${kata_dir}/versions.yaml"
@@ -199,10 +200,17 @@ parts:
       kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
 
       cd ${kata_dir}/tools/packaging/kernel
+      kernel_dir_prefix="kata-linux-"
 
       # Setup and build kernel
-      ./build-kernel.sh -v ${kernel_version} -d setup
-      kernel_dir_prefix="kata-linux-"
+      if [ "$(uname -m)" = "x86_64" ]; then
+        kernel_version="$(${yq} r $versions_file assets.kernel-experimental.tag)"
+        kernel_version=${kernel_version#v}
+        kernel_dir_prefix="kata-linux-experimental-"
+        ./build-kernel.sh -e -v ${kernel_version} -d setup
+      else
+        ./build-kernel.sh -v ${kernel_version} -d setup
+      fi
       cd ${kernel_dir_prefix}*
       make -j $(($(nproc)-1)) EXTRAVERSION=".container"
 
@@ -327,6 +335,22 @@ parts:
       # Hack: move qemu to /
       "snap/kata-containers/current/": "./"
 
+  cloud-hypervisor:
+    plugin: nil
+    after: [godeps]
+    override-build: |
+      export GOPATH=${SNAPCRAFT_STAGE}/gopath
+      yq=${SNAPCRAFT_STAGE}/yq
+      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
+      versions_file="${kata_dir}/versions.yaml"
+      version="$(${yq} r ${versions_file} assets.hypervisor.cloud_hypervisor.version)"
+      url="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/${version}"
+      curl -L ${url}/cloud-hypervisor-static -o cloud-hypervisor
+      curl -LO ${url}/clh-remote
+
+      install -D cloud-hypervisor ${SNAPCRAFT_PART_INSTALL}/usr/bin/cloud-hypervisor
+      install -D clh-remote ${SNAPCRAFT_PART_INSTALL}/usr/bin/clh-remote
+
 apps:
   runtime:
     command: usr/bin/kata-runtime

src/agent/Cargo.lock

@@ -544,7 +544,9 @@ dependencies = [
  "rustjail",
  "scan_fmt",
  "scopeguard",
+ "serde",
  "serde_json",
+ "serial_test",
  "slog",
  "slog-scope",
  "slog-stdlog",
@@ -552,6 +554,7 @@ dependencies = [
  "thiserror",
  "tokio",
  "tokio-vsock",
+ "toml",
  "tracing",
  "tracing-opentelemetry",
  "tracing-subscriber",
@@ -591,6 +594,24 @@ dependencies = [
  "rle-decode-fast",
 ]
 
+[[package]]
+name = "libseccomp"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36ad71a5b66ceef3acfe6a3178b29b4da063f8bcb2c36dab666d52a7a9cfdb86"
+dependencies = [
+ "libc",
+ "libseccomp-sys",
+ "nix 0.17.0",
+ "pkg-config",
+]
+
+[[package]]
+name = "libseccomp-sys"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "539912de229a4fc16e507e8df12a394038a524a5b5b6c92045ad344472aac475"
+
 [[package]]
 name = "lock_api"
 version = "0.4.4"
@@ -760,6 +781,19 @@ dependencies = [
  "void",
 ]
 
+[[package]]
+name = "nix"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50e4785f2c3b7589a0d0c1dd60285e1188adac4006e8abd6dd578e1567027363"
+dependencies = [
+ "bitflags",
+ "cc",
+ "cfg-if 0.1.10",
+ "libc",
+ "void",
+]
+
 [[package]]
 name = "nix"
 version = "0.19.1"
@@ -974,6 +1008,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
+[[package]]
+name = "pkg-config"
+version = "0.3.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c"
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.10"
@@ -1281,6 +1321,7 @@ dependencies = [
  "inotify",
  "lazy_static",
  "libc",
+ "libseccomp",
  "nix 0.21.0",
  "oci",
  "path-absolutize",
@@ -1323,18 +1364,18 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
 [[package]]
 name = "serde"
-version = "1.0.126"
+version = "1.0.129"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03"
+checksum = "d1f72836d2aa753853178eda473a3b9d8e4eefdaf20523b919677e6de489f8f1"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.126"
+version = "1.0.129"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43"
+checksum = "e57ae87ad533d9a56427558b516d0adac283614e347abf85b0dc0cbbf0a249f3"
 dependencies = [
  "proc-macro2 1.0.26",
  "quote 1.0.9",
@@ -1618,6 +1659,15 @@ dependencies = [
  "vsock",
 ]
 
+[[package]]
+name = "toml"
+version = "0.5.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "tracing"
 version = "0.1.26"

src/agent/Cargo.toml

@@ -6,7 +6,6 @@ edition = "2018"
 
 [dependencies]
 oci = { path = "oci" }
-logging = { path = "../../pkg/logging" }
 rustjail = { path = "rustjail" }
 protocols = { path = "protocols" }
 lazy_static = "1.3.0"
@@ -20,6 +19,7 @@ scan_fmt = "0.2.3"
 scopeguard = "1.0.0"
 thiserror = "1.0.26"
 regex = "1"
+serial_test = "0.5.1"
 
 # Async helpers
 async-trait = "0.1.42"
@@ -35,11 +35,10 @@ rtnetlink = "0.8.0"
 netlink-packet-utils = "0.4.1"
 ipnetwork = "0.17.0"
 
-# slog:
-# - Dynamic keys required to allow HashMap keys to be slog::Serialized.
-# - The 'max_*' features allow changing the log level at runtime
-#   (by stopping the compiler from removing log calls).
-slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_info"] }
+# Note: this crate sets the slog 'max_*' features which allows the log level
+# to be modified at runtime.
+logging = { path = "../../pkg/logging" }
+slog = "2.5.2"
 slog-scope = "4.1.2"
 
 # Redirect ttrpc log calls
@@ -58,6 +57,10 @@ tracing-opentelemetry = "0.13.0"
 opentelemetry = { version = "0.14.0", features = ["rt-tokio-current-thread"]}
 vsock-exporter = { path = "vsock-exporter" }
 
+# Configuration
+serde = { version = "1.0.129", features = ["derive"] }
+toml = "0.5.8"
+
 [dev-dependencies]
 tempfile = "3.1.0"
 
@@ -70,3 +73,6 @@ members = [
 
 [profile.release]
 lto = true
+
+[features]
+seccomp = ["rustjail/seccomp"]

src/agent/Makefile

@@ -27,6 +27,20 @@ COMMIT_MSG = $(if $(COMMIT),$(COMMIT),unknown)
 # Exported to allow cargo to see it
 export VERSION_COMMIT := $(if $(COMMIT),$(VERSION)-$(COMMIT),$(VERSION))
 
+EXTRA_RUSTFEATURES :=
+
+##VAR SECCOMP=yes|no define if agent enables seccomp feature
+SECCOMP := yes
+
+# Enable seccomp feature of rust build
+ifeq ($(SECCOMP),yes)
+    override EXTRA_RUSTFEATURES += seccomp
+endif
+
+ifneq ($(EXTRA_RUSTFEATURES),)
+    override EXTRA_RUSTFEATURES := --features $(EXTRA_RUSTFEATURES)
+endif
+
 include ../../utils.mk
 
 TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET)
@@ -90,15 +104,14 @@ default: $(TARGET) show-header
 $(TARGET): $(GENERATED_CODE) $(TARGET_PATH)
 
 $(TARGET_PATH): $(SOURCES) | show-summary
-	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE)
+	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
 
 $(GENERATED_FILES): %: %.in
 	@sed $(foreach r,$(GENERATED_REPLACEMENTS),-e 's|@$r@|$($r)|g') "$<" > "$@"
 
 ##TARGET optimize: optimized  build
 optimize: $(SOURCES) | show-summary show-header
-	@RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny-warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE)
-
+	@RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny-warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
 
 ##TARGET clippy: run clippy linter
 clippy: $(GENERATED_CODE)
@@ -127,7 +140,7 @@ vendor:
 
 #TARGET test: run cargo tests
 test:
-	@cargo test --all --target $(TRIPLE) -- --nocapture
+	@cargo test --all --target $(TRIPLE) $(EXTRA_RUSTFEATURES) -- --nocapture
 
 ##TARGET check: run test
 check: clippy format

src/agent/README.md

@@ -19,6 +19,7 @@ After that, we drafted the initial code here, and any contributions are welcome.
 | I/O stream              | :white_check_mark: |
 | Cgroups                 | :white_check_mark: |
 | Capabilities, `rlimit`, readonly path, masked path, users | :white_check_mark: |
+| Seccomp                 | :white_check_mark: |
 | container stats (`stats_container`)                     | :white_check_mark: |
 | Hooks                   | :white_check_mark: |
 | **Agent Features & APIs** |

src/agent/protocols/protos/types.proto

@@ -46,6 +46,7 @@ message Route {
 	string device = 3;
 	string source = 4;
 	uint32 scope = 5;
+	IPFamily family = 6;
 }
 
 message ARPNeighbor {

src/agent/rustjail/Cargo.toml

@@ -30,7 +30,11 @@ tokio = { version = "1.2.0", features = ["sync", "io-util", "process", "time", "
 futures = "0.3"
 async-trait = "0.1.31"
 inotify = "0.9.2"
+libseccomp = { version = "0.1.3", optional = true }
 
 [dev-dependencies]
 serial_test = "0.5.0"
 tempfile = "3.1.0"
+
+[features]
+seccomp = ["libseccomp"]

src/agent/rustjail/src/container.rs

@@ -25,6 +25,8 @@ use crate::cgroups::mock::Manager as FsManager;
 use crate::cgroups::Manager;
 use crate::log_child;
 use crate::process::Process;
+#[cfg(feature = "seccomp")]
+use crate::seccomp;
 use crate::specconv::CreateOpts;
 use crate::{mount, validator};
 
@@ -151,7 +153,7 @@ lazy_static! {
             },
             LinuxDevice {
                 path: "/dev/full".to_string(),
-                r#type: String::from("c"),
+                r#type: "c".to_string(),
                 major: 1,
                 minor: 7,
                 file_mode: Some(0o666),
@@ -593,11 +595,22 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         })?;
     }
 
-    // NoNewPeiviledges, Drop capabilities
+    // NoNewPrivileges
     if oci_process.no_new_privileges {
         capctl::prctl::set_no_new_privs().map_err(|_| anyhow!("cannot set no new privileges"))?;
     }
 
+    // Without NoNewPrivileges, we need to set seccomp
+    // before dropping capabilities because the calling thread
+    // must have the CAP_SYS_ADMIN.
+    #[cfg(feature = "seccomp")]
+    if !oci_process.no_new_privileges {
+        if let Some(ref scmp) = linux.seccomp {
+            seccomp::init_seccomp(scmp)?;
+        }
+    }
+
+    // Drop capabilities
     if oci_process.capabilities.is_some() {
         let c = oci_process.capabilities.as_ref().unwrap();
         capabilities::drop_privileges(cfd_log, c)?;
@@ -641,7 +654,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     let exec_file = Path::new(&args[0]);
     log_child!(cfd_log, "process command: {:?}", &args);
     if !exec_file.exists() {
-        find_file(exec_file).ok_or_else(|| anyhow!("the file {} is not exist", &args[0]))?;
+        find_file(exec_file).ok_or_else(|| anyhow!("the file {} was not found", &args[0]))?;
     }
 
     // notify parent that the child's ready to start
@@ -669,6 +682,16 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         unistd::read(fd, &mut buf)?;
     }
 
+    // With NoNewPrivileges, we should set seccomp as close to
+    // do_exec as possible in order to reduce the amount of
+    // system calls in the seccomp profiles.
+    #[cfg(feature = "seccomp")]
+    if oci_process.no_new_privileges {
+        if let Some(ref scmp) = linux.seccomp {
+            seccomp::init_seccomp(scmp)?;
+        }
+    }
+
     do_exec(&args);
 }
 
@@ -833,6 +856,20 @@ impl BaseContainer for LinuxContainer {
         }
         let linux = spec.linux.as_ref().unwrap();
 
+        if p.oci.capabilities.is_none() {
+            // No capabilities, inherit from container process
+            let process = spec
+                .process
+                .as_ref()
+                .ok_or_else(|| anyhow!("no process config"))?;
+            p.oci.capabilities = Some(
+                process
+                    .capabilities
+                    .clone()
+                    .ok_or_else(|| anyhow!("missing process capabilities"))?,
+            );
+        }
+
         let (pfd_log, cfd_log) = unistd::pipe().context("failed to create pipe")?;
 
         let _ = fcntl::fcntl(pfd_log, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC))

src/agent/rustjail/src/lib.rs

@@ -34,6 +34,8 @@ pub mod container;
 pub mod mount;
 pub mod pipestream;
 pub mod process;
+#[cfg(feature = "seccomp")]
+pub mod seccomp;
 pub mod specconv;
 pub mod sync;
 pub mod sync_with_async;

src/agent/rustjail/src/mount.rs

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use anyhow::{anyhow, bail, Context, Result};
+use anyhow::{anyhow, Context, Result};
 use libc::uid_t;
 use nix::errno::Errno;
 use nix::fcntl::{self, OFlag};
@@ -19,7 +19,7 @@ use std::fs::{self, OpenOptions};
 use std::mem::MaybeUninit;
 use std::os::unix;
 use std::os::unix::io::RawFd;
-use std::path::{Path, PathBuf};
+use std::path::{Component, Path, PathBuf};
 
 use path_absolutize::*;
 use std::fs::File;
@@ -828,18 +828,35 @@ fn default_symlinks() -> Result<()> {
     }
     Ok(())
 }
+
+fn dev_rel_path(path: &str) -> Option<&Path> {
+    let path = Path::new(path);
+
+    if !path.starts_with("/dev")
+        || path == Path::new("/dev")
+        || path.components().any(|c| c == Component::ParentDir)
+    {
+        return None;
+    }
+    path.strip_prefix("/").ok()
+}
+
 fn create_devices(devices: &[LinuxDevice], bind: bool) -> Result<()> {
-    let op: fn(&LinuxDevice) -> Result<()> = if bind { bind_dev } else { mknod_dev };
+    let op: fn(&LinuxDevice, &Path) -> Result<()> = if bind { bind_dev } else { mknod_dev };
     let old = stat::umask(Mode::from_bits_truncate(0o000));
     for dev in DEFAULT_DEVICES.iter() {
-        op(dev)?;
+        let path = Path::new(&dev.path[1..]);
+        op(dev, path).context(format!("Creating container device {:?}", dev))?;
     }
     for dev in devices {
-        if !dev.path.starts_with("/dev") || dev.path.contains("..") {
+        let path = dev_rel_path(&dev.path).ok_or_else(|| {
             let msg = format!("{} is not a valid device path", dev.path);
-            bail!(anyhow!(msg));
+            anyhow!(msg)
+        })?;
+        if let Some(dir) = path.parent() {
+            fs::create_dir_all(dir).context(format!("Creating container device {:?}", dev))?;
         }
-        op(dev)?;
+        op(dev, path).context(format!("Creating container device {:?}", dev))?;
     }
     stat::umask(old);
     Ok(())
@@ -861,21 +878,21 @@ lazy_static! {
     };
 }
 
-fn mknod_dev(dev: &LinuxDevice) -> Result<()> {
+fn mknod_dev(dev: &LinuxDevice, relpath: &Path) -> Result<()> {
     let f = match LINUXDEVICETYPE.get(dev.r#type.as_str()) {
         Some(v) => v,
         None => return Err(anyhow!("invalid spec".to_string())),
     };
 
     stat::mknod(
-        &dev.path[1..],
+        relpath,
         *f,
         Mode::from_bits_truncate(dev.file_mode.unwrap_or(0)),
         nix::sys::stat::makedev(dev.major as u64, dev.minor as u64),
     )?;
 
     unistd::chown(
-        &dev.path[1..],
+        relpath,
         Some(Uid::from_raw(dev.uid.unwrap_or(0) as uid_t)),
         Some(Gid::from_raw(dev.gid.unwrap_or(0) as uid_t)),
     )?;
@@ -883,9 +900,9 @@ fn mknod_dev(dev: &LinuxDevice) -> Result<()> {
     Ok(())
 }
 
-fn bind_dev(dev: &LinuxDevice) -> Result<()> {
+fn bind_dev(dev: &LinuxDevice, relpath: &Path) -> Result<()> {
     let fd = fcntl::open(
-        &dev.path[1..],
+        relpath,
         OFlag::O_RDWR | OFlag::O_CREAT,
         Mode::from_bits_truncate(0o644),
     )?;
@@ -894,7 +911,7 @@ fn bind_dev(dev: &LinuxDevice) -> Result<()> {
 
     mount(
         Some(&*dev.path),
-        &dev.path[1..],
+        relpath,
         None::<&str>,
         MsFlags::MS_BIND,
         None::<&str>,
@@ -1258,11 +1275,12 @@ mod tests {
             uid: Some(unistd::getuid().as_raw()),
             gid: Some(unistd::getgid().as_raw()),
         };
+        let path = Path::new("fifo");
 
-        let ret = mknod_dev(&dev);
+        let ret = mknod_dev(&dev, path);
         assert!(ret.is_ok(), "Should pass. Got: {:?}", ret);
 
-        let ret = stat::stat("fifo");
+        let ret = stat::stat(path);
         assert!(ret.is_ok(), "Should pass. Got: {:?}", ret);
     }
     #[test]
@@ -1379,4 +1397,26 @@ mod tests {
             assert!(result == t.result, "{}", msg);
         }
     }
+
+    #[test]
+    fn test_dev_rel_path() {
+        // Valid device paths
+        assert_eq!(dev_rel_path("/dev/sda").unwrap(), Path::new("dev/sda"));
+        assert_eq!(dev_rel_path("//dev/sda").unwrap(), Path::new("dev/sda"));
+        assert_eq!(
+            dev_rel_path("/dev/vfio/99").unwrap(),
+            Path::new("dev/vfio/99")
+        );
+        assert_eq!(dev_rel_path("/dev/...").unwrap(), Path::new("dev/..."));
+        assert_eq!(dev_rel_path("/dev/a..b").unwrap(), Path::new("dev/a..b"));
+        assert_eq!(dev_rel_path("/dev//foo").unwrap(), Path::new("dev/foo"));
+
+        // Bad device paths
+        assert!(dev_rel_path("/devfoo").is_none());
+        assert!(dev_rel_path("/etc/passwd").is_none());
+        assert!(dev_rel_path("/dev/../etc/passwd").is_none());
+        assert!(dev_rel_path("dev/foo").is_none());
+        assert!(dev_rel_path("").is_none());
+        assert!(dev_rel_path("/dev").is_none());
+    }
 }

src/agent/rustjail/src/process.rs

@@ -24,6 +24,16 @@ use tokio::io::{split, ReadHalf, WriteHalf};
 use tokio::sync::Mutex;
 use tokio::sync::Notify;
 
+macro_rules! close_process_stream {
+    ($self: ident, $stream:ident, $stream_type: ident) => {
+        if $self.$stream.is_some() {
+            $self.close_stream(StreamType::$stream_type);
+            let _ = unistd::close($self.$stream.unwrap());
+            $self.$stream = None;
+        }
+    };
+}
+
 #[derive(Debug, PartialEq, Eq, Hash, Clone)]
 pub enum StreamType {
     Stdin,
@@ -147,6 +157,22 @@ impl Process {
         notify.notify_one();
     }
 
+    pub fn close_stdin(&mut self) {
+        close_process_stream!(self, term_master, TermMaster);
+        close_process_stream!(self, parent_stdin, ParentStdin);
+
+        self.notify_term_close();
+    }
+
+    pub fn cleanup_process_stream(&mut self) {
+        close_process_stream!(self, parent_stdin, ParentStdin);
+        close_process_stream!(self, parent_stdout, ParentStdout);
+        close_process_stream!(self, parent_stderr, ParentStderr);
+        close_process_stream!(self, term_master, TermMaster);
+
+        self.notify_term_close();
+    }
+
     fn get_fd(&self, stream_type: &StreamType) -> Option<RawFd> {
         match stream_type {
             StreamType::Stdin => self.stdin,

src/agent/rustjail/src/seccomp.rs

@@ -0,0 +1,237 @@
+// Copyright 2021 Sony Group Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use anyhow::{anyhow, Result};
+use libseccomp::*;
+use oci::{LinuxSeccomp, LinuxSeccompArg};
+use std::str::FromStr;
+
+fn get_filter_attr_from_flag(flag: &str) -> Result<ScmpFilterAttr> {
+    match flag {
+        "SECCOMP_FILTER_FLAG_TSYNC" => Ok(ScmpFilterAttr::CtlTsync),
+        "SECCOMP_FILTER_FLAG_LOG" => Ok(ScmpFilterAttr::CtlLog),
+        "SECCOMP_FILTER_FLAG_SPEC_ALLOW" => Ok(ScmpFilterAttr::CtlSsb),
+        _ => Err(anyhow!("Invalid seccomp flag")),
+    }
+}
+
+// get_rule_conditions gets rule conditions for a system call from the args.
+fn get_rule_conditions(args: &[LinuxSeccompArg]) -> Result<Vec<ScmpArgCompare>> {
+    let mut conditions: Vec<ScmpArgCompare> = Vec::new();
+
+    for arg in args {
+        if arg.op.is_empty() {
+            return Err(anyhow!("seccomp opreator is required"));
+        }
+
+        let cond = ScmpArgCompare::new(
+            arg.index,
+            ScmpCompareOp::from_str(&arg.op)?,
+            arg.value,
+            Some(arg.value_two),
+        );
+
+        conditions.push(cond);
+    }
+
+    Ok(conditions)
+}
+
+// init_seccomp creates a seccomp filter and loads it for the current process
+// including all the child processes.
+pub fn init_seccomp(scmp: &LinuxSeccomp) -> Result<()> {
+    let def_action = ScmpAction::from_str(scmp.default_action.as_str(), Some(libc::EPERM as u32))?;
+
+    // Create a new filter context
+    let mut filter = ScmpFilterContext::new_filter(def_action)?;
+
+    // Add extra architectures
+    for arch in &scmp.architectures {
+        let scmp_arch = ScmpArch::from_str(arch)?;
+        filter.add_arch(scmp_arch)?;
+    }
+
+    // Unset no new privileges bit
+    filter.set_no_new_privs_bit(false)?;
+
+    // Add a rule for each system call
+    for syscall in &scmp.syscalls {
+        if syscall.names.is_empty() {
+            return Err(anyhow!("syscall name is required"));
+        }
+
+        let action = ScmpAction::from_str(&syscall.action, Some(syscall.errno_ret))?;
+        if action == def_action {
+            continue;
+        }
+
+        for name in &syscall.names {
+            let syscall_num = get_syscall_from_name(name, None)?;
+
+            if syscall.args.is_empty() {
+                filter.add_rule(action, syscall_num, None)?;
+            } else {
+                let conditions = get_rule_conditions(&syscall.args)?;
+                filter.add_rule(action, syscall_num, Some(&conditions))?;
+            }
+        }
+    }
+
+    // Set filter attributes for each seccomp flag
+    for flag in &scmp.flags {
+        let scmp_attr = get_filter_attr_from_flag(flag)?;
+        filter.set_filter_attr(scmp_attr, 1)?;
+    }
+
+    // Load the filter
+    filter.load()?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::skip_if_not_root;
+    use libc::{dup3, process_vm_readv, EPERM, O_CLOEXEC};
+    use std::io::Error;
+    use std::ptr::null;
+
+    macro_rules! syscall_assert {
+        ($e1: expr, $e2: expr) => {
+            let mut errno: i32 = 0;
+            if $e1 < 0 {
+                errno = -Error::last_os_error().raw_os_error().unwrap();
+            }
+            assert_eq!(errno, $e2);
+        };
+    }
+
+    #[test]
+    fn test_get_filter_attr_from_flag() {
+        skip_if_not_root!();
+
+        assert_eq!(
+            get_filter_attr_from_flag("SECCOMP_FILTER_FLAG_TSYNC").unwrap(),
+            ScmpFilterAttr::CtlTsync
+        );
+
+        assert_eq!(get_filter_attr_from_flag("ERROR").is_err(), true);
+    }
+
+    #[test]
+    fn test_init_seccomp() {
+        skip_if_not_root!();
+
+        let data = r#"{
+            "defaultAction": "SCMP_ACT_ALLOW",
+            "architectures": [
+            ],
+            "flags": [
+                "SECCOMP_FILTER_FLAG_LOG"
+            ],
+            "syscalls": [
+                {
+                   "names": [
+                        "dup3"
+                    ],
+                    "action": "SCMP_ACT_ERRNO"
+                },
+                {
+                   "names": [
+                        "process_vm_readv"
+                    ],
+                    "action": "SCMP_ACT_ERRNO",
+                    "errnoRet": 111,
+                    "args": [
+                        {
+                            "index": 0,
+                            "value": 10,
+                            "op": "SCMP_CMP_EQ"
+                        }
+                    ]
+                },
+                {
+                   "names": [
+                        "process_vm_readv"
+                    ],
+                    "action": "SCMP_ACT_ERRNO",
+                    "errnoRet": 111,
+                    "args": [
+                        {
+                            "index": 0,
+                            "value": 20,
+                            "op": "SCMP_CMP_EQ"
+                        }
+                    ]
+                },
+                {
+                   "names": [
+                        "process_vm_readv"
+                    ],
+                    "action": "SCMP_ACT_ERRNO",
+                    "errnoRet": 222,
+                    "args": [
+                        {
+                            "index": 0,
+                            "value": 30,
+                            "op": "SCMP_CMP_EQ"
+                        },
+                        {
+                            "index": 2,
+                            "value": 40,
+                            "op": "SCMP_CMP_EQ"
+                        }
+                    ]
+                }
+            ]
+        }"#;
+
+        let mut scmp: oci::LinuxSeccomp = serde_json::from_str(data).unwrap();
+        let mut arch: Vec<oci::Arch>;
+
+        if cfg!(target_endian = "little") {
+            // For little-endian architectures
+            arch = vec![
+                "SCMP_ARCH_X86".to_string(),
+                "SCMP_ARCH_X32".to_string(),
+                "SCMP_ARCH_X86_64".to_string(),
+                "SCMP_ARCH_AARCH64".to_string(),
+                "SCMP_ARCH_ARM".to_string(),
+                "SCMP_ARCH_PPC64LE".to_string(),
+            ];
+        } else {
+            // For big-endian architectures
+            arch = vec!["SCMP_ARCH_S390X".to_string()];
+        }
+
+        scmp.architectures.append(&mut arch);
+
+        init_seccomp(&scmp).unwrap();
+
+        // Basic syscall with simple rule
+        syscall_assert!(unsafe { dup3(0, 1, O_CLOEXEC) }, -EPERM);
+
+        // Syscall with permitted arguments
+        syscall_assert!(unsafe { process_vm_readv(1, null(), 0, null(), 0, 0) }, 0);
+
+        // Multiple arguments with OR rules with ERRNO
+        syscall_assert!(
+            unsafe { process_vm_readv(10, null(), 0, null(), 0, 0) },
+            -111
+        );
+        syscall_assert!(
+            unsafe { process_vm_readv(20, null(), 0, null(), 0, 0) },
+            -111
+        );
+
+        // Multiple arguments with AND rules with ERRNO
+        syscall_assert!(unsafe { process_vm_readv(30, null(), 0, null(), 0, 0) }, 0);
+        syscall_assert!(
+            unsafe { process_vm_readv(30, null(), 40, null(), 0, 0) },
+            -222
+        );
+    }
+}

src/agent/samples/configuration-all-endpoints.toml

@@ -0,0 +1,41 @@
+# This is an agent configuration file example.
+dev_mode = true
+server_addr = 'vsock://8:2048'
+
+[endpoints]
+# All endpoints are allowed
+allowed = [
+        "AddARPNeighborsRequest",
+        "AddSwapRequest",
+        "CloseStdinRequest",
+        "CopyFileRequest",
+        "CreateContainerRequest",
+        "CreateSandboxRequest",
+        "DestroySandboxRequest",
+        "ExecProcessRequest",
+        "GetMetricsRequest",
+        "GetOOMEventRequest",
+        "GuestDetailsRequest",
+        "ListInterfacesRequest",
+        "ListRoutesRequest",
+        "MemHotplugByProbeRequest",
+        "OnlineCPUMemRequest",
+        "PauseContainerRequest",
+        "PullImageRequest",
+        "ReadStreamRequest",
+        "RemoveContainerRequest",
+        "ReseedRandomDevRequest",
+        "ResumeContainerRequest",
+        "SetGuestDateTimeRequest",
+        "SignalProcessRequest",
+        "StartContainerRequest",
+        "StartTracingRequest",
+        "StatsContainerRequest",
+        "StopTracingRequest",
+        "TtyWinResizeRequest",
+        "UpdateContainerRequest",
+        "UpdateInterfaceRequest",
+        "UpdateRoutesRequest",
+        "WaitProcessRequest",
+        "WriteStreamRequest"
+]

src/agent/src/config.rs

@@ -2,10 +2,13 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 //
-use crate::tracer;
+use crate::rpc;
 use anyhow::{bail, ensure, Context, Result};
+use serde::Deserialize;
+use std::collections::HashSet;
 use std::env;
 use std::fs;
+use std::str::FromStr;
 use std::time;
 use tracing::instrument;
 
@@ -19,6 +22,7 @@ const DEBUG_CONSOLE_VPORT_OPTION: &str = "agent.debug_console_vport";
 const LOG_VPORT_OPTION: &str = "agent.log_vport";
 const CONTAINER_PIPE_SIZE_OPTION: &str = "agent.container_pipe_size";
 const UNIFIED_CGROUP_HIERARCHY_OPTION: &str = "agent.unified_cgroup_hierarchy";
+const CONFIG_FILE: &str = "agent.config_file";
 
 const DEFAULT_LOG_LEVEL: slog::Level = slog::Level::Info;
 const DEFAULT_HOTPLUG_TIMEOUT: time::Duration = time::Duration::from_secs(3);
@@ -29,7 +33,7 @@ const VSOCK_PORT: u16 = 1024;
 // Environment variables used for development and testing
 const SERVER_ADDR_ENV_VAR: &str = "KATA_AGENT_SERVER_ADDR";
 const LOG_LEVEL_ENV_VAR: &str = "KATA_AGENT_LOG_LEVEL";
-const TRACE_TYPE_ENV_VAR: &str = "KATA_AGENT_TRACE_TYPE";
+const TRACING_ENV_VAR: &str = "KATA_AGENT_TRACING";
 
 const ERR_INVALID_LOG_LEVEL: &str = "invalid log level";
 const ERR_INVALID_LOG_LEVEL_PARAM: &str = "invalid log level parameter";
@@ -47,6 +51,17 @@ const ERR_INVALID_CONTAINER_PIPE_SIZE_PARAM: &str = "unable to parse container p
 const ERR_INVALID_CONTAINER_PIPE_SIZE_KEY: &str = "invalid container pipe size key name";
 const ERR_INVALID_CONTAINER_PIPE_NEGATIVE: &str = "container pipe size should not be negative";
 
+#[derive(Debug, Default, Deserialize)]
+pub struct EndpointsConfig {
+    pub allowed: Vec<String>,
+}
+
+#[derive(Debug, Default)]
+pub struct AgentEndpoints {
+    pub allowed: HashSet<String>,
+    pub all_allowed: bool,
+}
+
 #[derive(Debug)]
 pub struct AgentConfig {
     pub debug_console: bool,
@@ -58,7 +73,38 @@ pub struct AgentConfig {
     pub container_pipe_size: i32,
     pub server_addr: String,
     pub unified_cgroup_hierarchy: bool,
-    pub tracing: tracer::TraceType,
+    pub tracing: bool,
+    pub endpoints: AgentEndpoints,
+    pub supports_seccomp: bool,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct AgentConfigBuilder {
+    pub debug_console: Option<bool>,
+    pub dev_mode: Option<bool>,
+    pub log_level: Option<String>,
+    pub hotplug_timeout: Option<time::Duration>,
+    pub debug_console_vport: Option<i32>,
+    pub log_vport: Option<i32>,
+    pub container_pipe_size: Option<i32>,
+    pub server_addr: Option<String>,
+    pub unified_cgroup_hierarchy: Option<bool>,
+    pub tracing: Option<bool>,
+    pub endpoints: Option<EndpointsConfig>,
+}
+
+macro_rules! config_override {
+    ($builder:ident, $config:ident, $field:ident) => {
+        if let Some(v) = $builder.$field {
+            $config.$field = v;
+        }
+    };
+
+    ($builder:ident, $config:ident, $field:ident, $func: ident) => {
+        if let Some(v) = $builder.$field {
+            $config.$field = $func(&v)?;
+        }
+    };
 }
 
 // parse_cmdline_param parse commandline parameters.
@@ -91,8 +137,8 @@ macro_rules! parse_cmdline_param {
     };
 }
 
-impl AgentConfig {
-    pub fn new() -> AgentConfig {
+impl Default for AgentConfig {
+    fn default() -> Self {
         AgentConfig {
             debug_console: false,
             dev_mode: false,
@@ -103,34 +149,84 @@ impl AgentConfig {
             container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
             server_addr: format!("{}:{}", VSOCK_ADDR, VSOCK_PORT),
             unified_cgroup_hierarchy: false,
-            tracing: tracer::TraceType::Disabled,
+            tracing: false,
+            endpoints: Default::default(),
+            supports_seccomp: rpc::have_seccomp(),
         }
     }
+}
 
+impl FromStr for AgentConfig {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let agent_config_builder: AgentConfigBuilder =
+            toml::from_str(s).map_err(anyhow::Error::new)?;
+        let mut agent_config: AgentConfig = Default::default();
+
+        // Overwrite default values with the configuration files ones.
+        config_override!(agent_config_builder, agent_config, debug_console);
+        config_override!(agent_config_builder, agent_config, dev_mode);
+        config_override!(
+            agent_config_builder,
+            agent_config,
+            log_level,
+            logrus_to_slog_level
+        );
+        config_override!(agent_config_builder, agent_config, hotplug_timeout);
+        config_override!(agent_config_builder, agent_config, debug_console_vport);
+        config_override!(agent_config_builder, agent_config, log_vport);
+        config_override!(agent_config_builder, agent_config, container_pipe_size);
+        config_override!(agent_config_builder, agent_config, server_addr);
+        config_override!(agent_config_builder, agent_config, unified_cgroup_hierarchy);
+        config_override!(agent_config_builder, agent_config, tracing);
+
+        // Populate the allowed endpoints hash set, if we got any from the config file.
+        if let Some(endpoints) = agent_config_builder.endpoints {
+            for ep in endpoints.allowed {
+                agent_config.endpoints.allowed.insert(ep);
+            }
+        }
+
+        Ok(agent_config)
+    }
+}
+
+impl AgentConfig {
     #[instrument]
-    pub fn parse_cmdline(&mut self, file: &str) -> Result<()> {
+    pub fn from_cmdline(file: &str) -> Result<AgentConfig> {
+        let mut config: AgentConfig = Default::default();
         let cmdline = fs::read_to_string(file)?;
         let params: Vec<&str> = cmdline.split_ascii_whitespace().collect();
         for param in params.iter() {
+            // If we get a configuration file path from the command line, we
+            // generate our config from it.
+            // The agent will fail to start if the configuration file is not present,
+            // or if it can't be parsed properly.
+            if param.starts_with(format!("{}=", CONFIG_FILE).as_str()) {
+                let config_file = get_string_value(param)?;
+                return AgentConfig::from_config_file(&config_file);
+            }
+
             // parse cmdline flags
-            parse_cmdline_param!(param, DEBUG_CONSOLE_FLAG, self.debug_console);
-            parse_cmdline_param!(param, DEV_MODE_FLAG, self.dev_mode);
+            parse_cmdline_param!(param, DEBUG_CONSOLE_FLAG, config.debug_console);
+            parse_cmdline_param!(param, DEV_MODE_FLAG, config.dev_mode);
 
             // Support "bare" tracing option for backwards compatibility with
             // Kata 1.x.
             if param == &TRACE_MODE_OPTION {
-                self.tracing = tracer::TraceType::Isolated;
+                config.tracing = true;
                 continue;
             }
 
-            parse_cmdline_param!(param, TRACE_MODE_OPTION, self.tracing, get_trace_type);
+            parse_cmdline_param!(param, TRACE_MODE_OPTION, config.tracing, get_bool_value);
 
             // parse cmdline options
-            parse_cmdline_param!(param, LOG_LEVEL_OPTION, self.log_level, get_log_level);
+            parse_cmdline_param!(param, LOG_LEVEL_OPTION, config.log_level, get_log_level);
             parse_cmdline_param!(
                 param,
                 SERVER_ADDR_OPTION,
-                self.server_addr,
+                config.server_addr,
                 get_string_value
             );
 
@@ -138,7 +234,7 @@ impl AgentConfig {
             parse_cmdline_param!(
                 param,
                 HOTPLUG_TIMOUT_OPTION,
-                self.hotplug_timeout,
+                config.hotplug_timeout,
                 get_hotplug_timeout,
                 |hotplug_timeout: time::Duration| hotplug_timeout.as_secs() > 0
             );
@@ -147,14 +243,14 @@ impl AgentConfig {
             parse_cmdline_param!(
                 param,
                 DEBUG_CONSOLE_VPORT_OPTION,
-                self.debug_console_vport,
+                config.debug_console_vport,
                 get_vsock_port,
                 |port| port > 0
             );
             parse_cmdline_param!(
                 param,
                 LOG_VPORT_OPTION,
-                self.log_vport,
+                config.log_vport,
                 get_vsock_port,
                 |port| port > 0
             );
@@ -162,34 +258,47 @@ impl AgentConfig {
             parse_cmdline_param!(
                 param,
                 CONTAINER_PIPE_SIZE_OPTION,
-                self.container_pipe_size,
+                config.container_pipe_size,
                 get_container_pipe_size
             );
             parse_cmdline_param!(
                 param,
                 UNIFIED_CGROUP_HIERARCHY_OPTION,
-                self.unified_cgroup_hierarchy,
+                config.unified_cgroup_hierarchy,
                 get_bool_value
             );
         }
 
         if let Ok(addr) = env::var(SERVER_ADDR_ENV_VAR) {
-            self.server_addr = addr;
+            config.server_addr = addr;
         }
 
         if let Ok(addr) = env::var(LOG_LEVEL_ENV_VAR) {
             if let Ok(level) = logrus_to_slog_level(&addr) {
-                self.log_level = level;
+                config.log_level = level;
             }
         }
 
-        if let Ok(value) = env::var(TRACE_TYPE_ENV_VAR) {
-            if let Ok(result) = value.parse::<tracer::TraceType>() {
-                self.tracing = result;
-            }
+        if let Ok(value) = env::var(TRACING_ENV_VAR) {
+            let name_value = format!("{}={}", TRACING_ENV_VAR, value);
+
+            config.tracing = get_bool_value(&name_value)?;
         }
 
-        Ok(())
+        // We did not get a configuration file: allow all endpoints.
+        config.endpoints.all_allowed = true;
+
+        Ok(config)
+    }
+
+    #[instrument]
+    pub fn from_config_file(file: &str) -> Result<AgentConfig> {
+        let config = fs::read_to_string(file)?;
+        AgentConfig::from_str(&config)
+    }
+
+    pub fn is_allowed_endpoint(&self, ep: &str) -> bool {
+        self.endpoints.all_allowed || self.endpoints.allowed.contains(ep)
     }
 }
 
@@ -236,25 +345,6 @@ fn get_log_level(param: &str) -> Result<slog::Level> {
     logrus_to_slog_level(fields[1])
 }
 
-#[instrument]
-fn get_trace_type(param: &str) -> Result<tracer::TraceType> {
-    ensure!(!param.is_empty(), "invalid trace type parameter");
-
-    let fields: Vec<&str> = param.split('=').collect();
-    ensure!(
-        fields[0] == TRACE_MODE_OPTION,
-        "invalid trace type key name"
-    );
-
-    if fields.len() == 1 {
-        return Ok(tracer::TraceType::Isolated);
-    }
-
-    let result = fields[1].parse::<tracer::TraceType>()?;
-
-    Ok(result)
-}
-
 #[instrument]
 fn get_hotplug_timeout(param: &str) -> Result<time::Duration> {
     let fields: Vec<&str> = param.split('=').collect();
@@ -339,10 +429,6 @@ mod tests {
     use std::time;
     use tempfile::tempdir;
 
-    const ERR_INVALID_TRACE_TYPE_PARAM: &str = "invalid trace type parameter";
-    const ERR_INVALID_TRACE_TYPE: &str = "invalid trace type";
-    const ERR_INVALID_TRACE_TYPE_KEY: &str = "invalid trace type key name";
-
     // Parameters:
     //
     // 1: expected Result
@@ -371,7 +457,7 @@ mod tests {
 
     #[test]
     fn test_new() {
-        let config = AgentConfig::new();
+        let config: AgentConfig = Default::default();
         assert!(!config.debug_console);
         assert!(!config.dev_mode);
         assert_eq!(config.log_level, DEFAULT_LOG_LEVEL);
@@ -379,7 +465,7 @@ mod tests {
     }
 
     #[test]
-    fn test_parse_cmdline() {
+    fn test_from_cmdline() {
         const TEST_SERVER_ADDR: &str = "vsock://-1:1024";
 
         #[derive(Debug)]
@@ -393,7 +479,7 @@ mod tests {
             container_pipe_size: i32,
             server_addr: &'a str,
             unified_cgroup_hierarchy: bool,
-            tracing: tracer::TraceType,
+            tracing: bool,
         }
 
         impl Default for TestData<'_> {
@@ -408,7 +494,7 @@ mod tests {
                     container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
                     server_addr: TEST_SERVER_ADDR,
                     unified_cgroup_hierarchy: false,
-                    tracing: tracer::TraceType::Disabled,
+                    tracing: false,
                 }
             }
         }
@@ -667,64 +753,121 @@ mod tests {
             },
             TestData {
                 contents: "trace",
-                tracing: tracer::TraceType::Disabled,
+                tracing: false,
                 ..Default::default()
             },
             TestData {
                 contents: ".trace",
-                tracing: tracer::TraceType::Disabled,
+                tracing: false,
                 ..Default::default()
             },
             TestData {
                 contents: "agent.tracer",
-                tracing: tracer::TraceType::Disabled,
+                tracing: false,
                 ..Default::default()
             },
             TestData {
                 contents: "agent.trac",
-                tracing: tracer::TraceType::Disabled,
+                tracing: false,
                 ..Default::default()
             },
             TestData {
                 contents: "agent.trace",
-                tracing: tracer::TraceType::Isolated,
+                tracing: true,
                 ..Default::default()
             },
             TestData {
-                contents: "agent.trace=isolated",
-                tracing: tracer::TraceType::Isolated,
+                contents: "agent.trace=true",
+                tracing: true,
                 ..Default::default()
             },
             TestData {
-                contents: "agent.trace=disabled",
-                tracing: tracer::TraceType::Disabled,
+                contents: "agent.trace=false",
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.trace=0",
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.trace=1",
+                tracing: true,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.trace=a",
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.trace=foo",
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.trace=.",
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.trace=,",
+                tracing: false,
                 ..Default::default()
             },
             TestData {
                 contents: "",
-                env_vars: vec!["KATA_AGENT_TRACE_TYPE=isolated"],
-                tracing: tracer::TraceType::Isolated,
+                env_vars: vec!["KATA_AGENT_TRACING="],
+                tracing: false,
                 ..Default::default()
             },
             TestData {
                 contents: "",
-                env_vars: vec!["KATA_AGENT_TRACE_TYPE=disabled"],
-                tracing: tracer::TraceType::Disabled,
+                env_vars: vec!["KATA_AGENT_TRACING=''"],
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_TRACING=0"],
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_TRACING=."],
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_TRACING=,"],
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_TRACING=foo"],
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_TRACING=1"],
+                tracing: true,
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_TRACING=true"],
+                tracing: true,
                 ..Default::default()
             },
         ];
 
         let dir = tempdir().expect("failed to create tmpdir");
 
-        // First, check a missing file is handled
-        let file_path = dir.path().join("enoent");
-
-        let filename = file_path.to_str().expect("failed to create filename");
-
-        let mut config = AgentConfig::new();
-        let result = config.parse_cmdline(&filename.to_owned());
-        assert!(result.is_err());
-
         // Now, test various combinations of file contents and environment
         // variables.
         for (i, d) in tests.iter().enumerate() {
@@ -753,22 +896,7 @@ mod tests {
                 vars_to_unset.push(name);
             }
 
-            let mut config = AgentConfig::new();
-            assert!(!config.debug_console, "{}", msg);
-            assert!(!config.dev_mode, "{}", msg);
-            assert!(!config.unified_cgroup_hierarchy, "{}", msg);
-            assert_eq!(
-                config.hotplug_timeout,
-                time::Duration::from_secs(3),
-                "{}",
-                msg
-            );
-            assert_eq!(config.container_pipe_size, 0, "{}", msg);
-            assert_eq!(config.server_addr, TEST_SERVER_ADDR, "{}", msg);
-            assert_eq!(config.tracing, tracer::TraceType::Disabled, "{}", msg);
-
-            let result = config.parse_cmdline(filename);
-            assert!(result.is_ok(), "{}", msg);
+            let config = AgentConfig::from_cmdline(filename).expect("Failed to parse command line");
 
             assert_eq!(d.debug_console, config.debug_console, "{}", msg);
             assert_eq!(d.dev_mode, config.dev_mode, "{}", msg);
@@ -1220,60 +1348,33 @@ Caused by:
     }
 
     #[test]
-    fn test_get_trace_type() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            param: &'a str,
-            result: Result<tracer::TraceType>,
-        }
+    fn test_config_builder_from_string() {
+        let config = AgentConfig::from_str(
+            r#"
+               dev_mode = true
+               server_addr = 'vsock://8:2048'
 
-        let tests = &[
-            TestData {
-                param: "",
-                result: Err(anyhow!(ERR_INVALID_TRACE_TYPE_PARAM)),
-            },
-            TestData {
-                param: "agent.tracer",
-                result: Err(anyhow!(ERR_INVALID_TRACE_TYPE_KEY)),
-            },
-            TestData {
-                param: "agent.trac",
-                result: Err(anyhow!(ERR_INVALID_TRACE_TYPE_KEY)),
-            },
-            TestData {
-                param: "agent.trace=",
-                result: Err(anyhow!(ERR_INVALID_TRACE_TYPE)),
-            },
-            TestData {
-                param: "agent.trace==",
-                result: Err(anyhow!(ERR_INVALID_TRACE_TYPE)),
-            },
-            TestData {
-                param: "agent.trace=foo",
-                result: Err(anyhow!(ERR_INVALID_TRACE_TYPE)),
-            },
-            TestData {
-                param: "agent.trace",
-                result: Ok(tracer::TraceType::Isolated),
-            },
-            TestData {
-                param: "agent.trace=isolated",
-                result: Ok(tracer::TraceType::Isolated),
-            },
-            TestData {
-                param: "agent.trace=disabled",
-                result: Ok(tracer::TraceType::Disabled),
-            },
-        ];
+               [endpoints]
+               allowed = ["CreateContainer", "StartContainer"]
+              "#,
+        )
+        .unwrap();
 
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
+        // Verify that the all_allowed flag is false
+        assert!(!config.endpoints.all_allowed);
 
-            let result = get_trace_type(d.param);
+        // Verify that the override worked
+        assert!(config.dev_mode);
+        assert_eq!(config.server_addr, "vsock://8:2048");
+        assert_eq!(
+            config.endpoints.allowed,
+            vec!["CreateContainer".to_string(), "StartContainer".to_string()]
+                .iter()
+                .cloned()
+                .collect()
+        );
 
-            let msg = format!("{}: result: {:?}", msg, result);
-
-            assert_result!(d.result, result, msg);
-        }
+        // Verify that the default values are valid
+        assert_eq!(config.hotplug_timeout, DEFAULT_HOTPLUG_TIMEOUT);
     }
 }

src/agent/src/device.rs

@@ -7,7 +7,10 @@ use libc::{c_uint, major, minor};
 use nix::sys::stat;
 use regex::Regex;
 use std::collections::HashMap;
+use std::ffi::OsStr;
+use std::fmt;
 use std::fs;
+use std::os::unix::ffi::OsStrExt;
 use std::os::unix::fs::MetadataExt;
 use std::path::Path;
 use std::str::FromStr;
@@ -17,10 +20,6 @@ use tokio::sync::Mutex;
 #[cfg(target_arch = "s390x")]
 use crate::ccw;
 use crate::linux_abi::*;
-use crate::mount::{
-    DRIVER_BLK_CCW_TYPE, DRIVER_BLK_TYPE, DRIVER_MMIO_BLK_TYPE, DRIVER_NVDIMM_TYPE,
-    DRIVER_SCSI_TYPE,
-};
 use crate::pci;
 use crate::sandbox::Sandbox;
 use crate::uevent::{wait_for_uevent, Uevent, UeventMatcher};
@@ -38,6 +37,22 @@ macro_rules! sl {
 
 const VM_ROOTFS: &str = "/";
 
+pub const DRIVER_9P_TYPE: &str = "9p";
+pub const DRIVER_VIRTIOFS_TYPE: &str = "virtio-fs";
+pub const DRIVER_BLK_TYPE: &str = "blk";
+pub const DRIVER_BLK_CCW_TYPE: &str = "blk-ccw";
+pub const DRIVER_MMIO_BLK_TYPE: &str = "mmioblk";
+pub const DRIVER_SCSI_TYPE: &str = "scsi";
+pub const DRIVER_NVDIMM_TYPE: &str = "nvdimm";
+pub const DRIVER_EPHEMERAL_TYPE: &str = "ephemeral";
+pub const DRIVER_LOCAL_TYPE: &str = "local";
+pub const DRIVER_WATCHABLE_BIND_TYPE: &str = "watchable-bind";
+// VFIO device to be bound to a guest kernel driver
+pub const DRIVER_VFIO_GK_TYPE: &str = "vfio-gk";
+// VFIO device to be bound to vfio-pci and made available inside the
+// container as a VFIO device node
+pub const DRIVER_VFIO_TYPE: &str = "vfio";
+
 #[derive(Debug)]
 struct DevIndexEntry {
     idx: usize,
@@ -47,17 +62,89 @@ struct DevIndexEntry {
 #[derive(Debug)]
 struct DevIndex(HashMap<String, DevIndexEntry>);
 
-#[instrument]
-pub fn rescan_pci_bus() -> Result<()> {
-    online_device(SYSFS_PCI_BUS_RESCAN_FILE)
-}
-
 #[instrument]
 pub fn online_device(path: &str) -> Result<()> {
     fs::write(path, "1")?;
     Ok(())
 }
 
+// Force a given PCI device to bind to the given driver, does
+// basically the same thing as
+//    driverctl set-override <PCI address> <driver>
+#[instrument]
+pub fn pci_driver_override<T, U>(syspci: T, dev: pci::Address, drv: U) -> Result<()>
+where
+    T: AsRef<OsStr> + std::fmt::Debug,
+    U: AsRef<OsStr> + std::fmt::Debug,
+{
+    let syspci = Path::new(&syspci);
+    let drv = drv.as_ref();
+    info!(sl!(), "rebind_pci_driver: {} => {:?}", dev, drv);
+
+    let devpath = syspci.join("devices").join(dev.to_string());
+    let overridepath = &devpath.join("driver_override");
+
+    fs::write(overridepath, drv.as_bytes())?;
+
+    let drvpath = &devpath.join("driver");
+    let need_unbind = match fs::read_link(drvpath) {
+        Ok(d) if d.file_name() == Some(drv) => return Ok(()), // Nothing to do
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => false, // No current driver
+        Err(e) => return Err(anyhow!("Error checking driver on {}: {}", dev, e)),
+        Ok(_) => true, // Current driver needs unbinding
+    };
+    if need_unbind {
+        let unbindpath = &drvpath.join("unbind");
+        fs::write(unbindpath, dev.to_string())?;
+    }
+    let probepath = syspci.join("drivers_probe");
+    fs::write(probepath, dev.to_string())?;
+    Ok(())
+}
+
+// Represents an IOMMU group
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct IommuGroup(u32);
+
+impl fmt::Display for IommuGroup {
+    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+        write!(f, "{}", self.0)
+    }
+}
+
+// Determine the IOMMU group of a PCI device
+#[instrument]
+fn pci_iommu_group<T>(syspci: T, dev: pci::Address) -> Result<Option<IommuGroup>>
+where
+    T: AsRef<OsStr> + std::fmt::Debug,
+{
+    let syspci = Path::new(&syspci);
+    let grouppath = syspci
+        .join("devices")
+        .join(dev.to_string())
+        .join("iommu_group");
+
+    match fs::read_link(&grouppath) {
+        // Device has no group
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
+        Err(e) => Err(anyhow!("Error reading link {:?}: {}", &grouppath, e)),
+        Ok(group) => {
+            if let Some(group) = group.file_name() {
+                if let Some(group) = group.to_str() {
+                    if let Ok(group) = group.parse::<u32>() {
+                        return Ok(Some(IommuGroup(group)));
+                    }
+                }
+            }
+            Err(anyhow!(
+                "Unexpected IOMMU group link {:?} => {:?}",
+                grouppath,
+                group
+            ))
+        }
+    }
+}
+
 // pcipath_to_sysfs fetches the sysfs path for a PCI path, relative to
 // the sysfs path for the PCI host bridge, based on the PCI path
 // provided.
@@ -67,7 +154,7 @@ pub fn pcipath_to_sysfs(root_bus_sysfs: &str, pcipath: &pci::Path) -> Result<Str
     let mut relpath = String::new();
 
     for i in 0..pcipath.len() {
-        let bdf = format!("{}:{}.0", bus, pcipath[i]);
+        let bdf = format!("{}:{}", bus, pcipath[i]);
 
         relpath = format!("{}/{}", relpath, bdf);
 
@@ -162,8 +249,6 @@ pub async fn get_virtio_blk_pci_device_name(
     let sysfs_rel_path = pcipath_to_sysfs(&root_bus_sysfs, pcipath)?;
     let matcher = VirtioBlkPciMatcher::new(&sysfs_rel_path);
 
-    rescan_pci_bus()?;
-
     let uev = wait_for_uevent(sandbox, matcher).await?;
     Ok(format!("{}/{}", SYSTEM_DEV_PATH, &uev.devname))
 }
@@ -255,6 +340,72 @@ pub async fn wait_for_pmem_device(sandbox: &Arc<Mutex<Sandbox>>, devpath: &str)
     Ok(())
 }
 
+#[derive(Debug)]
+struct PciMatcher {
+    devpath: String,
+}
+
+impl PciMatcher {
+    fn new(relpath: &str) -> Result<PciMatcher> {
+        let root_bus = create_pci_root_bus_path();
+        Ok(PciMatcher {
+            devpath: format!("{}{}", root_bus, relpath),
+        })
+    }
+}
+
+impl UeventMatcher for PciMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.devpath == self.devpath
+    }
+}
+
+pub async fn wait_for_pci_device(
+    sandbox: &Arc<Mutex<Sandbox>>,
+    pcipath: &pci::Path,
+) -> Result<pci::Address> {
+    let root_bus_sysfs = format!("{}{}", SYSFS_DIR, create_pci_root_bus_path());
+    let sysfs_rel_path = pcipath_to_sysfs(&root_bus_sysfs, pcipath)?;
+    let matcher = PciMatcher::new(&sysfs_rel_path)?;
+
+    let uev = wait_for_uevent(sandbox, matcher).await?;
+
+    let addr = uev
+        .devpath
+        .rsplit('/')
+        .next()
+        .ok_or_else(|| anyhow!("Bad device path {:?} in uevent", &uev.devpath))?;
+    let addr = pci::Address::from_str(addr)?;
+    Ok(addr)
+}
+
+#[derive(Debug)]
+struct VfioMatcher {
+    syspath: String,
+}
+
+impl VfioMatcher {
+    fn new(grp: IommuGroup) -> VfioMatcher {
+        VfioMatcher {
+            syspath: format!("/devices/virtual/vfio/{}", grp),
+        }
+    }
+}
+
+impl UeventMatcher for VfioMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.devpath == self.syspath
+    }
+}
+
+#[instrument]
+async fn get_vfio_device_name(sandbox: &Arc<Mutex<Sandbox>>, grp: IommuGroup) -> Result<String> {
+    let matcher = VfioMatcher::new(grp);
+
+    let uev = wait_for_uevent(sandbox, matcher).await?;
+    Ok(format!("{}/{}", SYSTEM_DEV_PATH, &uev.devname))
+}
+
 /// Scan SCSI bus for the given SCSI address(SCSI-Id and LUN)
 #[instrument]
 fn scan_scsi_bus(scsi_addr: &str) -> Result<()> {
@@ -285,24 +436,27 @@ fn scan_scsi_bus(scsi_addr: &str) -> Result<()> {
     Ok(())
 }
 
-// update_spec_device_list takes a device description provided by the caller,
-// trying to find it on the guest. Once this device has been identified, the
-// "real" information that can be read from inside the VM is used to update
-// the same device in the list of devices provided through the OCI spec.
-// This is needed to update information about minor/major numbers that cannot
-// be predicted from the caller.
+// update_spec_device updates the device list in the OCI spec to make
+// it include details appropriate for the VM, instead of the host.  It
+// is given the host path to the device (to locate the device in the
+// original OCI spec) and the VM path which it uses to determine the
+// VM major/minor numbers, and the final path with which to present
+// the device in the (inner) container
 #[instrument]
-fn update_spec_device_list(device: &Device, spec: &mut Spec, devidx: &DevIndex) -> Result<()> {
+fn update_spec_device(
+    spec: &mut Spec,
+    devidx: &DevIndex,
+    host_path: &str,
+    vm_path: &str,
+    final_path: &str,
+) -> Result<()> {
     let major_id: c_uint;
     let minor_id: c_uint;
 
     // If no container_path is provided, we won't be able to match and
     // update the device in the OCI spec device list. This is an error.
-    if device.container_path.is_empty() {
-        return Err(anyhow!(
-            "container_path cannot empty for device {:?}",
-            device
-        ));
+    if host_path.is_empty() {
+        return Err(anyhow!("Host path cannot empty for device"));
     }
 
     let linux = spec
@@ -310,11 +464,11 @@ fn update_spec_device_list(device: &Device, spec: &mut Spec, devidx: &DevIndex)
         .as_mut()
         .ok_or_else(|| anyhow!("Spec didn't container linux field"))?;
 
-    if !Path::new(&device.vm_path).exists() {
-        return Err(anyhow!("vm_path:{} doesn't exist", device.vm_path));
+    if !Path::new(vm_path).exists() {
+        return Err(anyhow!("vm_path:{} doesn't exist", vm_path));
     }
 
-    let meta = fs::metadata(&device.vm_path)?;
+    let meta = fs::metadata(vm_path)?;
     let dev_id = meta.rdev();
     unsafe {
         major_id = major(dev_id);
@@ -323,24 +477,27 @@ fn update_spec_device_list(device: &Device, spec: &mut Spec, devidx: &DevIndex)
 
     info!(
         sl!(),
-        "got the device: dev_path: {}, major: {}, minor: {}\n", &device.vm_path, major_id, minor_id
+        "update_spec_device(): vm_path={}, major: {}, minor: {}\n", vm_path, major_id, minor_id
     );
 
-    if let Some(idxdata) = devidx.0.get(device.container_path.as_str()) {
+    if let Some(idxdata) = devidx.0.get(host_path) {
         let dev = &mut linux.devices[idxdata.idx];
         let host_major = dev.major;
         let host_minor = dev.minor;
 
         dev.major = major_id as i64;
         dev.minor = minor_id as i64;
+        dev.path = final_path.to_string();
 
         info!(
             sl!(),
-            "change the device from major: {} minor: {} to vm device major: {} minor: {}",
+            "change the device from path: {} major: {} minor: {} to vm device path: {} major: {} minor: {}",
+            host_path,
             host_major,
             host_minor,
-            major_id,
-            minor_id
+            dev.path,
+            dev.major,
+            dev.minor,
         );
 
         // Resources must be updated since they are used to identify
@@ -361,7 +518,7 @@ fn update_spec_device_list(device: &Device, spec: &mut Spec, devidx: &DevIndex)
     } else {
         Err(anyhow!(
             "Should have found a matching device {} in the spec",
-            device.vm_path
+            vm_path
         ))
     }
 }
@@ -379,7 +536,13 @@ async fn virtiommio_blk_device_handler(
         return Err(anyhow!("Invalid path for virtio mmio blk device"));
     }
 
-    update_spec_device_list(device, spec, devidx)
+    update_spec_device(
+        spec,
+        devidx,
+        &device.container_path,
+        &device.vm_path,
+        &device.container_path,
+    )
 }
 
 // device.Id should be a PCI path string
@@ -395,7 +558,13 @@ async fn virtio_blk_device_handler(
 
     dev.vm_path = get_virtio_blk_pci_device_name(sandbox, &pcipath).await?;
 
-    update_spec_device_list(&dev, spec, devidx)
+    update_spec_device(
+        spec,
+        devidx,
+        &dev.container_path,
+        &dev.vm_path,
+        &dev.container_path,
+    )
 }
 
 // device.id should be a CCW path string
@@ -410,7 +579,13 @@ async fn virtio_blk_ccw_device_handler(
     let mut dev = device.clone();
     let ccw_device = ccw::Device::from_str(&device.id)?;
     dev.vm_path = get_virtio_blk_ccw_device_name(sandbox, &ccw_device).await?;
-    update_spec_device_list(&dev, spec, devidx)
+    update_spec_device(
+        spec,
+        devidx,
+        &dev.container_path,
+        &dev.vm_path,
+        &dev.container_path,
+    )
 }
 
 #[cfg(not(target_arch = "s390x"))]
@@ -434,7 +609,13 @@ async fn virtio_scsi_device_handler(
 ) -> Result<()> {
     let mut dev = device.clone();
     dev.vm_path = get_scsi_device_name(sandbox, &device.id).await?;
-    update_spec_device_list(&dev, spec, devidx)
+    update_spec_device(
+        spec,
+        devidx,
+        &dev.container_path,
+        &dev.vm_path,
+        &dev.container_path,
+    )
 }
 
 #[instrument]
@@ -448,7 +629,79 @@ async fn virtio_nvdimm_device_handler(
         return Err(anyhow!("Invalid path for nvdimm device"));
     }
 
-    update_spec_device_list(device, spec, devidx)
+    update_spec_device(
+        spec,
+        devidx,
+        &device.container_path,
+        &device.vm_path,
+        &device.container_path,
+    )
+}
+
+fn split_vfio_option(opt: &str) -> Option<(&str, &str)> {
+    let mut tokens = opt.split('=');
+    let hostbdf = tokens.next()?;
+    let path = tokens.next()?;
+    if tokens.next().is_some() {
+        None
+    } else {
+        Some((hostbdf, path))
+    }
+}
+
+// device.options should have one entry for each PCI device in the VFIO group
+// Each option should have the form "DDDD:BB:DD.F=<pcipath>"
+//     DDDD:BB:DD.F is the device's PCI address in the host
+//     <pcipath> is a PCI path to the device in the guest (see pci.rs)
+async fn vfio_device_handler(
+    device: &Device,
+    spec: &mut Spec,
+    sandbox: &Arc<Mutex<Sandbox>>,
+    devidx: &DevIndex,
+) -> Result<()> {
+    let vfio_in_guest = device.field_type != DRIVER_VFIO_GK_TYPE;
+    let mut group = None;
+
+    for opt in device.options.iter() {
+        let (_, pcipath) =
+            split_vfio_option(opt).ok_or_else(|| anyhow!("Malformed VFIO option {:?}", opt))?;
+        let pcipath = pci::Path::from_str(pcipath)?;
+
+        let guestdev = wait_for_pci_device(sandbox, &pcipath).await?;
+        if vfio_in_guest {
+            pci_driver_override(SYSFS_BUS_PCI_PATH, guestdev, "vfio-pci")?;
+
+            let devgroup = pci_iommu_group(SYSFS_BUS_PCI_PATH, guestdev)?;
+            if devgroup.is_none() {
+                // Devices must have an IOMMU group to be usable via VFIO
+                return Err(anyhow!("{} has no IOMMU group", guestdev));
+            }
+
+            if group.is_some() && group != devgroup {
+                // If PCI devices associated with the same VFIO device
+                // (and therefore group) in the host don't end up in
+                // the same group in the guest, something has gone
+                // horribly wrong
+                return Err(anyhow!(
+                    "{} is not in guest IOMMU group {}",
+                    guestdev,
+                    group.unwrap()
+                ));
+            }
+
+            group = devgroup;
+        }
+    }
+
+    if vfio_in_guest {
+        // If there are any devices at all, logic above ensures that group is not None
+        let group = group.unwrap();
+        let vmpath = get_vfio_device_name(sandbox, group).await?;
+
+        update_spec_device(spec, devidx, &device.container_path, &vmpath, &vmpath)?;
+    }
+
+    Ok(())
 }
 
 impl DevIndex {
@@ -520,6 +773,9 @@ async fn add_device(
         DRIVER_MMIO_BLK_TYPE => virtiommio_blk_device_handler(device, spec, sandbox, devidx).await,
         DRIVER_NVDIMM_TYPE => virtio_nvdimm_device_handler(device, spec, sandbox, devidx).await,
         DRIVER_SCSI_TYPE => virtio_scsi_device_handler(device, spec, sandbox, devidx).await,
+        DRIVER_VFIO_GK_TYPE | DRIVER_VFIO_TYPE => {
+            vfio_device_handler(device, spec, sandbox, devidx).await
+        }
         _ => Err(anyhow!("Unknown device type {}", device.field_type)),
     }
 }
@@ -584,28 +840,28 @@ mod tests {
     }
 
     #[test]
-    fn test_update_spec_device_list() {
+    fn test_update_spec_device() {
         let (major, minor) = (7, 2);
-        let mut device = Device::default();
         let mut spec = Spec::default();
 
         // container_path empty
+        let container_path = "";
+        let vm_path = "";
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_err());
 
-        device.container_path = "/dev/null".to_string();
-
         // linux is empty
+        let container_path = "/dev/null";
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_err());
 
         spec.linux = Some(Linux::default());
 
         // linux.devices is empty
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_err());
 
         spec.linux.as_mut().unwrap().devices = vec![oci::LinuxDevice {
@@ -617,26 +873,32 @@ mod tests {
 
         // vm_path empty
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_err());
 
-        device.vm_path = "/dev/null".to_string();
+        let vm_path = "/dev/null";
 
         // guest and host path are not the same
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
-        assert!(res.is_err(), "device={:?} spec={:?}", device, spec);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
+        assert!(
+            res.is_err(),
+            "container_path={:?} vm_path={:?} spec={:?}",
+            container_path,
+            vm_path,
+            spec
+        );
 
-        spec.linux.as_mut().unwrap().devices[0].path = device.container_path.clone();
+        spec.linux.as_mut().unwrap().devices[0].path = container_path.to_string();
 
         // spec.linux.resources is empty
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_ok());
 
         // update both devices and cgroup lists
         spec.linux.as_mut().unwrap().devices = vec![oci::LinuxDevice {
-            path: device.container_path.clone(),
+            path: container_path.to_string(),
             major,
             minor,
             ..oci::LinuxDevice::default()
@@ -652,12 +914,12 @@ mod tests {
         });
 
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_ok());
     }
 
     #[test]
-    fn test_update_spec_device_list_guest_host_conflict() {
+    fn test_update_spec_device_guest_host_conflict() {
         let null_rdev = fs::metadata("/dev/null").unwrap().rdev();
         let zero_rdev = fs::metadata("/dev/zero").unwrap().rdev();
         let full_rdev = fs::metadata("/dev/full").unwrap().rdev();
@@ -708,20 +970,14 @@ mod tests {
         };
         let devidx = DevIndex::new(&spec);
 
-        let dev_a = Device {
-            container_path: "/dev/a".to_string(),
-            vm_path: "/dev/zero".to_string(),
-            ..Device::default()
-        };
+        let container_path_a = "/dev/a";
+        let vm_path_a = "/dev/zero";
 
         let guest_major_a = stat::major(zero_rdev) as i64;
         let guest_minor_a = stat::minor(zero_rdev) as i64;
 
-        let dev_b = Device {
-            container_path: "/dev/b".to_string(),
-            vm_path: "/dev/full".to_string(),
-            ..Device::default()
-        };
+        let container_path_b = "/dev/b";
+        let vm_path_b = "/dev/full";
 
         let guest_major_b = stat::major(full_rdev) as i64;
         let guest_minor_b = stat::minor(full_rdev) as i64;
@@ -738,7 +994,13 @@ mod tests {
         assert_eq!(Some(host_major_b), specresources.devices[1].major);
         assert_eq!(Some(host_minor_b), specresources.devices[1].minor);
 
-        let res = update_spec_device_list(&dev_a, &mut spec, &devidx);
+        let res = update_spec_device(
+            &mut spec,
+            &devidx,
+            container_path_a,
+            vm_path_a,
+            container_path_a,
+        );
         assert!(res.is_ok());
 
         let specdevices = &spec.linux.as_ref().unwrap().devices;
@@ -753,7 +1015,13 @@ mod tests {
         assert_eq!(Some(host_major_b), specresources.devices[1].major);
         assert_eq!(Some(host_minor_b), specresources.devices[1].minor);
 
-        let res = update_spec_device_list(&dev_b, &mut spec, &devidx);
+        let res = update_spec_device(
+            &mut spec,
+            &devidx,
+            container_path_b,
+            vm_path_b,
+            container_path_b,
+        );
         assert!(res.is_ok());
 
         let specdevices = &spec.linux.as_ref().unwrap().devices;
@@ -770,7 +1038,7 @@ mod tests {
     }
 
     #[test]
-    fn test_update_spec_device_list_char_block_conflict() {
+    fn test_update_spec_device_char_block_conflict() {
         let null_rdev = fs::metadata("/dev/null").unwrap().rdev();
 
         let guest_major = stat::major(null_rdev) as i64;
@@ -819,11 +1087,8 @@ mod tests {
         };
         let devidx = DevIndex::new(&spec);
 
-        let dev = Device {
-            container_path: "/dev/char".to_string(),
-            vm_path: "/dev/null".to_string(),
-            ..Device::default()
-        };
+        let container_path = "/dev/char";
+        let vm_path = "/dev/null";
 
         let specresources = spec.linux.as_ref().unwrap().resources.as_ref().unwrap();
         assert_eq!(Some(host_major), specresources.devices[0].major);
@@ -831,7 +1096,7 @@ mod tests {
         assert_eq!(Some(host_major), specresources.devices[1].major);
         assert_eq!(Some(host_minor), specresources.devices[1].minor);
 
-        let res = update_spec_device_list(&dev, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_ok());
 
         // Only the char device, not the block device should be updated
@@ -842,6 +1107,43 @@ mod tests {
         assert_eq!(Some(host_minor), specresources.devices[1].minor);
     }
 
+    #[test]
+    fn test_update_spec_device_final_path() {
+        let null_rdev = fs::metadata("/dev/null").unwrap().rdev();
+        let guest_major = stat::major(null_rdev) as i64;
+        let guest_minor = stat::minor(null_rdev) as i64;
+
+        let host_path = "/dev/host";
+        let host_major: i64 = 99;
+        let host_minor: i64 = 99;
+
+        let mut spec = Spec {
+            linux: Some(Linux {
+                devices: vec![oci::LinuxDevice {
+                    path: host_path.to_string(),
+                    r#type: "c".to_string(),
+                    major: host_major,
+                    minor: host_minor,
+                    ..oci::LinuxDevice::default()
+                }],
+                ..Linux::default()
+            }),
+            ..Spec::default()
+        };
+        let devidx = DevIndex::new(&spec);
+
+        let vm_path = "/dev/null";
+        let final_path = "/dev/final";
+
+        let res = update_spec_device(&mut spec, &devidx, host_path, vm_path, final_path);
+        assert!(res.is_ok());
+
+        let specdevices = &spec.linux.as_ref().unwrap().devices;
+        assert_eq!(guest_major, specdevices[0].major);
+        assert_eq!(guest_minor, specdevices[0].minor);
+        assert_eq!(final_path, specdevices[0].path);
+    }
+
     #[test]
     fn test_pcipath_to_sysfs() {
         let testdir = tempdir().expect("failed to create tmpdir");
@@ -1068,4 +1370,112 @@ mod tests {
         assert!(!matcher_b.is_match(&uev_a));
         assert!(!matcher_a.is_match(&uev_b));
     }
+
+    #[tokio::test]
+    async fn test_vfio_matcher() {
+        let grpa = IommuGroup(1);
+        let grpb = IommuGroup(22);
+
+        let mut uev_a = crate::uevent::Uevent::default();
+        uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
+        uev_a.devname = format!("vfio/{}", grpa);
+        uev_a.devpath = format!("/devices/virtual/vfio/{}", grpa);
+        let matcher_a = VfioMatcher::new(grpa);
+
+        let mut uev_b = uev_a.clone();
+        uev_b.devpath = format!("/devices/virtual/vfio/{}", grpb);
+        let matcher_b = VfioMatcher::new(grpb);
+
+        assert!(matcher_a.is_match(&uev_a));
+        assert!(matcher_b.is_match(&uev_b));
+        assert!(!matcher_b.is_match(&uev_a));
+        assert!(!matcher_a.is_match(&uev_b));
+    }
+
+    #[test]
+    fn test_split_vfio_option() {
+        assert_eq!(
+            split_vfio_option("0000:01:00.0=02/01"),
+            Some(("0000:01:00.0", "02/01"))
+        );
+        assert_eq!(split_vfio_option("0000:01:00.0=02/01=rubbish"), None);
+        assert_eq!(split_vfio_option("0000:01:00.0"), None);
+    }
+
+    #[test]
+    fn test_pci_driver_override() {
+        let testdir = tempdir().expect("failed to create tmpdir");
+        let syspci = testdir.path(); // Path to mock /sys/bus/pci
+
+        let dev0 = pci::Address::new(0, 0, pci::SlotFn::new(0, 0).unwrap());
+        let dev0path = syspci.join("devices").join(dev0.to_string());
+        let dev0drv = dev0path.join("driver");
+        let dev0override = dev0path.join("driver_override");
+
+        let drvapath = syspci.join("drivers").join("drv_a");
+        let drvaunbind = drvapath.join("unbind");
+
+        let probepath = syspci.join("drivers_probe");
+
+        // Start mocking dev0 as being unbound
+        fs::create_dir_all(&dev0path).unwrap();
+
+        pci_driver_override(syspci, dev0, "drv_a").unwrap();
+        assert_eq!(fs::read_to_string(&dev0override).unwrap(), "drv_a");
+        assert_eq!(fs::read_to_string(&probepath).unwrap(), dev0.to_string());
+
+        // Now mock dev0 already being attached to drv_a
+        fs::create_dir_all(&drvapath).unwrap();
+        std::os::unix::fs::symlink(&drvapath, dev0drv).unwrap();
+        std::fs::remove_file(&probepath).unwrap();
+
+        pci_driver_override(syspci, dev0, "drv_a").unwrap(); // no-op
+        assert_eq!(fs::read_to_string(&dev0override).unwrap(), "drv_a");
+        assert!(!probepath.exists());
+
+        // Now try binding to a different driver
+        pci_driver_override(syspci, dev0, "drv_b").unwrap();
+        assert_eq!(fs::read_to_string(&dev0override).unwrap(), "drv_b");
+        assert_eq!(fs::read_to_string(&probepath).unwrap(), dev0.to_string());
+        assert_eq!(fs::read_to_string(&drvaunbind).unwrap(), dev0.to_string());
+    }
+
+    #[test]
+    fn test_pci_iommu_group() {
+        let testdir = tempdir().expect("failed to create tmpdir"); // mock /sys
+        let syspci = testdir.path().join("bus").join("pci");
+
+        // Mock dev0, which has no group
+        let dev0 = pci::Address::new(0, 0, pci::SlotFn::new(0, 0).unwrap());
+        let dev0path = syspci.join("devices").join(dev0.to_string());
+
+        fs::create_dir_all(&dev0path).unwrap();
+
+        // Test dev0
+        assert!(pci_iommu_group(&syspci, dev0).unwrap().is_none());
+
+        // Mock dev1, which is in group 12
+        let dev1 = pci::Address::new(0, 1, pci::SlotFn::new(0, 0).unwrap());
+        let dev1path = syspci.join("devices").join(dev1.to_string());
+        let dev1group = dev1path.join("iommu_group");
+
+        fs::create_dir_all(&dev1path).unwrap();
+        std::os::unix::fs::symlink("../../../kernel/iommu_groups/12", &dev1group).unwrap();
+
+        // Test dev1
+        assert_eq!(
+            pci_iommu_group(&syspci, dev1).unwrap(),
+            Some(IommuGroup(12))
+        );
+
+        // Mock dev2, which has a bogus group (dir instead of symlink)
+        let dev2 = pci::Address::new(0, 2, pci::SlotFn::new(0, 0).unwrap());
+        let dev2path = syspci.join("devices").join(dev2.to_string());
+        let dev2group = dev2path.join("iommu_group");
+
+        fs::create_dir_all(&dev2group).unwrap();
+
+        // Test dev2
+        assert!(pci_iommu_group(&syspci, dev2).is_err());
+    }
 }

src/agent/src/linux_abi.rs

@@ -9,7 +9,6 @@
 use std::fs;
 
 pub const SYSFS_DIR: &str = "/sys";
-pub const SYSFS_PCI_BUS_RESCAN_FILE: &str = "/sys/bus/pci/rescan";
 #[cfg(any(
     target_arch = "powerpc64",
     target_arch = "s390x",
@@ -84,6 +83,8 @@ pub const SYSFS_MEMORY_ONLINE_PATH: &str = "/sys/devices/system/memory";
 
 pub const SYSFS_SCSI_HOST_PATH: &str = "/sys/class/scsi_host";
 
+pub const SYSFS_BUS_PCI_PATH: &str = "/sys/bus/pci";
+
 pub const SYSFS_CGROUPPATH: &str = "/sys/fs/cgroup";
 pub const SYSFS_ONLINE_FILE: &str = "online";
 
@@ -95,6 +96,7 @@ pub const SYSTEM_DEV_PATH: &str = "/dev";
 // Linux UEvent related consts.
 pub const U_EVENT_ACTION: &str = "ACTION";
 pub const U_EVENT_ACTION_ADD: &str = "add";
+pub const U_EVENT_ACTION_REMOVE: &str = "remove";
 pub const U_EVENT_DEV_PATH: &str = "DEVPATH";
 pub const U_EVENT_SUB_SYSTEM: &str = "SUBSYSTEM";
 pub const U_EVENT_SEQ_NUM: &str = "SEQNUM";

src/agent/src/main.rs

@@ -77,11 +77,11 @@ mod rpc;
 mod tracer;
 
 const NAME: &str = "kata-agent";
-const KERNEL_CMDLINE_FILE: &str = "/proc/cmdline";
 
 lazy_static! {
-    static ref AGENT_CONFIG: Arc<RwLock<AgentConfig>> =
-        Arc::new(RwLock::new(config::AgentConfig::new()));
+    static ref AGENT_CONFIG: Arc<RwLock<AgentConfig>> = Arc::new(RwLock::new(
+        AgentConfig::from_cmdline("/proc/cmdline").unwrap()
+    ));
 }
 
 #[instrument]
@@ -134,15 +134,11 @@ async fn real_main() -> std::result::Result<(), Box<dyn std::error::Error>> {
 
     console::initialize();
 
-    lazy_static::initialize(&AGENT_CONFIG);
-
     // support vsock log
     let (rfd, wfd) = unistd::pipe2(OFlag::O_CLOEXEC)?;
 
     let (shutdown_tx, shutdown_rx) = channel(true);
 
-    let agent_config = AGENT_CONFIG.clone();
-
     let init_mode = unistd::getpid() == Pid::from_raw(1);
     if init_mode {
         // dup a new file descriptor for this temporary logger writer,
@@ -163,20 +159,15 @@ async fn real_main() -> std::result::Result<(), Box<dyn std::error::Error>> {
             e
         })?;
 
-        let mut config = agent_config.write().await;
-        config.parse_cmdline(KERNEL_CMDLINE_FILE)?;
+        lazy_static::initialize(&AGENT_CONFIG);
 
-        init_agent_as_init(&logger, config.unified_cgroup_hierarchy)?;
+        init_agent_as_init(&logger, AGENT_CONFIG.read().await.unified_cgroup_hierarchy)?;
         drop(logger_async_guard);
     } else {
-        // once parsed cmdline and set the config, release the write lock
-        // as soon as possible in case other thread would get read lock on
-        // it.
-        let mut config = agent_config.write().await;
-        config.parse_cmdline(KERNEL_CMDLINE_FILE)?;
+        lazy_static::initialize(&AGENT_CONFIG);
     }
-    let config = agent_config.read().await;
 
+    let config = AGENT_CONFIG.read().await;
     let log_vport = config.log_vport as u32;
 
     let log_handle = tokio::spawn(create_logger_task(rfd, log_vport, shutdown_rx.clone()));
@@ -205,16 +196,16 @@ async fn real_main() -> std::result::Result<(), Box<dyn std::error::Error>> {
         ttrpc_log_guard = Ok(slog_stdlog::init().map_err(|e| e)?);
     }
 
-    if config.tracing != tracer::TraceType::Disabled {
-        let _ = tracer::setup_tracing(NAME, &logger, &config)?;
+    if config.tracing {
+        tracer::setup_tracing(NAME, &logger)?;
     }
 
-    let root = span!(tracing::Level::TRACE, "root-span", work_units = 2);
+    let root_span = span!(tracing::Level::TRACE, "root-span");
 
     // XXX: Start the root trace transaction.
     //
     // XXX: Note that *ALL* spans needs to start after this point!!
-    let _enter = root.enter();
+    let span_guard = root_span.enter();
 
     // Start the sandbox and wait for its ttRPC server to end
     start_sandbox(&logger, &config, init_mode, &mut tasks, shutdown_rx.clone()).await?;
@@ -238,19 +229,29 @@ async fn real_main() -> std::result::Result<(), Box<dyn std::error::Error>> {
     // Wait for all threads to finish
     let results = join_all(tasks).await;
 
-    for result in results {
-        if let Err(e) = result {
-            return Err(anyhow!(e).into());
-        }
-    }
+    // force flushing spans
+    drop(span_guard);
+    drop(root_span);
 
-    if config.tracing != tracer::TraceType::Disabled {
+    if config.tracing {
         tracer::end_tracing();
     }
 
     eprintln!("{} shutdown complete", NAME);
 
-    Ok(())
+    let mut wait_errors: Vec<tokio::task::JoinError> = vec![];
+    for result in results {
+        if let Err(e) = result {
+            eprintln!("wait task error: {:#?}", e);
+            wait_errors.push(e);
+        }
+    }
+
+    if wait_errors.is_empty() {
+        Ok(())
+    } else {
+        Err(anyhow!("wait all tasks failed: {:#?}", wait_errors).into())
+    }
 }
 
 fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {

src/agent/src/mount.rs

@@ -22,6 +22,9 @@ use regex::Regex;
 
 use crate::device::{
     get_scsi_device_name, get_virtio_blk_pci_device_name, online_device, wait_for_pmem_device,
+    DRIVER_9P_TYPE, DRIVER_BLK_CCW_TYPE, DRIVER_BLK_TYPE, DRIVER_EPHEMERAL_TYPE, DRIVER_LOCAL_TYPE,
+    DRIVER_MMIO_BLK_TYPE, DRIVER_NVDIMM_TYPE, DRIVER_SCSI_TYPE, DRIVER_VIRTIOFS_TYPE,
+    DRIVER_WATCHABLE_BIND_TYPE,
 };
 use crate::linux_abi::*;
 use crate::pci;
@@ -33,17 +36,6 @@ use anyhow::{anyhow, Context, Result};
 use slog::Logger;
 use tracing::instrument;
 
-pub const DRIVER_9P_TYPE: &str = "9p";
-pub const DRIVER_VIRTIOFS_TYPE: &str = "virtio-fs";
-pub const DRIVER_BLK_TYPE: &str = "blk";
-pub const DRIVER_BLK_CCW_TYPE: &str = "blk-ccw";
-pub const DRIVER_MMIO_BLK_TYPE: &str = "mmioblk";
-pub const DRIVER_SCSI_TYPE: &str = "scsi";
-pub const DRIVER_NVDIMM_TYPE: &str = "nvdimm";
-pub const DRIVER_EPHEMERAL_TYPE: &str = "ephemeral";
-pub const DRIVER_LOCAL_TYPE: &str = "local";
-pub const DRIVER_WATCHABLE_BIND_TYPE: &str = "watchable-bind";
-
 pub const TYPE_ROOTFS: &str = "rootfs";
 
 pub const MOUNT_GUEST_TAG: &str = "kataShared";

src/agent/src/netlink.rs

@@ -6,6 +6,7 @@
 use anyhow::{anyhow, Context, Result};
 use futures::{future, StreamExt, TryStreamExt};
 use ipnetwork::{IpNetwork, Ipv4Network, Ipv6Network};
+use nix::errno::Errno;
 use protobuf::RepeatedField;
 use protocols::types::{ARPNeighbor, IPAddress, IPFamily, Interface, Route};
 use rtnetlink::{new_connection, packet, IpVersion};
@@ -312,7 +313,6 @@ impl Handle {
 
         for route in list {
             let link = self.find_link(LinkFilter::Name(&route.device)).await?;
-            let is_v6 = is_ipv6(route.get_gateway()) || is_ipv6(route.get_dest());
 
             const MAIN_TABLE: u8 = packet::constants::RT_TABLE_MAIN;
             const UNICAST: u8 = packet::constants::RTN_UNICAST;
@@ -334,7 +334,7 @@ impl Handle {
 
             // `rtnetlink` offers a separate request builders for different IP versions (IP v4 and v6).
             // This if branch is a bit clumsy because it does almost the same.
-            if is_v6 {
+            if route.get_family() == IPFamily::v6 {
                 let dest_addr = if !route.dest.is_empty() {
                     Ipv6Network::from_str(&route.dest)?
                 } else {
@@ -364,14 +364,17 @@ impl Handle {
                     request = request.gateway(ip);
                 }
 
-                request.execute().await.with_context(|| {
-                    format!(
-                        "Failed to add IP v6 route (src: {}, dst: {}, gtw: {})",
-                        route.get_source(),
-                        route.get_dest(),
-                        route.get_gateway()
-                    )
-                })?;
+                if let Err(rtnetlink::Error::NetlinkError(message)) = request.execute().await {
+                    if Errno::from_i32(message.code.abs()) != Errno::EEXIST {
+                        return Err(anyhow!(
+                            "Failed to add IP v6 route (src: {}, dst: {}, gtw: {},Err: {})",
+                            route.get_source(),
+                            route.get_dest(),
+                            route.get_gateway(),
+                            message
+                        ));
+                    }
+                }
             } else {
                 let dest_addr = if !route.dest.is_empty() {
                     Ipv4Network::from_str(&route.dest)?
@@ -402,7 +405,17 @@ impl Handle {
                     request = request.gateway(ip);
                 }
 
-                request.execute().await?;
+                if let Err(rtnetlink::Error::NetlinkError(message)) = request.execute().await {
+                    if Errno::from_i32(message.code.abs()) != Errno::EEXIST {
+                        return Err(anyhow!(
+                            "Failed to add IP v4 route (src: {}, dst: {}, gtw: {},Err: {})",
+                            route.get_source(),
+                            route.get_dest(),
+                            route.get_gateway(),
+                            message
+                        ));
+                    }
+                }
             }
         }
 
@@ -594,10 +607,6 @@ fn format_address(data: &[u8]) -> Result<String> {
     }
 }
 
-fn is_ipv6(str: &str) -> bool {
-    Ipv6Addr::from_str(str).is_ok()
-}
-
 fn parse_mac_address(addr: &str) -> Result<[u8; 6]> {
     let mut split = addr.splitn(6, ':');
 
@@ -932,16 +941,6 @@ mod tests {
         assert_eq!(bytes, [0xAB, 0x0C, 0xDE, 0x12, 0x34, 0x56]);
     }
 
-    #[test]
-    fn check_ipv6() {
-        assert!(is_ipv6("::1"));
-        assert!(is_ipv6("2001:0:3238:DFE1:63::FEFB"));
-
-        assert!(!is_ipv6(""));
-        assert!(!is_ipv6("127.0.0.1"));
-        assert!(!is_ipv6("10.10.10.10"));
-    }
-
     fn clean_env_for_test_add_one_arp_neighbor(dummy_name: &str, ip: &str) {
         // ip link delete dummy
         Command::new("ip")

src/agent/src/pci.rs

@@ -9,51 +9,143 @@ use std::str::FromStr;
 
 use anyhow::anyhow;
 
-// The PCI spec reserves 5 bits for slot number (a.k.a. device
-// number), giving slots 0..31
+// The PCI spec reserves 5 bits (0..31) for slot number (a.k.a. device
+// number)
 const SLOT_BITS: u8 = 5;
 const SLOT_MAX: u8 = (1 << SLOT_BITS) - 1;
 
-// Represents a PCI function's slot number (a.k.a. device number),
-// giving its location on a single bus
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-pub struct Slot(u8);
+// The PCI spec reserves 3 bits (0..7) for function number
+const FUNCTION_BITS: u8 = 3;
+const FUNCTION_MAX: u8 = (1 << FUNCTION_BITS) - 1;
 
-impl Slot {
-    pub fn new<T: TryInto<u8> + fmt::Display + Copy>(v: T) -> anyhow::Result<Self> {
-        if let Ok(v8) = v.try_into() {
-            if v8 <= SLOT_MAX {
-                return Ok(Slot(v8));
+// Represents a PCI function's slot (a.k.a. device) and function
+// numbers, giving its location on a single logical bus
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub struct SlotFn(u8);
+
+impl SlotFn {
+    pub fn new<T, U>(ss: T, f: U) -> anyhow::Result<Self>
+    where
+        T: TryInto<u8> + fmt::Display + Copy,
+        U: TryInto<u8> + fmt::Display + Copy,
+    {
+        let ss8 = match ss.try_into() {
+            Ok(ss8) if ss8 <= SLOT_MAX => ss8,
+            _ => {
+                return Err(anyhow!(
+                    "PCI slot {} should be in range [0..{:#x}]",
+                    ss,
+                    SLOT_MAX
+                ));
             }
-        }
-        Err(anyhow!(
-            "PCI slot {} should be in range [0..{:#x}]",
-            v,
-            SLOT_MAX
-        ))
+        };
+
+        let f8 = match f.try_into() {
+            Ok(f8) if f8 <= FUNCTION_MAX => f8,
+            _ => {
+                return Err(anyhow!(
+                    "PCI function {} should be in range [0..{:#x}]",
+                    f,
+                    FUNCTION_MAX
+                ));
+            }
+        };
+
+        Ok(SlotFn(ss8 << FUNCTION_BITS | f8))
+    }
+
+    pub fn slot(self) -> u8 {
+        self.0 >> FUNCTION_BITS
+    }
+
+    pub fn function(self) -> u8 {
+        self.0 & FUNCTION_MAX
     }
 }
 
-impl FromStr for Slot {
+impl FromStr for SlotFn {
     type Err = anyhow::Error;
 
     fn from_str(s: &str) -> anyhow::Result<Self> {
-        let v = isize::from_str_radix(s, 16)?;
-        Slot::new(v)
+        let mut tokens = s.split('.').fuse();
+        let slot = tokens.next();
+        let func = tokens.next();
+
+        if slot.is_none() || tokens.next().is_some() {
+            return Err(anyhow!(
+                "PCI slot/function {} should have the format SS.F",
+                s
+            ));
+        }
+
+        let slot = isize::from_str_radix(slot.unwrap(), 16)?;
+        let func = match func {
+            Some(func) => isize::from_str_radix(func, 16)?,
+            None => 0,
+        };
+
+        SlotFn::new(slot, func)
     }
 }
 
-impl fmt::Display for Slot {
+impl fmt::Display for SlotFn {
     fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
-        write!(f, "{:02x}", self.0)
+        write!(f, "{:02x}.{:01x}", self.slot(), self.function())
+    }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub struct Address {
+    domain: u16,
+    bus: u8,
+    slotfn: SlotFn,
+}
+
+impl Address {
+    pub fn new(domain: u16, bus: u8, slotfn: SlotFn) -> Self {
+        Address {
+            domain,
+            bus,
+            slotfn,
+        }
+    }
+}
+
+impl FromStr for Address {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> anyhow::Result<Self> {
+        let mut tokens = s.split(':').fuse();
+        let domain = tokens.next();
+        let bus = tokens.next();
+        let slotfn = tokens.next();
+
+        if domain.is_none() || bus.is_none() || slotfn.is_none() || tokens.next().is_some() {
+            return Err(anyhow!(
+                "PCI address {} should have the format DDDD:BB:SS.F",
+                s
+            ));
+        }
+
+        let domain = u16::from_str_radix(domain.unwrap(), 16)?;
+        let bus = u8::from_str_radix(bus.unwrap(), 16)?;
+        let slotfn = SlotFn::from_str(slotfn.unwrap())?;
+
+        Ok(Address::new(domain, bus, slotfn))
+    }
+}
+
+impl fmt::Display for Address {
+    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+        write!(f, "{:04x}:{:02x}:{}", self.domain, self.bus, self.slotfn)
     }
 }
 
 #[derive(Clone, Debug, PartialEq, Eq)]
-pub struct Path(Vec<Slot>);
+pub struct Path(Vec<SlotFn>);
 
 impl Path {
-    pub fn new(slots: Vec<Slot>) -> anyhow::Result<Self> {
+    pub fn new(slots: Vec<SlotFn>) -> anyhow::Result<Self> {
         if slots.is_empty() {
             return Err(anyhow!("PCI path must have at least one element"));
         }
@@ -63,7 +155,7 @@ impl Path {
 
 // Let Path be treated as a slice of Slots
 impl Deref for Path {
-    type Target = [Slot];
+    type Target = [SlotFn];
 
     fn deref(&self) -> &Self::Target {
         &self.0
@@ -85,83 +177,170 @@ impl FromStr for Path {
     type Err = anyhow::Error;
 
     fn from_str(s: &str) -> anyhow::Result<Self> {
-        let rslots: anyhow::Result<Vec<Slot>> = s.split('/').map(Slot::from_str).collect();
+        let rslots: anyhow::Result<Vec<SlotFn>> = s.split('/').map(SlotFn::from_str).collect();
         Path::new(rslots?)
     }
 }
 
 #[cfg(test)]
 mod tests {
-    use crate::pci::{Path, Slot};
+    use super::*;
     use std::str::FromStr;
 
     #[test]
-    fn test_slot() {
+    fn test_slotfn() {
         // Valid slots
-        let slot = Slot::new(0x00).unwrap();
-        assert_eq!(format!("{}", slot), "00");
+        let sf = SlotFn::new(0x00, 0x0).unwrap();
+        assert_eq!(format!("{}", sf), "00.0");
 
-        let slot = Slot::from_str("00").unwrap();
-        assert_eq!(format!("{}", slot), "00");
+        let sf = SlotFn::from_str("00.0").unwrap();
+        assert_eq!(format!("{}", sf), "00.0");
 
-        let slot = Slot::new(31).unwrap();
-        let slot2 = Slot::from_str("1f").unwrap();
-        assert_eq!(slot, slot2);
+        let sf = SlotFn::from_str("00").unwrap();
+        assert_eq!(format!("{}", sf), "00.0");
+
+        let sf = SlotFn::new(31, 7).unwrap();
+        let sf2 = SlotFn::from_str("1f.7").unwrap();
+        assert_eq!(sf, sf2);
 
         // Bad slots
-        let slot = Slot::new(-1);
-        assert!(slot.is_err());
+        let sf = SlotFn::new(-1, 0);
+        assert!(sf.is_err());
 
-        let slot = Slot::new(32);
-        assert!(slot.is_err());
+        let sf = SlotFn::new(32, 0);
+        assert!(sf.is_err());
 
-        let slot = Slot::from_str("20");
-        assert!(slot.is_err());
+        let sf = SlotFn::from_str("20.0");
+        assert!(sf.is_err());
 
-        let slot = Slot::from_str("xy");
-        assert!(slot.is_err());
+        let sf = SlotFn::from_str("20");
+        assert!(sf.is_err());
 
-        let slot = Slot::from_str("00/");
-        assert!(slot.is_err());
+        let sf = SlotFn::from_str("xy.0");
+        assert!(sf.is_err());
 
-        let slot = Slot::from_str("");
-        assert!(slot.is_err());
+        let sf = SlotFn::from_str("xy");
+        assert!(sf.is_err());
+
+        // Bad functions
+        let sf = SlotFn::new(0, -1);
+        assert!(sf.is_err());
+
+        let sf = SlotFn::new(0, 8);
+        assert!(sf.is_err());
+
+        let sf = SlotFn::from_str("00.8");
+        assert!(sf.is_err());
+
+        let sf = SlotFn::from_str("00.x");
+        assert!(sf.is_err());
+
+        // Bad formats
+        let sf = SlotFn::from_str("");
+        assert!(sf.is_err());
+
+        let sf = SlotFn::from_str("00.0.0");
+        assert!(sf.is_err());
+
+        let sf = SlotFn::from_str("00.0/");
+        assert!(sf.is_err());
+
+        let sf = SlotFn::from_str("00/");
+        assert!(sf.is_err());
+    }
+
+    #[test]
+    fn test_address() {
+        // Valid addresses
+        let sf0_0 = SlotFn::new(0, 0).unwrap();
+        let sf1f_7 = SlotFn::new(0x1f, 7).unwrap();
+
+        let addr = Address::new(0, 0, sf0_0);
+        assert_eq!(format!("{}", addr), "0000:00:00.0");
+        let addr2 = Address::from_str("0000:00:00.0").unwrap();
+        assert_eq!(addr, addr2);
+
+        let addr = Address::new(0xffff, 0xff, sf1f_7);
+        assert_eq!(format!("{}", addr), "ffff:ff:1f.7");
+        let addr2 = Address::from_str("ffff:ff:1f.7").unwrap();
+        assert_eq!(addr, addr2);
+
+        // Bad addresses
+        let addr = Address::from_str("10000:00:00.0");
+        assert!(addr.is_err());
+
+        let addr = Address::from_str("0000:100:00.0");
+        assert!(addr.is_err());
+
+        let addr = Address::from_str("0000:00:20.0");
+        assert!(addr.is_err());
+
+        let addr = Address::from_str("0000:00:00.8");
+        assert!(addr.is_err());
+
+        let addr = Address::from_str("xyz");
+        assert!(addr.is_err());
+
+        let addr = Address::from_str("xyxy:xy:xy.z");
+        assert!(addr.is_err());
+
+        let addr = Address::from_str("0000:00:00.0:00");
+        assert!(addr.is_err());
     }
 
     #[test]
     fn test_path() {
-        let slot3 = Slot::new(0x03).unwrap();
-        let slot4 = Slot::new(0x04).unwrap();
-        let slot5 = Slot::new(0x05).unwrap();
+        let sf3_0 = SlotFn::new(0x03, 0).unwrap();
+        let sf4_0 = SlotFn::new(0x04, 0).unwrap();
+        let sf5_0 = SlotFn::new(0x05, 0).unwrap();
+        let sfa_5 = SlotFn::new(0x0a, 5).unwrap();
+        let sfb_6 = SlotFn::new(0x0b, 6).unwrap();
+        let sfc_7 = SlotFn::new(0x0c, 7).unwrap();
 
         // Valid paths
-        let pcipath = Path::new(vec![slot3]).unwrap();
-        assert_eq!(format!("{}", pcipath), "03");
+        let pcipath = Path::new(vec![sf3_0]).unwrap();
+        assert_eq!(format!("{}", pcipath), "03.0");
+        let pcipath2 = Path::from_str("03.0").unwrap();
+        assert_eq!(pcipath, pcipath2);
         let pcipath2 = Path::from_str("03").unwrap();
         assert_eq!(pcipath, pcipath2);
         assert_eq!(pcipath.len(), 1);
-        assert_eq!(pcipath[0], slot3);
+        assert_eq!(pcipath[0], sf3_0);
 
-        let pcipath = Path::new(vec![slot3, slot4]).unwrap();
-        assert_eq!(format!("{}", pcipath), "03/04");
+        let pcipath = Path::new(vec![sf3_0, sf4_0]).unwrap();
+        assert_eq!(format!("{}", pcipath), "03.0/04.0");
+        let pcipath2 = Path::from_str("03.0/04.0").unwrap();
+        assert_eq!(pcipath, pcipath2);
         let pcipath2 = Path::from_str("03/04").unwrap();
         assert_eq!(pcipath, pcipath2);
         assert_eq!(pcipath.len(), 2);
-        assert_eq!(pcipath[0], slot3);
-        assert_eq!(pcipath[1], slot4);
+        assert_eq!(pcipath[0], sf3_0);
+        assert_eq!(pcipath[1], sf4_0);
 
-        let pcipath = Path::new(vec![slot3, slot4, slot5]).unwrap();
-        assert_eq!(format!("{}", pcipath), "03/04/05");
+        let pcipath = Path::new(vec![sf3_0, sf4_0, sf5_0]).unwrap();
+        assert_eq!(format!("{}", pcipath), "03.0/04.0/05.0");
+        let pcipath2 = Path::from_str("03.0/04.0/05.0").unwrap();
+        assert_eq!(pcipath, pcipath2);
         let pcipath2 = Path::from_str("03/04/05").unwrap();
         assert_eq!(pcipath, pcipath2);
         assert_eq!(pcipath.len(), 3);
-        assert_eq!(pcipath[0], slot3);
-        assert_eq!(pcipath[1], slot4);
-        assert_eq!(pcipath[2], slot5);
+        assert_eq!(pcipath[0], sf3_0);
+        assert_eq!(pcipath[1], sf4_0);
+        assert_eq!(pcipath[2], sf5_0);
+
+        let pcipath = Path::new(vec![sfa_5, sfb_6, sfc_7]).unwrap();
+        assert_eq!(format!("{}", pcipath), "0a.5/0b.6/0c.7");
+        let pcipath2 = Path::from_str("0a.5/0b.6/0c.7").unwrap();
+        assert_eq!(pcipath, pcipath2);
+        assert_eq!(pcipath.len(), 3);
+        assert_eq!(pcipath[0], sfa_5);
+        assert_eq!(pcipath[1], sfb_6);
+        assert_eq!(pcipath[2], sfc_7);
 
         // Bad paths
         assert!(Path::new(vec!()).is_err());
         assert!(Path::from_str("20").is_err());
+        assert!(Path::from_str("00.8").is_err());
         assert!(Path::from_str("//").is_err());
         assert!(Path::from_str("xyz").is_err());
     }

src/agent/src/rpc.rs

@@ -3,7 +3,6 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use crate::pci;
 use async_trait::async_trait;
 use rustjail::{pipestream::PipeStream, process::StreamType};
 use tokio::io::{AsyncReadExt, AsyncWriteExt, ReadHalf};
@@ -21,7 +20,7 @@ use ttrpc::{
 
 use anyhow::{anyhow, Context, Result};
 use oci::{LinuxNamespace, Root, Spec};
-use protobuf::{RepeatedField, SingularPtrField};
+use protobuf::{Message, RepeatedField, SingularPtrField};
 use protocols::agent::{
     AddSwapRequest, AgentDetails, CopyFileRequest, GuestDetailsResponse, Interfaces, Metrics,
     OOMEvent, ReadStreamResponse, Routes, StatsContainerResponse, WaitProcessResponse,
@@ -44,12 +43,13 @@ use nix::sys::stat;
 use nix::unistd::{self, Pid};
 use rustjail::process::ProcessOperations;
 
-use crate::device::{add_devices, pcipath_to_sysfs, rescan_pci_bus, update_device_cgroup};
+use crate::device::{add_devices, get_virtio_blk_pci_device_name, update_device_cgroup};
 use crate::linux_abi::*;
 use crate::metrics::get_metrics;
 use crate::mount::{add_storages, baremount, remove_mounts, STORAGE_HANDLER_LIST};
 use crate::namespace::{NSTYPEIPC, NSTYPEPID, NSTYPEUTS};
 use crate::network::setup_guest_dns;
+use crate::pci;
 use crate::random;
 use crate::sandbox::Sandbox;
 use crate::version::{AGENT_VERSION, API_VERSION};
@@ -86,6 +86,21 @@ macro_rules! sl {
     };
 }
 
+macro_rules! is_allowed {
+    ($req:ident) => {
+        if !AGENT_CONFIG
+            .read()
+            .await
+            .is_allowed_endpoint($req.descriptor().name())
+        {
+            return Err(ttrpc_error(
+                ttrpc::Code::UNIMPLEMENTED,
+                format!("{} is blocked", $req.descriptor().name()),
+            ));
+        }
+    };
+}
+
 #[derive(Clone, Debug)]
 pub struct AgentService {
     sandbox: Arc<Mutex<Sandbox>>,
@@ -134,10 +149,6 @@ impl AgentService {
 
         info!(sl!(), "receive createcontainer, spec: {:?}", &oci);
 
-        // re-scan PCI bus
-        // looking for hidden devices
-        rescan_pci_bus().context("Could not rescan PCI bus")?;
-
         // Some devices need some extra processing (the ones invoked with
         // --device for instance), and that's what this call is doing. It
         // updates the devices listed in the OCI spec, so that they actually
@@ -422,7 +433,7 @@ impl AgentService {
             .get_container(&cid)
             .ok_or_else(|| anyhow!("Invalid container id"))?;
 
-        let mut p = match ctr.processes.get_mut(&pid) {
+        let p = match ctr.processes.get_mut(&pid) {
             Some(p) => p,
             None => {
                 // Lost race, pick up exit code from channel
@@ -433,7 +444,7 @@ impl AgentService {
 
         // need to close all fd
         // ignore errors for some fd might be closed by stream
-        let _ = cleanup_process(&mut p);
+        p.cleanup_process_stream();
 
         resp.status = p.exit_code;
         // broadcast exit code to all parallel watchers
@@ -535,6 +546,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::CreateContainerRequest,
     ) -> ttrpc::Result<Empty> {
         trace_rpc_call!(ctx, "create_container", req);
+        is_allowed!(req);
         match self.do_create_container(req).await {
             Err(e) => Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string())),
             Ok(_) => Ok(Empty::new()),
@@ -547,6 +559,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::StartContainerRequest,
     ) -> ttrpc::Result<Empty> {
         trace_rpc_call!(ctx, "start_container", req);
+        is_allowed!(req);
         match self.do_start_container(req).await {
             Err(e) => Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string())),
             Ok(_) => Ok(Empty::new()),
@@ -559,6 +572,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::RemoveContainerRequest,
     ) -> ttrpc::Result<Empty> {
         trace_rpc_call!(ctx, "remove_container", req);
+        is_allowed!(req);
         match self.do_remove_container(req).await {
             Err(e) => Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string())),
             Ok(_) => Ok(Empty::new()),
@@ -571,6 +585,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::ExecProcessRequest,
     ) -> ttrpc::Result<Empty> {
         trace_rpc_call!(ctx, "exec_process", req);
+        is_allowed!(req);
         match self.do_exec_process(req).await {
             Err(e) => Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string())),
             Ok(_) => Ok(Empty::new()),
@@ -583,6 +598,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::SignalProcessRequest,
     ) -> ttrpc::Result<Empty> {
         trace_rpc_call!(ctx, "signal_process", req);
+        is_allowed!(req);
         match self.do_signal_process(req).await {
             Err(e) => Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string())),
             Ok(_) => Ok(Empty::new()),
@@ -595,6 +611,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::WaitProcessRequest,
     ) -> ttrpc::Result<WaitProcessResponse> {
         trace_rpc_call!(ctx, "wait_process", req);
+        is_allowed!(req);
         self.do_wait_process(req)
             .await
             .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))
@@ -606,6 +623,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::UpdateContainerRequest,
     ) -> ttrpc::Result<Empty> {
         trace_rpc_call!(ctx, "update_container", req);
+        is_allowed!(req);
         let cid = req.container_id.clone();
         let res = req.resources;
 
@@ -641,6 +659,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::StatsContainerRequest,
     ) -> ttrpc::Result<StatsContainerResponse> {
         trace_rpc_call!(ctx, "stats_container", req);
+        is_allowed!(req);
         let cid = req.container_id;
         let s = Arc::clone(&self.sandbox);
         let mut sandbox = s.lock().await;
@@ -662,6 +681,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::PauseContainerRequest,
     ) -> ttrpc::Result<protocols::empty::Empty> {
         trace_rpc_call!(ctx, "pause_container", req);
+        is_allowed!(req);
         let cid = req.get_container_id();
         let s = Arc::clone(&self.sandbox);
         let mut sandbox = s.lock().await;
@@ -685,6 +705,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::ResumeContainerRequest,
     ) -> ttrpc::Result<protocols::empty::Empty> {
         trace_rpc_call!(ctx, "resume_container", req);
+        is_allowed!(req);
         let cid = req.get_container_id();
         let s = Arc::clone(&self.sandbox);
         let mut sandbox = s.lock().await;
@@ -707,6 +728,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         _ctx: &TtrpcContext,
         req: protocols::agent::WriteStreamRequest,
     ) -> ttrpc::Result<WriteStreamResponse> {
+        is_allowed!(req);
         self.do_write_stream(req)
             .await
             .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))
@@ -717,6 +739,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         _ctx: &TtrpcContext,
         req: protocols::agent::ReadStreamRequest,
     ) -> ttrpc::Result<ReadStreamResponse> {
+        is_allowed!(req);
         self.do_read_stream(req, true)
             .await
             .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))
@@ -727,6 +750,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         _ctx: &TtrpcContext,
         req: protocols::agent::ReadStreamRequest,
     ) -> ttrpc::Result<ReadStreamResponse> {
+        is_allowed!(req);
         self.do_read_stream(req, false)
             .await
             .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))
@@ -738,6 +762,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::CloseStdinRequest,
     ) -> ttrpc::Result<Empty> {
         trace_rpc_call!(ctx, "close_stdin", req);
+        is_allowed!(req);
 
         let cid = req.container_id.clone();
         let eid = req.exec_id;
@@ -751,19 +776,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
             )
         })?;
 
-        if p.term_master.is_some() {
-            p.close_stream(StreamType::TermMaster);
-            let _ = unistd::close(p.term_master.unwrap());
-            p.term_master = None;
-        }
-
-        if p.parent_stdin.is_some() {
-            p.close_stream(StreamType::ParentStdin);
-            let _ = unistd::close(p.parent_stdin.unwrap());
-            p.parent_stdin = None;
-        }
-
-        p.notify_term_close();
+        p.close_stdin();
 
         Ok(Empty::new())
     }
@@ -774,6 +787,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::TtyWinResizeRequest,
     ) -> ttrpc::Result<Empty> {
         trace_rpc_call!(ctx, "tty_win_resize", req);
+        is_allowed!(req);
 
         let cid = req.container_id.clone();
         let eid = req.exec_id.clone();
@@ -814,6 +828,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::UpdateInterfaceRequest,
     ) -> ttrpc::Result<Interface> {
         trace_rpc_call!(ctx, "update_interface", req);
+        is_allowed!(req);
 
         let interface = req.interface.into_option().ok_or_else(|| {
             ttrpc_error(
@@ -841,6 +856,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::UpdateRoutesRequest,
     ) -> ttrpc::Result<Routes> {
         trace_rpc_call!(ctx, "update_routes", req);
+        is_allowed!(req);
 
         let new_routes = req
             .routes
@@ -881,6 +897,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::ListInterfacesRequest,
     ) -> ttrpc::Result<Interfaces> {
         trace_rpc_call!(ctx, "list_interfaces", req);
+        is_allowed!(req);
 
         let list = self
             .sandbox
@@ -908,6 +925,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::ListRoutesRequest,
     ) -> ttrpc::Result<Routes> {
         trace_rpc_call!(ctx, "list_routes", req);
+        is_allowed!(req);
 
         let list = self
             .sandbox
@@ -930,14 +948,16 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::StartTracingRequest,
     ) -> ttrpc::Result<Empty> {
         info!(sl!(), "start_tracing {:?}", req);
+        is_allowed!(req);
         Ok(Empty::new())
     }
 
     async fn stop_tracing(
         &self,
         _ctx: &TtrpcContext,
-        _req: protocols::agent::StopTracingRequest,
+        req: protocols::agent::StopTracingRequest,
     ) -> ttrpc::Result<Empty> {
+        is_allowed!(req);
         Ok(Empty::new())
     }
 
@@ -947,6 +967,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::CreateSandboxRequest,
     ) -> ttrpc::Result<Empty> {
         trace_rpc_call!(ctx, "create_sandbox", req);
+        is_allowed!(req);
 
         {
             let sandbox = self.sandbox.clone();
@@ -1012,6 +1033,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::DestroySandboxRequest,
     ) -> ttrpc::Result<Empty> {
         trace_rpc_call!(ctx, "destroy_sandbox", req);
+        is_allowed!(req);
 
         let s = Arc::clone(&self.sandbox);
         let mut sandbox = s.lock().await;
@@ -1033,6 +1055,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::AddARPNeighborsRequest,
     ) -> ttrpc::Result<Empty> {
         trace_rpc_call!(ctx, "add_arp_neighbors", req);
+        is_allowed!(req);
 
         let neighs = req
             .neighbors
@@ -1066,6 +1089,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         ctx: &TtrpcContext,
         req: protocols::agent::OnlineCPUMemRequest,
     ) -> ttrpc::Result<Empty> {
+        is_allowed!(req);
         let s = Arc::clone(&self.sandbox);
         let sandbox = s.lock().await;
         trace_rpc_call!(ctx, "online_cpu_mem", req);
@@ -1083,6 +1107,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::ReseedRandomDevRequest,
     ) -> ttrpc::Result<Empty> {
         trace_rpc_call!(ctx, "reseed_random_dev", req);
+        is_allowed!(req);
 
         random::reseed_rng(req.data.as_slice())
             .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
@@ -1096,6 +1121,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::GuestDetailsRequest,
     ) -> ttrpc::Result<GuestDetailsResponse> {
         trace_rpc_call!(ctx, "get_guest_details", req);
+        is_allowed!(req);
 
         info!(sl!(), "get guest details!");
         let mut resp = GuestDetailsResponse::new();
@@ -1124,6 +1150,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::MemHotplugByProbeRequest,
     ) -> ttrpc::Result<Empty> {
         trace_rpc_call!(ctx, "mem_hotplug_by_probe", req);
+        is_allowed!(req);
 
         do_mem_hotplug_by_probe(&req.memHotplugProbeAddr)
             .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
@@ -1137,6 +1164,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::SetGuestDateTimeRequest,
     ) -> ttrpc::Result<Empty> {
         trace_rpc_call!(ctx, "set_guest_date_time", req);
+        is_allowed!(req);
 
         do_set_guest_date_time(req.Sec, req.Usec)
             .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
@@ -1150,6 +1178,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::CopyFileRequest,
     ) -> ttrpc::Result<Empty> {
         trace_rpc_call!(ctx, "copy_file", req);
+        is_allowed!(req);
 
         do_copy_file(&req).map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
 
@@ -1162,6 +1191,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::GetMetricsRequest,
     ) -> ttrpc::Result<Metrics> {
         trace_rpc_call!(ctx, "get_metrics", req);
+        is_allowed!(req);
 
         match get_metrics(&req) {
             Err(e) => Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string())),
@@ -1176,8 +1206,9 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
     async fn get_oom_event(
         &self,
         _ctx: &TtrpcContext,
-        _req: protocols::agent::GetOOMEventRequest,
+        req: protocols::agent::GetOOMEventRequest,
     ) -> ttrpc::Result<OOMEvent> {
+        is_allowed!(req);
         let sandbox = self.sandbox.clone();
         let s = sandbox.lock().await;
         let event_rx = &s.event_rx.clone();
@@ -1203,8 +1234,11 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         req: protocols::agent::AddSwapRequest,
     ) -> ttrpc::Result<Empty> {
         trace_rpc_call!(ctx, "add_swap", req);
+        is_allowed!(req);
 
-        do_add_swap(&req).map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
+        do_add_swap(&self.sandbox, &req)
+            .await
+            .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
 
         Ok(Empty::new())
     }
@@ -1288,11 +1322,19 @@ fn get_memory_info(block_size: bool, hotplug: bool) -> Result<(u64, bool)> {
     Ok((size, plug))
 }
 
+pub fn have_seccomp() -> bool {
+    if cfg!(feature = "seccomp") {
+        return true;
+    }
+
+    false
+}
+
 fn get_agent_details() -> AgentDetails {
     let mut detail = AgentDetails::new();
 
     detail.set_version(AGENT_VERSION.to_string());
-    detail.set_supports_seccomp(false);
+    detail.set_supports_seccomp(have_seccomp());
     detail.init_daemon = unistd::getpid() == Pid::from_raw(1);
 
     detail.device_handlers = RepeatedField::new();
@@ -1557,43 +1599,13 @@ fn do_copy_file(req: &CopyFileRequest) -> Result<()> {
     Ok(())
 }
 
-pub fn path_name_lookup<P: std::clone::Clone + AsRef<Path> + std::fmt::Debug>(
-    path: P,
-    lookup: &str,
-) -> Result<(PathBuf, String)> {
-    for entry in fs::read_dir(path.clone())? {
-        let entry = entry?;
-        if let Some(name) = entry.path().file_name() {
-            if let Some(name) = name.to_str() {
-                if Some(0) == name.find(lookup) {
-                    return Ok((entry.path(), name.to_string()));
-                }
-            }
-        }
-    }
-    Err(anyhow!("cannot get {} dir in {:?}", lookup, path))
-}
-
-fn do_add_swap(req: &AddSwapRequest) -> Result<()> {
-    // re-scan PCI bus
-    // looking for hidden devices
-    rescan_pci_bus().context("Could not rescan PCI bus")?;
-
+async fn do_add_swap(sandbox: &Arc<Mutex<Sandbox>>, req: &AddSwapRequest) -> Result<()> {
     let mut slots = Vec::new();
     for slot in &req.PCIPath {
-        slots.push(pci::Slot::new(*slot as u8)?);
+        slots.push(pci::SlotFn::new(*slot, 0)?);
     }
     let pcipath = pci::Path::new(slots)?;
-    let root_bus_sysfs = format!("{}{}", SYSFS_DIR, create_pci_root_bus_path());
-    let sysfs_rel_path = format!(
-        "{}{}",
-        root_bus_sysfs,
-        pcipath_to_sysfs(&root_bus_sysfs, &pcipath)?
-    );
-    let (mut virtio_path, _) = path_name_lookup(sysfs_rel_path, "virtio")?;
-    virtio_path.push("block");
-    let (_, dev_name) = path_name_lookup(virtio_path, "vd")?;
-    let dev_name = format!("/dev/{}", dev_name);
+    let dev_name = get_virtio_blk_pci_device_name(sandbox, &pcipath).await?;
 
     let c_str = CString::new(dev_name)?;
     let ret = unsafe { libc::swapon(c_str.as_ptr() as *const c_char, 0) };
@@ -1637,11 +1649,6 @@ fn setup_bundle(cid: &str, spec: &mut Spec) -> Result<PathBuf> {
         readonly: spec_root.readonly,
     });
 
-    info!(
-        sl!(),
-        "{:?}",
-        spec.process.as_ref().unwrap().console_size.as_ref()
-    );
     let _ = spec.save(config_path.to_str().unwrap());
 
     let olddir = unistd::getcwd().context("cannot getcwd")?;
@@ -1650,37 +1657,6 @@ fn setup_bundle(cid: &str, spec: &mut Spec) -> Result<PathBuf> {
     Ok(olddir)
 }
 
-fn cleanup_process(p: &mut Process) -> Result<()> {
-    if p.parent_stdin.is_some() {
-        p.close_stream(StreamType::ParentStdin);
-        unistd::close(p.parent_stdin.unwrap())?;
-    }
-
-    if p.parent_stdout.is_some() {
-        p.close_stream(StreamType::ParentStdout);
-        unistd::close(p.parent_stdout.unwrap())?;
-    }
-
-    if p.parent_stderr.is_some() {
-        p.close_stream(StreamType::ParentStderr);
-        unistd::close(p.parent_stderr.unwrap())?;
-    }
-
-    if p.term_master.is_some() {
-        p.close_stream(StreamType::TermMaster);
-        unistd::close(p.term_master.unwrap())?;
-    }
-
-    p.notify_term_close();
-
-    p.parent_stdin = None;
-    p.parent_stdout = None;
-    p.parent_stderr = None;
-    p.term_master = None;
-
-    Ok(())
-}
-
 fn load_kernel_module(module: &protocols::agent::KernelModule) -> Result<()> {
     if module.name.is_empty() {
         return Err(anyhow!("Kernel module name is empty"));

src/agent/src/sandbox.rs

@@ -464,7 +464,10 @@ mod tests {
         baremount(src, dst, "bind", MsFlags::MS_BIND, "", logger)
     }
 
+    use serial_test::serial;
+
     #[tokio::test]
+    #[serial]
     async fn set_sandbox_storage() {
         let logger = slog::Logger::root(slog::Discard, o!());
         let mut s = Sandbox::new(&logger).unwrap();
@@ -499,6 +502,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn remove_sandbox_storage() {
         skip_if_not_root!();
 
@@ -555,6 +559,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn unset_and_remove_sandbox_storage() {
         skip_if_not_root!();
 
@@ -606,6 +611,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn unset_sandbox_storage() {
         let logger = slog::Logger::root(slog::Discard, o!());
         let mut s = Sandbox::new(&logger).unwrap();
@@ -689,6 +695,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn get_container_entry_exist() {
         skip_if_not_root!();
         let logger = slog::Logger::root(slog::Discard, o!());
@@ -702,6 +709,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn get_container_no_entry() {
         let logger = slog::Logger::root(slog::Discard, o!());
         let mut s = Sandbox::new(&logger).unwrap();
@@ -711,6 +719,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn add_and_get_container() {
         skip_if_not_root!();
         let logger = slog::Logger::root(slog::Discard, o!());
@@ -722,6 +731,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn update_shared_pidns() {
         skip_if_not_root!();
         let logger = slog::Logger::root(slog::Discard, o!());
@@ -740,6 +750,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn add_guest_hooks() {
         let logger = slog::Logger::root(slog::Discard, o!());
         let mut s = Sandbox::new(&logger).unwrap();
@@ -763,6 +774,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn test_sandbox_set_destroy() {
         let logger = slog::Logger::root(slog::Discard, o!());
         let mut s = Sandbox::new(&logger).unwrap();

src/agent/src/tracer.rs

@@ -3,60 +3,17 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use crate::config::AgentConfig;
 use anyhow::Result;
 use opentelemetry::sdk::propagation::TraceContextPropagator;
 use opentelemetry::{global, sdk::trace::Config, trace::TracerProvider};
 use slog::{info, o, Logger};
 use std::collections::HashMap;
-use std::error::Error;
-use std::fmt;
-use std::str::FromStr;
 use tracing_opentelemetry::OpenTelemetryLayer;
 use tracing_subscriber::layer::SubscriberExt;
 use tracing_subscriber::Registry;
 use ttrpc::r#async::TtrpcContext;
 
-#[derive(Debug, PartialEq)]
-pub enum TraceType {
-    Disabled,
-    Isolated,
-}
-
-#[derive(Debug)]
-pub struct TraceTypeError {
-    details: String,
-}
-
-impl TraceTypeError {
-    fn new(msg: &str) -> TraceTypeError {
-        TraceTypeError {
-            details: msg.into(),
-        }
-    }
-}
-
-impl Error for TraceTypeError {}
-
-impl fmt::Display for TraceTypeError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}", self.details)
-    }
-}
-
-impl FromStr for TraceType {
-    type Err = TraceTypeError;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        match s {
-            "isolated" => Ok(TraceType::Isolated),
-            "disabled" => Ok(TraceType::Disabled),
-            _ => Err(TraceTypeError::new("invalid trace type")),
-        }
-    }
-}
-
-pub fn setup_tracing(name: &'static str, logger: &Logger, _agent_cfg: &AgentConfig) -> Result<()> {
+pub fn setup_tracing(name: &'static str, logger: &Logger) -> Result<()> {
     let logger = logger.new(o!("subsystem" => "vsock-tracer"));
 
     let exporter = vsock_exporter::Exporter::builder()

src/agent/src/uevent.rs

@@ -97,10 +97,18 @@ impl Uevent {
         })
     }
 
+    #[instrument]
+    async fn process_remove(&self, logger: &Logger, sandbox: &Arc<Mutex<Sandbox>>) {
+        let mut sb = sandbox.lock().await;
+        sb.uevent_map.remove(&self.devpath);
+    }
+
     #[instrument]
     async fn process(&self, logger: &Logger, sandbox: &Arc<Mutex<Sandbox>>) {
         if self.action == U_EVENT_ACTION_ADD {
             return self.process_add(logger, sandbox).await;
+        } else if self.action == U_EVENT_ACTION_REMOVE {
+            return self.process_remove(logger, sandbox).await;
         }
         debug!(*logger, "ignoring event"; "uevent" => format!("{:?}", self));
     }
@@ -111,10 +119,13 @@ pub async fn wait_for_uevent(
     sandbox: &Arc<Mutex<Sandbox>>,
     matcher: impl UeventMatcher,
 ) -> Result<Uevent> {
+    let logprefix = format!("Waiting for {:?}", &matcher);
+
+    info!(sl!(), "{}", logprefix);
     let mut sb = sandbox.lock().await;
     for uev in sb.uevent_map.values() {
         if matcher.is_match(uev) {
-            info!(sl!(), "Device {:?} found in device map", uev);
+            info!(sl!(), "{}: found {:?} in uevent map", logprefix, &uev);
             return Ok(uev.clone());
         }
     }
@@ -129,7 +140,8 @@ pub async fn wait_for_uevent(
     sb.uevent_watchers.push(Some((Box::new(matcher), tx)));
     drop(sb); // unlock
 
-    info!(sl!(), "Waiting on channel for uevent notification\n");
+    info!(sl!(), "{}: waiting on channel", logprefix);
+
     let hotplug_timeout = AGENT_CONFIG.read().await.hotplug_timeout;
 
     let uev = match tokio::time::timeout(hotplug_timeout, rx).await {
@@ -146,6 +158,7 @@ pub async fn wait_for_uevent(
         }
     };
 
+    info!(sl!(), "{}: found {:?} on channel", logprefix, &uev);
     Ok(uev)
 }
 

src/agent/src/watcher.rs

@@ -49,7 +49,7 @@ struct Storage {
     /// the source becomes too large, either in number of files (>16) or total size (>1MB).
     watch: bool,
 
-    /// The list of files to watch from the source mount point and updated in the target one.
+    /// The list of files, directories, symlinks to watch from the source mount point and updated in the target one.
     watched_files: HashMap<PathBuf, SystemTime>,
 }
 
@@ -79,6 +79,20 @@ impl Drop for Storage {
     }
 }
 
+async fn copy(from: impl AsRef<Path>, to: impl AsRef<Path>) -> Result<()> {
+    if fs::symlink_metadata(&from).await?.file_type().is_symlink() {
+        // if source is a symlink, create new symlink with same link source. If
+        // the symlink exists, remove and create new one:
+        if fs::symlink_metadata(&to).await.is_ok() {
+            fs::remove_file(&to).await?;
+        }
+        fs::symlink(fs::read_link(&from).await?, &to).await?;
+    } else {
+        fs::copy(from, to).await?;
+    }
+    Ok(())
+}
+
 impl Storage {
     async fn new(storage: protos::Storage) -> Result<Storage> {
         let entry = Storage {
@@ -93,6 +107,16 @@ impl Storage {
     async fn update_target(&self, logger: &Logger, source_path: impl AsRef<Path>) -> Result<()> {
         let source_file_path = source_path.as_ref();
 
+        // if we are creating a directory: just create it, nothing more to do
+        if source_file_path.symlink_metadata()?.file_type().is_dir() {
+            fs::create_dir_all(source_file_path)
+                .await
+                .with_context(|| {
+                    format!("Unable to mkdir all for {}", source_file_path.display())
+                })?
+        }
+
+        // Assume we are dealing with either a file or a symlink now:
         let dest_file_path = if self.source_mount_point.is_file() {
             // Simple file to file copy
             // Assume target mount is a file path
@@ -110,19 +134,13 @@ impl Storage {
             dest_file_path
         };
 
-        debug!(
-            logger,
-            "Copy from {} to {}",
-            source_file_path.display(),
-            dest_file_path.display()
-        );
-        fs::copy(&source_file_path, &dest_file_path)
+        copy(&source_file_path, &dest_file_path)
             .await
             .with_context(|| {
                 format!(
                     "Copy from {} to {} failed",
                     source_file_path.display(),
-                    dest_file_path.display()
+                    dest_file_path.display(),
                 )
             })?;
 
@@ -135,7 +153,7 @@ impl Storage {
         let mut remove_list = Vec::new();
         let mut updated_files: Vec<PathBuf> = Vec::new();
 
-        // Remove deleted files for tracking list
+        // Remove deleted files for tracking list.
         self.watched_files.retain(|st, _| {
             if st.exists() {
                 true
@@ -147,10 +165,19 @@ impl Storage {
 
         // Delete from target
         for path in remove_list {
-            // File has been deleted, remove it from target mount
             let target = self.make_target_path(path)?;
-            debug!(logger, "Removing file from mount: {}", target.display());
-            let _ = fs::remove_file(target).await;
+            // The target may be a directory or a file. If it is a directory that is removed,
+            // we'll remove all files under that directory as well. Because of this, there's a
+            // chance the target (a subdirectory or file under a prior removed target) was already
+            // removed. Make sure we check if the target exists before checking the metadata, and
+            // don't return an error if the remove fails
+            if target.exists() && target.symlink_metadata()?.file_type().is_dir() {
+                debug!(logger, "Removing a directory: {}", target.display());
+                let _ = fs::remove_dir_all(target).await;
+            } else {
+                debug!(logger, "Removing a file: {}", target.display());
+                let _ = fs::remove_file(target).await;
+            }
         }
 
         // Scan new & changed files
@@ -182,15 +209,16 @@ impl Storage {
         let mut size: u64 = 0;
         debug!(logger, "Scanning path: {}", path.display());
 
-        if path.is_file() {
-            let metadata = path
-                .metadata()
-                .with_context(|| format!("Failed to query metadata for: {}", path.display()))?;
+        let metadata = path
+            .symlink_metadata()
+            .with_context(|| format!("Failed to query metadata for: {}", path.display()))?;
 
-            let modified = metadata
-                .modified()
-                .with_context(|| format!("Failed to get modified date for: {}", path.display()))?;
+        let modified = metadata
+            .modified()
+            .with_context(|| format!("Failed to get modified date for: {}", path.display()))?;
 
+        // Treat files and symlinks the same:
+        if path.is_file() || metadata.file_type().is_symlink() {
             size += metadata.len();
 
             // Insert will return old entry if any
@@ -212,6 +240,16 @@ impl Storage {
                 }
             );
         } else {
+            // Handling regular directories - check  to see if this directory is already being tracked, and
+            // track if not:
+            if self
+                .watched_files
+                .insert(path.to_path_buf(), modified)
+                .is_none()
+            {
+                update_list.push(path.to_path_buf());
+            }
+
             // Scan dir recursively
             let mut entries = fs::read_dir(path)
                 .await
@@ -612,7 +650,7 @@ mod tests {
             .unwrap();
 
         // setup storage3: many files, but still watchable
-        for i in 1..MAX_ENTRIES_PER_STORAGE + 1 {
+        for i in 1..MAX_ENTRIES_PER_STORAGE {
             fs::write(src3_path.join(format!("{}.txt", i)), "original").unwrap();
         }
 
@@ -622,6 +660,9 @@ mod tests {
             ..Default::default()
         };
 
+        // delay 20 ms between writes to files in order to ensure filesystem timestamps are unique
+        thread::sleep(Duration::from_millis(20));
+
         entries
             .add(std::iter::once(storage0), &logger)
             .await
@@ -674,7 +715,7 @@ mod tests {
             std::fs::read_dir(entries.0[3].target_mount_point.as_path())
                 .unwrap()
                 .count(),
-            MAX_ENTRIES_PER_STORAGE
+            MAX_ENTRIES_PER_STORAGE - 1
         );
 
         // Add two files to storage 0, verify it is updated without needing to run check:
@@ -692,6 +733,9 @@ mod tests {
             "updated"
         );
 
+        // delay 20 ms between writes to files in order to ensure filesystem timestamps are unique
+        thread::sleep(Duration::from_millis(20));
+
         //
         // Prepare for second check: update mount sources
         //
@@ -744,7 +788,7 @@ mod tests {
             std::fs::read_dir(entries.0[3].target_mount_point.as_path())
                 .unwrap()
                 .count(),
-            MAX_ENTRIES_PER_STORAGE + 1
+            MAX_ENTRIES_PER_STORAGE
         );
 
         // verify that we can remove files as well, but that it isn't observed until check is run
@@ -822,15 +866,20 @@ mod tests {
         fs::remove_file(source_dir.path().join("big.txt")).unwrap();
         fs::remove_file(source_dir.path().join("too-big.txt")).unwrap();
 
-        // Up to 16 files should be okay:
-        for i in 1..MAX_ENTRIES_PER_STORAGE + 1 {
+        assert_eq!(entry.scan(&logger).await.unwrap(), 0);
+
+        // Up to 15 files should be okay (can watch 15 files + 1 directory)
+        for i in 1..MAX_ENTRIES_PER_STORAGE {
             fs::write(source_dir.path().join(format!("{}.txt", i)), "original").unwrap();
         }
 
-        assert_eq!(entry.scan(&logger).await.unwrap(), MAX_ENTRIES_PER_STORAGE);
+        assert_eq!(
+            entry.scan(&logger).await.unwrap(),
+            MAX_ENTRIES_PER_STORAGE - 1
+        );
 
-        // 17 files is too many:
-        fs::write(source_dir.path().join("17.txt"), "updated").unwrap();
+        // 16 files wll be too many:
+        fs::write(source_dir.path().join("16.txt"), "updated").unwrap();
         thread::sleep(Duration::from_secs(1));
 
         // Expect to receive a MountTooManyFiles error
@@ -843,6 +892,180 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn test_copy() {
+        // prepare tmp src/destination
+        let source_dir = tempfile::tempdir().unwrap();
+        let dest_dir = tempfile::tempdir().unwrap();
+
+        // verify copy of a regular file
+        let src_file = source_dir.path().join("file.txt");
+        let dst_file = dest_dir.path().join("file.txt");
+        fs::write(&src_file, "foo").unwrap();
+        copy(&src_file, &dst_file).await.unwrap();
+        // verify destination:
+        assert!(!fs::symlink_metadata(dst_file)
+            .unwrap()
+            .file_type()
+            .is_symlink());
+
+        // verify copy of a symlink
+        let src_symlink_file = source_dir.path().join("symlink_file.txt");
+        let dst_symlink_file = dest_dir.path().join("symlink_file.txt");
+        tokio::fs::symlink(&src_file, &src_symlink_file)
+            .await
+            .unwrap();
+        copy(src_symlink_file, &dst_symlink_file).await.unwrap();
+        // verify destination:
+        assert!(fs::symlink_metadata(&dst_symlink_file)
+            .unwrap()
+            .file_type()
+            .is_symlink());
+        assert_eq!(fs::read_link(&dst_symlink_file).unwrap(), src_file);
+        assert_eq!(fs::read_to_string(&dst_symlink_file).unwrap(), "foo")
+    }
+
+    #[tokio::test]
+    async fn watch_directory_verify_dir_removal() {
+        let source_dir = tempfile::tempdir().unwrap();
+        let dest_dir = tempfile::tempdir().unwrap();
+
+        let mut entry = Storage::new(protos::Storage {
+            source: source_dir.path().display().to_string(),
+            mount_point: dest_dir.path().display().to_string(),
+            ..Default::default()
+        })
+        .await
+        .unwrap();
+        let logger = slog::Logger::root(slog::Discard, o!());
+
+        // create a path we'll remove later
+        fs::create_dir_all(source_dir.path().join("tmp")).unwrap();
+        fs::write(&source_dir.path().join("tmp/test-file"), "foo").unwrap();
+        assert_eq!(entry.scan(&logger).await.unwrap(), 3); // root, ./tmp, test-file
+
+        // Verify expected directory, file:
+        assert_eq!(
+            std::fs::read_dir(dest_dir.path().join("tmp"))
+                .unwrap()
+                .count(),
+            1
+        );
+        assert_eq!(std::fs::read_dir(&dest_dir).unwrap().count(), 1);
+
+        // Now, remove directory, and verify that the directory (and its file) are removed:
+        fs::remove_dir_all(source_dir.path().join("tmp")).unwrap();
+        thread::sleep(Duration::from_secs(1));
+        assert_eq!(entry.scan(&logger).await.unwrap(), 0);
+
+        assert_eq!(std::fs::read_dir(&dest_dir).unwrap().count(), 0);
+
+        assert_eq!(entry.scan(&logger).await.unwrap(), 0);
+    }
+
+    #[tokio::test]
+    async fn watch_directory_with_symlinks() {
+        // Prepare source directory:
+        // ..2021_10_29_03_10_48.161654083/file.txt
+        // ..data -> ..2021_10_29_03_10_48.161654083
+        // file.txt -> ..data/file.txt
+
+        let source_dir = tempfile::tempdir().unwrap();
+        let actual_dir = source_dir.path().join("..2021_10_29_03_10_48.161654083");
+        let actual_file = actual_dir.join("file.txt");
+        let sym_dir = source_dir.path().join("..data");
+        let sym_file = source_dir.path().join("file.txt");
+
+        let relative_to_dir = PathBuf::from("..2021_10_29_03_10_48.161654083");
+
+        // create backing file/path
+        fs::create_dir_all(&actual_dir).unwrap();
+        fs::write(&actual_file, "two").unwrap();
+
+        // create indirection symlink directory that points to the directory that holds the actual file:
+        tokio::fs::symlink(&relative_to_dir, &sym_dir)
+            .await
+            .unwrap();
+
+        // create presented data file symlink:
+        tokio::fs::symlink(PathBuf::from("..data/file.txt"), sym_file)
+            .await
+            .unwrap();
+
+        let dest_dir = tempfile::tempdir().unwrap();
+
+        // delay 20 ms between writes to files in order to ensure filesystem timestamps are unique
+        thread::sleep(Duration::from_millis(20));
+
+        let mut entry = Storage::new(protos::Storage {
+            source: source_dir.path().display().to_string(),
+            mount_point: dest_dir.path().display().to_string(),
+            ..Default::default()
+        })
+        .await
+        .unwrap();
+
+        let logger = slog::Logger::root(slog::Discard, o!());
+
+        assert_eq!(entry.scan(&logger).await.unwrap(), 5);
+
+        // Should copy no files since nothing is changed since last check
+        assert_eq!(entry.scan(&logger).await.unwrap(), 0);
+
+        // now what, what is updated?
+        fs::write(actual_file, "updated").unwrap();
+
+        // delay 20 ms between writes to files in order to ensure filesystem timestamps are unique
+        thread::sleep(Duration::from_millis(20));
+
+        assert_eq!(entry.scan(&logger).await.unwrap(), 1);
+
+        assert_eq!(
+            fs::read_to_string(dest_dir.path().join("file.txt")).unwrap(),
+            "updated"
+        );
+
+        // Verify that resulting file.txt is a symlink:
+        assert!(
+            tokio::fs::symlink_metadata(dest_dir.path().join("file.txt"))
+                .await
+                .unwrap()
+                .file_type()
+                .is_symlink()
+        );
+
+        // Verify that .data directory is a symlink:
+        assert!(tokio::fs::symlink_metadata(&dest_dir.path().join("..data"))
+            .await
+            .unwrap()
+            .file_type()
+            .is_symlink());
+
+        // Should copy no new files after copy happened
+        assert_eq!(entry.scan(&logger).await.unwrap(), 0);
+
+        // Now, simulate configmap update.
+        //  - create a new actual dir/file,
+        //  - update the symlink directory to point to this one
+        //  - remove old dir/file
+        let new_actual_dir = source_dir.path().join("..2021_10_31");
+        let new_actual_file = new_actual_dir.join("file.txt");
+        fs::create_dir_all(&new_actual_dir).unwrap();
+        fs::write(&new_actual_file, "new configmap").unwrap();
+
+        tokio::fs::remove_file(&sym_dir).await.unwrap();
+        tokio::fs::symlink(PathBuf::from("..2021_10_31"), &sym_dir)
+            .await
+            .unwrap();
+        tokio::fs::remove_dir_all(&actual_dir).await.unwrap();
+
+        assert_eq!(entry.scan(&logger).await.unwrap(), 3); // file, file-dir, symlink
+        assert_eq!(
+            fs::read_to_string(dest_dir.path().join("file.txt")).unwrap(),
+            "new configmap"
+        );
+    }
+
     #[tokio::test]
     async fn watch_directory() {
         // Prepare source directory:
@@ -853,6 +1076,9 @@ mod tests {
         fs::create_dir_all(source_dir.path().join("A/B")).unwrap();
         fs::write(source_dir.path().join("A/B/1.txt"), "two").unwrap();
 
+        // delay 20 ms between writes to files in order to ensure filesystem timestamps are unique
+        thread::sleep(Duration::from_millis(20));
+
         let dest_dir = tempfile::tempdir().unwrap();
 
         let mut entry = Storage::new(protos::Storage {
@@ -865,13 +1091,11 @@ mod tests {
 
         let logger = slog::Logger::root(slog::Discard, o!());
 
-        assert_eq!(entry.scan(&logger).await.unwrap(), 2);
+        assert_eq!(entry.scan(&logger).await.unwrap(), 5);
 
         // Should copy no files since nothing is changed since last check
         assert_eq!(entry.scan(&logger).await.unwrap(), 0);
 
-        // Should copy 1 file
-        thread::sleep(Duration::from_secs(1));
         fs::write(source_dir.path().join("A/B/1.txt"), "updated").unwrap();
         assert_eq!(entry.scan(&logger).await.unwrap(), 1);
         assert_eq!(
@@ -879,6 +1103,9 @@ mod tests {
             "updated"
         );
 
+        // delay 20 ms between writes to files in order to ensure filesystem timestamps are unique
+        thread::sleep(Duration::from_millis(20));
+
         // Should copy no new files after copy happened
         assert_eq!(entry.scan(&logger).await.unwrap(), 0);
 
@@ -909,7 +1136,9 @@ mod tests {
 
         assert_eq!(entry.scan(&logger).await.unwrap(), 1);
 
-        thread::sleep(Duration::from_secs(1));
+        // delay 20 ms between writes to files in order to ensure filesystem timestamps are unique
+        thread::sleep(Duration::from_millis(20));
+
         fs::write(&source_file, "two").unwrap();
         assert_eq!(entry.scan(&logger).await.unwrap(), 1);
         assert_eq!(fs::read_to_string(&dest_file).unwrap(), "two");
@@ -935,8 +1164,9 @@ mod tests {
 
         let logger = slog::Logger::root(slog::Discard, o!());
 
-        assert_eq!(entry.scan(&logger).await.unwrap(), 1);
-        assert_eq!(entry.watched_files.len(), 1);
+        // expect the root directory and the file:
+        assert_eq!(entry.scan(&logger).await.unwrap(), 2);
+        assert_eq!(entry.watched_files.len(), 2);
 
         assert!(target_file.exists());
         assert!(entry.watched_files.contains_key(&source_file));
@@ -946,7 +1176,7 @@ mod tests {
 
         assert_eq!(entry.scan(&logger).await.unwrap(), 0);
 
-        assert_eq!(entry.watched_files.len(), 0);
+        assert_eq!(entry.watched_files.len(), 1);
         assert!(!target_file.exists());
     }
 
@@ -979,7 +1209,10 @@ mod tests {
         );
     }
 
+    use serial_test::serial;
+
     #[tokio::test]
+    #[serial]
     async fn create_tmpfs() {
         skip_if_not_root!();
 
@@ -989,11 +1222,14 @@ mod tests {
         watcher.mount(&logger).await.unwrap();
         assert!(is_mounted(WATCH_MOUNT_POINT_PATH).unwrap());
 
+        thread::sleep(Duration::from_millis(20));
+
         watcher.cleanup();
         assert!(!is_mounted(WATCH_MOUNT_POINT_PATH).unwrap());
     }
 
     #[tokio::test]
+    #[serial]
     async fn spawn_thread() {
         skip_if_not_root!();
 
@@ -1023,6 +1259,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn verify_container_cleanup_watching() {
         skip_if_not_root!();
 

src/agent/vsock-exporter/Cargo.toml

@@ -15,6 +15,6 @@ serde = { version = "1.0.126", features = ["derive"] }
 tokio-vsock = "0.3.1"
 bincode = "1.3.3"
 byteorder = "1.4.3"
-slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_info"] }
+slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug"] }
 async-trait = "0.1.50"
 tokio = "1.2.0"

src/runtime/.gitignore

@@ -8,7 +8,7 @@ coverage.html
 /config/*.toml
 config-generated.go
 /containerd-shim-kata-v2
-/cmd/containerd-shim-v2/monitor_address
+/pkg/containerd-shim-v2/monitor_address
 /data/kata-collect-data.sh
 /kata-monitor
 /kata-netmon

src/runtime/Makefile

@@ -190,6 +190,7 @@ DEFVALIDVHOSTUSERSTOREPATHS := [\"$(DEFVHOSTUSERSTOREPATH)\"]
 DEFFILEMEMBACKEND := ""
 DEFVALIDFILEMEMBACKENDS := [\"$(DEFFILEMEMBACKEND)\"]
 DEFMSIZE9P := 8192
+DEFVFIOMODE := guest-kernel
 
 # Default cgroup model
 DEFSANDBOXCGROUPONLY ?= false
@@ -459,6 +460,7 @@ USER_VARS += DEFENTROPYSOURCE
 USER_VARS += DEFVALIDENTROPYSOURCES
 USER_VARS += DEFSANDBOXCGROUPONLY
 USER_VARS += DEFBINDMOUNTS
+USER_VARS += DEFVFIOMODE
 USER_VARS += FEATURE_SELINUX
 USER_VARS += BUILDFLAGS
 
@@ -531,7 +533,7 @@ $(NETMON_RUNTIME_OUTPUT): $(SOURCES) VERSION
 runtime: $(RUNTIME_OUTPUT) $(CONFIGS)
 .DEFAULT: default
 
-build: default
+build: all
 
 #Install an executable file
 # params:
@@ -573,10 +575,11 @@ $(MONITOR_OUTPUT): $(SOURCES) $(GENERATED_FILES) $(MAKEFILE_LIST) .git-commit
 
 .PHONY: \
 	check \
-	check-go-static \
 	coverage \
 	default \
 	install \
+	lint \
+	pre-commit \
 	show-header \
 	show-summary \
 	show-variables \
@@ -595,8 +598,6 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit
 
 generate-config: $(CONFIGS)
 
-check: check-go-static
-
 test: install-hook go-test
 
 install-hook:
@@ -610,15 +611,34 @@ go-test: $(GENERATED_FILES)
 	go clean -testcache
 	go test -v -mod=vendor ./...
 
-check-go-static:
-	$(QUIET_CHECK)../../ci/go-no-os-exit.sh ./cli
-	$(QUIET_CHECK)../../ci/go-no-os-exit.sh ./virtcontainers
+fast-test: $(GENERATED_FILES)
+	go clean -testcache
+	for s in $$(go list ./...); do if ! go test -failfast -v -mod=vendor -p 1 $$s; then break; fi; done
+
+GOLANGCI_LINT_FILE := ../../../tests/.ci/.golangci.yml
+GOLANGCI_LINT_NAME = golangci-lint
+GOLANGCI_LINT_CMD := $(shell command -v $(GOLANGCI_LINT_NAME) 2>/dev/null)
+lint: all
+	if [ -z $(GOLANGCI_LINT_CMD) ] ; \
+	then \
+		echo "ERROR: command $(GOLANGCI_LINT_NAME) not found. Please install it first." >&2; exit 1; \
+	fi
+
+	if [ -f $(GOLANGCI_LINT_FILE) ] ; \
+	then \
+		echo "running $(GOLANGCI_LINT_NAME)..."; \
+		$(GOLANGCI_LINT_NAME) run -c $(GOLANGCI_LINT_FILE) ; \
+	else \
+		echo "ERROR: file $(GOLANGCI_LINT_FILE) not found. You should clone https://github.com/kata-containers/tests to run $(GOLANGCI_LINT_NAME) locally." >&2; exit 1; \
+	fi;
+
+pre-commit: lint fast-test
 
 coverage:
 	go test -v -mod=vendor -covermode=atomic -coverprofile=coverage.txt ./...
 	go tool cover -html=coverage.txt -o coverage.html
 
-install: default install-runtime install-containerd-shim-v2 install-monitor install-netmon
+install: all install-runtime install-containerd-shim-v2 install-monitor install-netmon
 
 install-bin: $(BINLIST)
 	$(QUIET_INST)$(foreach f,$(BINLIST),$(call INSTALL_EXEC,$f,$(BINDIR)))
@@ -675,6 +695,9 @@ show-usage: show-header
 	@printf "\n"
 	@printf "\tbuild                      : standard build (build everything).\n"
 	@printf "\ttest                       : run tests.\n"
+	@printf "\tpre-commit                 : run $(GOLANGCI_LINT_NAME) and tests locally.\n"
+	@printf "\tlint                       : run $(GOLANGCI_LINT_NAME).\n"
+	@printf "\tfast-test                  : run tests with failfast option.\n"
 	@printf "\tcheck                      : run code checks.\n"
 	@printf "\tclean                      : remove built files.\n"
 	@printf "\tcontainerd-shim-v2         : only build containerd shim v2.\n"

src/runtime/cmd/kata-monitor/main.go

@@ -7,6 +7,7 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"net/http"
 	"os"
 	goruntime "runtime"
@@ -25,7 +26,7 @@ var logLevel = flag.String("log-level", "info", "Log level of logrus(trace/debug
 var (
 	appName = "kata-monitor"
 	// version is the kata monitor version.
-	version = "0.1.0"
+	version = "0.2.0"
 
 	GitCommit = "unknown-commit"
 )
@@ -54,6 +55,15 @@ func printVersion(ver versionInfo) {
 	}
 }
 
+type endpoint struct {
+	handler http.HandlerFunc
+	path    string
+	desc    string
+}
+
+// global variable endpoints contains all available endpoints
+var endpoints []endpoint
+
 func main() {
 	ver := versionInfo{
 		AppName:   appName,
@@ -97,19 +107,62 @@ func main() {
 		panic(err)
 	}
 
-	// setup handlers, now only metrics is supported
+	// setup handlers, currently only metrics are supported
 	m := http.NewServeMux()
-	m.Handle("/metrics", http.HandlerFunc(km.ProcessMetricsRequest))
-	m.Handle("/sandboxes", http.HandlerFunc(km.ListSandboxes))
-	m.Handle("/agent-url", http.HandlerFunc(km.GetAgentURL))
+	endpoints = []endpoint{
+		{
+			path:    "/metrics",
+			desc:    "Get metrics from sandboxes.",
+			handler: km.ProcessMetricsRequest,
+		},
+		{
+			path:    "/sandboxes",
+			desc:    "List all Kata Containers sandboxes.",
+			handler: km.ListSandboxes,
+		},
+		{
+			path:    "/agent-url",
+			desc:    "Get sandbox agent URL.",
+			handler: km.GetAgentURL,
+		},
+		{
+			path:    "/debug/vars",
+			desc:    "Golang pprof `/debug/vars` endpoint for kata runtime shim process.",
+			handler: km.ExpvarHandler,
+		},
+		{
+			path:    "/debug/pprof/",
+			desc:    "Golang pprof `/debug/pprof/` endpoint for kata runtime shim process.",
+			handler: km.PprofIndex,
+		},
+		{
+			path:    "/debug/pprof/cmdline",
+			desc:    "Golang pprof `/debug/pprof/cmdline` endpoint for kata runtime shim process.",
+			handler: km.PprofCmdline,
+		},
+		{
+			path:    "/debug/pprof/profile",
+			desc:    "Golang pprof `/debug/pprof/profile` endpoint for kata runtime shim process.",
+			handler: km.PprofProfile,
+		},
+		{
+			path:    "/debug/pprof/symbol",
+			desc:    "Golang pprof `/debug/pprof/symbol` endpoint for kata runtime shim process.",
+			handler: km.PprofSymbol,
+		},
+		{
+			path:    "/debug/pprof/trace",
+			desc:    "Golang pprof `/debug/pprof/trace` endpoint for kata runtime shim process.",
+			handler: km.PprofTrace,
+		},
+	}
 
-	// for debug shim process
-	m.Handle("/debug/vars", http.HandlerFunc(km.ExpvarHandler))
-	m.Handle("/debug/pprof/", http.HandlerFunc(km.PprofIndex))
-	m.Handle("/debug/pprof/cmdline", http.HandlerFunc(km.PprofCmdline))
-	m.Handle("/debug/pprof/profile", http.HandlerFunc(km.PprofProfile))
-	m.Handle("/debug/pprof/symbol", http.HandlerFunc(km.PprofSymbol))
-	m.Handle("/debug/pprof/trace", http.HandlerFunc(km.PprofTrace))
+	for _, endpoint := range endpoints {
+		m.Handle(endpoint.path, endpoint.handler)
+	}
+
+	// root index page to show all endpoints in kata-monitor
+	m.Handle("/", http.HandlerFunc(indexPage))
 
 	// listening on the server
 	svr := &http.Server{
@@ -119,6 +172,23 @@ func main() {
 	logrus.Fatal(svr.ListenAndServe())
 }
 
+func indexPage(w http.ResponseWriter, r *http.Request) {
+	w.Write([]byte("Available HTTP endpoints:\n"))
+
+	spacing := 0
+	for _, endpoint := range endpoints {
+		if len(endpoint.path) > spacing {
+			spacing = len(endpoint.path)
+		}
+	}
+	spacing = spacing + 3
+
+	formattedString := fmt.Sprintf("%%-%ds: %%s\n", spacing)
+	for _, endpoint := range endpoints {
+		w.Write([]byte(fmt.Sprintf(formattedString, endpoint.path, endpoint.desc)))
+	}
+}
+
 // initLog setup logger
 func initLog() {
 	kataMonitorLog := logrus.WithFields(logrus.Fields{

src/runtime/cmd/kata-runtime/exit.go

@@ -1,28 +0,0 @@
-// Copyright (c) 2017 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-package main
-
-import "os"
-
-var atexitFuncs []func()
-
-var exitFunc = os.Exit
-
-// atexit registers a function f that will be run when exit is called. The
-// handlers so registered will be called the in reverse order of their
-// registration.
-func atexit(f func()) {
-	atexitFuncs = append(atexitFuncs, f)
-}
-
-// exit calls all atexit handlers before exiting the process with status.
-func exit(status int) {
-	for i := len(atexitFuncs) - 1; i >= 0; i-- {
-		f := atexitFuncs[i]
-		f()
-	}
-	exitFunc(status)
-}

src/runtime/cmd/kata-runtime/exit_test.go

@@ -1,42 +0,0 @@
-// Copyright (c) 2017 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-package main
-
-import (
-	"os"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-var testFoo string
-
-func testFunc() {
-	testFoo = "bar"
-}
-
-func TestExit(t *testing.T) {
-	assert := assert.New(t)
-
-	var testExitStatus int
-	exitFunc = func(status int) {
-		testExitStatus = status
-	}
-
-	defer func() {
-		exitFunc = os.Exit
-	}()
-
-	// test with no atexit functions added.
-	exit(1)
-	assert.Equal(testExitStatus, 1)
-
-	// test with a function added to the atexit list.
-	atexit(testFunc)
-	exit(0)
-	assert.Equal(testFoo, "bar")
-	assert.Equal(testExitStatus, 0)
-}

src/runtime/cmd/kata-runtime/kata-check_arm64_test.go

@@ -28,9 +28,9 @@ func setupCheckHostIsVMContainerCapable(assert *assert.Assertions, cpuInfoFile s
 func TestCCCheckCLIFunction(t *testing.T) {
 	var cpuData []testCPUData
 	moduleData := []testModuleData{
-		{filepath.Join(sysModuleDir, "kvm"), true, ""},
-		{filepath.Join(sysModuleDir, "vhost"), true, ""},
-		{filepath.Join(sysModuleDir, "vhost_net"), true, ""},
+		{filepath.Join(sysModuleDir, "kvm"), "", true},
+		{filepath.Join(sysModuleDir, "vhost"), "", true},
+		{filepath.Join(sysModuleDir, "vhost_net"), "", true},
 	}
 
 	genericCheckCLIFunction(t, cpuData, moduleData)

src/runtime/cmd/kata-runtime/kata-check_generic_test.go

@@ -39,7 +39,8 @@ func testSetCPUTypeGeneric(t *testing.T) {
 	_, config, err := makeRuntimeConfig(tmpdir)
 	assert.NoError(err)
 
-	setCPUtype(config.HypervisorType)
+	err = setCPUtype(config.HypervisorType)
+	assert.NoError(err)
 
 	assert.Equal(archRequiredCPUFlags, savedArchRequiredCPUFlags)
 	assert.Equal(archRequiredCPUAttribs, savedArchRequiredCPUAttribs)

src/runtime/cmd/kata-runtime/kata-check_ppc64le_test.go

@@ -47,8 +47,8 @@ func TestCCCheckCLIFunction(t *testing.T) {
 	}
 
 	moduleData := []testModuleData{
-		{filepath.Join(sysModuleDir, "kvm"), false, "Y"},
-		{filepath.Join(sysModuleDir, "kvm_hv"), false, "Y"},
+		{filepath.Join(sysModuleDir, "kvm"), "", true},
+		{filepath.Join(sysModuleDir, "kvm_hv"), "", true},
 	}
 
 	genericCheckCLIFunction(t, cpuData, moduleData)
@@ -58,51 +58,51 @@ func TestArchKernelParamHandler(t *testing.T) {
 	assert := assert.New(t)
 
 	type testData struct {
-		onVMM        bool
-		expectIgnore bool
 		fields       logrus.Fields
 		msg          string
+		onVMM        bool
+		expectIgnore bool
 	}
 
 	data := []testData{
-		{true, false, logrus.Fields{}, ""},
-		{false, false, logrus.Fields{}, ""},
+		{logrus.Fields{}, "", true, false},
+		{logrus.Fields{}, "", false, false},
 
 		{
-			false,
-			false,
 			logrus.Fields{
 				// wrong type
 				"parameter": 123,
 			},
 			"foo",
+			false,
+			false,
 		},
 
 		{
-			false,
-			false,
 			logrus.Fields{
 				"parameter": "unrestricted_guest",
 			},
 			"",
+			false,
+			false,
 		},
 
 		{
-			true,
-			true,
 			logrus.Fields{
 				"parameter": "unrestricted_guest",
 			},
 			"",
+			true,
+			true,
 		},
 
 		{
-			false,
-			true,
 			logrus.Fields{
 				"parameter": "nested",
 			},
 			"",
+			false,
+			true,
 		},
 	}
 

src/runtime/cmd/kata-runtime/kata-check_s390x_test.go

@@ -47,7 +47,7 @@ func TestCCCheckCLIFunction(t *testing.T) {
 	}
 
 	moduleData := []testModuleData{
-		{filepath.Join(sysModuleDir, "kvm"), false, "Y"},
+		{filepath.Join(sysModuleDir, "kvm"), "", true},
 	}
 
 	genericCheckCLIFunction(t, cpuData, moduleData)
@@ -57,51 +57,51 @@ func TestArchKernelParamHandler(t *testing.T) {
 	assert := assert.New(t)
 
 	type testData struct {
-		onVMM        bool
-		expectIgnore bool
 		fields       logrus.Fields
 		msg          string
+		onVMM        bool
+		expectIgnore bool
 	}
 
 	data := []testData{
-		{true, false, logrus.Fields{}, ""},
-		{false, false, logrus.Fields{}, ""},
+		{logrus.Fields{}, "", true, false},
+		{logrus.Fields{}, "", false, false},
 
 		{
-			false,
-			false,
 			logrus.Fields{
 				// wrong type
 				"parameter": 123,
 			},
 			"foo",
+			false,
+			false,
 		},
 
 		{
-			false,
-			false,
 			logrus.Fields{
 				"parameter": "unrestricted_guest",
 			},
 			"",
+			false,
+			false,
 		},
 
 		{
-			true,
-			true,
 			logrus.Fields{
 				"parameter": "unrestricted_guest",
 			},
 			"",
+			true,
+			true,
 		},
 
 		{
-			false,
-			true,
 			logrus.Fields{
 				"parameter": "nested",
 			},
 			"",
+			false,
+			true,
 		},
 	}
 

src/runtime/cmd/kata-runtime/kata-env.go

@@ -29,7 +29,7 @@ import (
 //
 // XXX: Increment for every change to the output format
 // (meaning any change to the EnvInfo type).
-const formatVersion = "1.0.25"
+const formatVersion = "1.0.26"
 
 // MetaInfo stores information on the format of the output itself
 type MetaInfo struct {
@@ -108,6 +108,7 @@ type HypervisorInfo struct {
 	EntropySource        string
 	SharedFS             string
 	VirtioFSDaemon       string
+	SocketPath           string
 	Msize9p              uint32
 	MemorySlots          uint32
 	PCIeRootPort         uint32
@@ -117,10 +118,8 @@ type HypervisorInfo struct {
 
 // AgentInfo stores agent details
 type AgentInfo struct {
-	TraceMode string
-	TraceType string
-	Debug     bool
-	Trace     bool
+	Debug bool
+	Trace bool
 }
 
 // DistroInfo stores host operating system distribution details.
@@ -131,13 +130,14 @@ type DistroInfo struct {
 
 // HostInfo stores host details
 type HostInfo struct {
-	Kernel             string
-	Architecture       string
-	Distro             DistroInfo
-	CPU                CPUInfo
-	Memory             MemoryInfo
-	VMContainerCapable bool
-	SupportVSocks      bool
+	AvailableGuestProtections []string
+	Kernel                    string
+	Architecture              string
+	Distro                    DistroInfo
+	CPU                       CPUInfo
+	Memory                    MemoryInfo
+	VMContainerCapable        bool
+	SupportVSocks             bool
 }
 
 // NetmonInfo stores netmon details
@@ -157,11 +157,11 @@ type EnvInfo struct {
 	Meta       MetaInfo
 	Image      ImageInfo
 	Initrd     InitrdInfo
-	Agent      AgentInfo
 	Hypervisor HypervisorInfo
-	Netmon     NetmonInfo
 	Runtime    RuntimeInfo
+	Netmon     NetmonInfo
 	Host       HostInfo
+	Agent      AgentInfo
 }
 
 func getMetaInfo() MetaInfo {
@@ -242,14 +242,17 @@ func getHostInfo() (HostInfo, error) {
 
 	memoryInfo := getMemoryInfo()
 
+	availableGuestProtection := vc.AvailableGuestProtections()
+
 	host := HostInfo{
-		Kernel:             hostKernelVersion,
-		Architecture:       arch,
-		Distro:             hostDistro,
-		CPU:                hostCPU,
-		Memory:             memoryInfo,
-		VMContainerCapable: hostVMContainerCapable,
-		SupportVSocks:      supportVSocks,
+		Kernel:                    hostKernelVersion,
+		Architecture:              arch,
+		Distro:                    hostDistro,
+		CPU:                       hostCPU,
+		Memory:                    memoryInfo,
+		AvailableGuestProtections: availableGuestProtection,
+		VMContainerCapable:        hostVMContainerCapable,
+		SupportVSocks:             supportVSocks,
 	}
 
 	return host, nil
@@ -303,13 +306,11 @@ func getAgentInfo(config oci.RuntimeConfig) (AgentInfo, error) {
 	agentConfig := config.AgentConfig
 	agent.Debug = agentConfig.Debug
 	agent.Trace = agentConfig.Trace
-	agent.TraceMode = agentConfig.TraceMode
-	agent.TraceType = agentConfig.TraceType
 
 	return agent, nil
 }
 
-func getHypervisorInfo(config oci.RuntimeConfig) HypervisorInfo {
+func getHypervisorInfo(config oci.RuntimeConfig) (HypervisorInfo, error) {
 	hypervisorPath := config.HypervisorConfig.HypervisorPath
 
 	version, err := getCommandVersion(hypervisorPath)
@@ -317,6 +318,19 @@ func getHypervisorInfo(config oci.RuntimeConfig) HypervisorInfo {
 		version = unknown
 	}
 
+	hypervisorType := config.HypervisorType
+
+	socketPath := unknown
+
+	// It is only reliable to make this call as root since a
+	// non-privileged user may not have access to /dev/vhost-vsock.
+	if os.Geteuid() == 0 {
+		socketPath, err = vc.GetHypervisorSocketTemplate(hypervisorType, &config.HypervisorConfig)
+		if err != nil {
+			return HypervisorInfo{}, err
+		}
+	}
+
 	return HypervisorInfo{
 		Debug:             config.HypervisorConfig.Debug,
 		MachineType:       config.HypervisorConfig.HypervisorMachineType,
@@ -331,7 +345,8 @@ func getHypervisorInfo(config oci.RuntimeConfig) HypervisorInfo {
 
 		HotplugVFIOOnRootBus: config.HypervisorConfig.HotplugVFIOOnRootBus,
 		PCIeRootPort:         config.HypervisorConfig.PCIeRootPort,
-	}
+		SocketPath:           socketPath,
+	}, nil
 }
 
 func getEnvInfo(configFile string, config oci.RuntimeConfig) (env EnvInfo, err error) {
@@ -356,7 +371,10 @@ func getEnvInfo(configFile string, config oci.RuntimeConfig) (env EnvInfo, err e
 		return EnvInfo{}, err
 	}
 
-	hypervisor := getHypervisorInfo(config)
+	hypervisor, err := getHypervisorInfo(config)
+	if err != nil {
+		return EnvInfo{}, err
+	}
 
 	image := ImageInfo{
 		Path: config.HypervisorConfig.ImagePath,

src/runtime/cmd/kata-runtime/kata-env_test.go

@@ -184,10 +184,6 @@ func getExpectedAgentDetails(config oci.RuntimeConfig) (AgentInfo, error) {
 	return AgentInfo{
 		Debug: agentConfig.Debug,
 		Trace: agentConfig.Trace,
-
-		// No trace mode/type set by default
-		TraceMode: "",
-		TraceType: "",
 	}, nil
 }
 
@@ -281,7 +277,7 @@ VERSION_ID="%s"
 }
 
 func getExpectedHypervisor(config oci.RuntimeConfig) HypervisorInfo {
-	return HypervisorInfo{
+	info := HypervisorInfo{
 		Version:           testHypervisorVersion,
 		Path:              config.HypervisorConfig.HypervisorPath,
 		MachineType:       config.HypervisorConfig.HypervisorMachineType,
@@ -296,6 +292,16 @@ func getExpectedHypervisor(config oci.RuntimeConfig) HypervisorInfo {
 		HotplugVFIOOnRootBus: config.HypervisorConfig.HotplugVFIOOnRootBus,
 		PCIeRootPort:         config.HypervisorConfig.PCIeRootPort,
 	}
+
+	if os.Geteuid() == 0 {
+		// This assumes the test hypervisor is a non-hybrid-vsock
+		// one (such as QEMU).
+		info.SocketPath = ""
+	} else {
+		info.SocketPath = unknown
+	}
+
+	return info
 }
 
 func getExpectedImage(config oci.RuntimeConfig) ImageInfo {
@@ -677,14 +683,10 @@ func TestEnvGetAgentInfo(t *testing.T) {
 	assert.True(t, agent.Debug)
 
 	agentConfig.Trace = true
-	agentConfig.TraceMode = "traceMode"
-	agentConfig.TraceType = "traceType"
 	config.AgentConfig = agentConfig
 	agent, err = getAgentInfo(config)
 	assert.NoError(t, err)
 	assert.True(t, agent.Trace)
-	assert.Equal(t, agent.TraceMode, "traceMode")
-	assert.Equal(t, agent.TraceType, "traceType")
 }
 
 func testEnvShowTOMLSettings(t *testing.T, tmpdir string, tmpfile *os.File) error {
@@ -1015,12 +1017,58 @@ func TestGetHypervisorInfo(t *testing.T) {
 	_, config, err := makeRuntimeConfig(tmpdir)
 	assert.NoError(err)
 
-	info := getHypervisorInfo(config)
+	info, err := getHypervisorInfo(config)
+	assert.NoError(err)
 	assert.Equal(info.Version, testHypervisorVersion)
 
 	err = os.Remove(config.HypervisorConfig.HypervisorPath)
 	assert.NoError(err)
 
-	info = getHypervisorInfo(config)
+	info, err = getHypervisorInfo(config)
+	assert.NoError(err)
 	assert.Equal(info.Version, unknown)
 }
+
+func TestGetHypervisorInfoSocket(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	_, config, err := makeRuntimeConfig(tmpdir)
+	assert.NoError(err)
+
+	type TestHypervisorDetails struct {
+		hType       vc.HypervisorType
+		hybridVsock bool
+	}
+
+	hypervisors := []TestHypervisorDetails{
+		{vc.AcrnHypervisor, false},
+		{vc.ClhHypervisor, true},
+		{vc.FirecrackerHypervisor, true},
+		{vc.MockHypervisor, false},
+		{vc.QemuHypervisor, false},
+	}
+
+	for i, details := range hypervisors {
+		msg := fmt.Sprintf("hypervisor[%d]: %+v", i, details)
+
+		config.HypervisorType = details.hType
+
+		info, err := getHypervisorInfo(config)
+		assert.NoError(err, msg)
+
+		if os.Geteuid() == 0 {
+			if !details.hybridVsock {
+				assert.Equal(info.SocketPath, "", msg)
+			} else {
+				assert.NotEmpty(info.SocketPath, msg)
+				assert.True(strings.HasPrefix(info.SocketPath, "/"), msg)
+			}
+		} else {
+			assert.Equal(info.SocketPath, unknown, msg)
+		}
+	}
+}

src/runtime/cmd/kata-runtime/main.go

@@ -30,12 +30,11 @@ import (
 	"github.com/urfave/cli"
 )
 
-// specConfig is the name of the file holding the containers configuration
-const specConfig = "config.json"
-
 // arch is the architecture for the running program
 const arch = goruntime.GOARCH
 
+var exitFunc = os.Exit
+
 var usage = fmt.Sprintf(`%s runtime
 
 %s is a command line program for running applications packaged
@@ -331,7 +330,7 @@ func handleShowConfig(context *cli.Context) {
 			fmt.Fprintf(defaultOutputFile, "%s\n", file)
 		}
 
-		exit(0)
+		exitFunc(0)
 	}
 }
 
@@ -451,7 +450,7 @@ func userWantsUsage(context *cli.Context) bool {
 func fatal(err error) {
 	kataLog.Error(err)
 	fmt.Fprintln(defaultErrorFile, err)
-	exit(1)
+	exitFunc(1)
 }
 
 type fatalWriter struct {

src/runtime/cmd/kata-runtime/main_test.go

@@ -13,7 +13,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path"
 	"path/filepath"
 	"regexp"
@@ -22,9 +21,7 @@ import (
 
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/vcmock"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -38,19 +35,12 @@ const (
 	testFileMode    = os.FileMode(0640)
 	testExeFileMode = os.FileMode(0750)
 
-	// small docker image used to create root filesystems from
-	testDockerImage = "busybox"
-
-	testBundle  = "bundle"
 	testConsole = "/dev/pts/999"
 )
 
 var (
 	// package variables set by calling TestMain()
-	testDir       = ""
-	testBundleDir = ""
-	tc            ktu.TestConstraint
-	ctrEngine     = katautils.CtrEngine{}
+	tc ktu.TestConstraint
 )
 
 // testingImpl is a concrete mock RVC implementation used for testing
@@ -79,57 +69,6 @@ func init() {
 	fmt.Printf("INFO: switching to fake virtcontainers implementation for testing\n")
 	vci = testingImpl
 
-	var err error
-
-	fmt.Printf("INFO: creating test directory\n")
-	testDir, err = ioutil.TempDir("", fmt.Sprintf("%s-", katautils.NAME))
-	if err != nil {
-		panic(fmt.Sprintf("ERROR: failed to create test directory: %v", err))
-	}
-	fmt.Printf("INFO: test directory is %v\n", testDir)
-
-	var output string
-	for _, name := range katautils.DockerLikeCtrEngines {
-		fmt.Printf("INFO: checking for container engine: %s\n", name)
-
-		output, err = ctrEngine.Init(name)
-		if err == nil {
-			break
-		}
-	}
-
-	if ctrEngine.Name == "" {
-		panic(fmt.Sprintf("ERROR: Docker-like container engine not accessible to current user: %v (error %v)",
-			output, err))
-	}
-
-	// Do this now to avoid hitting the test timeout value due to
-	// slow network response.
-	fmt.Printf("INFO: ensuring required container image (%v) is available\n", testDockerImage)
-	// Only hit the network if the image doesn't exist locally
-	_, err = ctrEngine.Inspect(testDockerImage)
-	if err == nil {
-		fmt.Printf("INFO: container image %v already exists locally\n", testDockerImage)
-	} else {
-		fmt.Printf("INFO: pulling container image %v\n", testDockerImage)
-		_, err = ctrEngine.Pull(testDockerImage)
-		if err != nil {
-			panic(err)
-		}
-	}
-
-	testBundleDir = filepath.Join(testDir, testBundle)
-	err = os.MkdirAll(testBundleDir, testDirMode)
-	if err != nil {
-		panic(fmt.Sprintf("ERROR: failed to create bundle directory %v: %v", testBundleDir, err))
-	}
-
-	fmt.Printf("INFO: creating OCI bundle in %v for tests to use\n", testBundleDir)
-	err = realMakeOCIBundle(testBundleDir)
-	if err != nil {
-		panic(fmt.Sprintf("ERROR: failed to create OCI bundle: %v", err))
-	}
-
 	tc = ktu.NewTestConstraint(false)
 }
 
@@ -143,8 +82,6 @@ func resetCLIGlobals() {
 func runUnitTests(m *testing.M) {
 	ret := m.Run()
 
-	os.RemoveAll(testDir)
-
 	os.Exit(ret)
 }
 
@@ -156,7 +93,7 @@ func TestMain(m *testing.M) {
 	if path.Base(os.Args[0]) == katautils.NAME+".coverage" ||
 		path.Base(os.Args[0]) == katautils.NAME {
 		main()
-		exit(0)
+		exitFunc(0)
 	}
 
 	runUnitTests(m)
@@ -232,146 +169,6 @@ func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConf
 	}, nil
 }
 
-// createOCIConfig creates an OCI configuration (spec) file in
-// the bundle directory specified (which must exist).
-func createOCIConfig(bundleDir string) error {
-	if bundleDir == "" {
-		return errors.New("BUG: Need bundle directory")
-	}
-
-	if !katautils.FileExists(bundleDir) {
-		return fmt.Errorf("BUG: Bundle directory %s does not exist", bundleDir)
-	}
-
-	var configCmd string
-
-	// Search for a suitable version of runc to use to generate
-	// the OCI config file.
-	for _, cmd := range []string{"docker-runc", "runc"} {
-		fullPath, err := exec.LookPath(cmd)
-		if err == nil {
-			configCmd = fullPath
-			break
-		}
-	}
-
-	if configCmd == "" {
-		return errors.New("Cannot find command to generate OCI config file")
-	}
-
-	_, err := utils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
-	if err != nil {
-		return err
-	}
-
-	specFile := filepath.Join(bundleDir, specConfig)
-	if !katautils.FileExists(specFile) {
-		return fmt.Errorf("generated OCI config file does not exist: %v", specFile)
-	}
-
-	return nil
-}
-
-// createRootfs creates a minimal root filesystem below the specified
-// directory.
-func createRootfs(dir string) error {
-	err := os.MkdirAll(dir, testDirMode)
-	if err != nil {
-		return err
-	}
-
-	container, err := ctrEngine.Create(testDockerImage)
-	if err != nil {
-		return err
-	}
-
-	err = ctrEngine.GetRootfs(container, dir)
-	if err != nil {
-		return err
-	}
-
-	// Clean up
-	_, err = ctrEngine.Rm(container)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// realMakeOCIBundle will create an OCI bundle (including the "config.json"
-// config file) in the directory specified (which must already exist).
-//
-// XXX: Note that tests should *NOT* call this function - they should
-// XXX: instead call makeOCIBundle().
-func realMakeOCIBundle(bundleDir string) error {
-	if bundleDir == "" {
-		return errors.New("BUG: Need bundle directory")
-	}
-
-	if !katautils.FileExists(bundleDir) {
-		return fmt.Errorf("BUG: Bundle directory %v does not exist", bundleDir)
-	}
-
-	err := createOCIConfig(bundleDir)
-	if err != nil {
-		return err
-	}
-
-	// Note the unusual parameter (a directory, not the config
-	// file to parse!)
-	spec, err := compatoci.ParseConfigJSON(bundleDir)
-	if err != nil {
-		return err
-	}
-
-	// Determine the rootfs directory name the OCI config refers to
-	ociRootPath := spec.Root.Path
-
-	rootfsDir := filepath.Join(bundleDir, ociRootPath)
-
-	if strings.HasPrefix(ociRootPath, "/") {
-		return fmt.Errorf("Cannot handle absolute rootfs as bundle must be unique to each test")
-	}
-
-	err = createRootfs(rootfsDir)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// Create an OCI bundle in the specified directory.
-//
-// Note that the directory will be created, but it's parent is expected to exist.
-//
-// This function works by copying the already-created test bundle. Ideally,
-// the bundle would be recreated for each test, but createRootfs() uses
-// docker which on some systems is too slow, resulting in the tests timing
-// out.
-func makeOCIBundle(bundleDir string) error {
-	from := testBundleDir
-	to := bundleDir
-
-	// only the basename of bundleDir needs to exist as bundleDir
-	// will get created by cp(1).
-	base := filepath.Dir(bundleDir)
-
-	for _, dir := range []string{from, base} {
-		if !katautils.FileExists(dir) {
-			return fmt.Errorf("BUG: directory %v should exist", dir)
-		}
-	}
-
-	output, err := utils.RunCommandFull([]string{"cp", "-a", from, to}, true)
-	if err != nil {
-		return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output)
-	}
-
-	return nil
-}
-
 func createCLIContextWithApp(flagSet *flag.FlagSet, app *cli.App) *cli.Context {
 	ctx := cli.NewContext(app, flagSet, nil)
 
@@ -390,69 +187,6 @@ func createCLIContext(flagset *flag.FlagSet) *cli.Context {
 	return createCLIContextWithApp(flagset, cli.NewApp())
 }
 
-func TestMakeOCIBundle(t *testing.T) {
-	assert := assert.New(t)
-
-	tmpdir, err := ioutil.TempDir(testDir, "")
-	assert.NoError(err)
-	defer os.RemoveAll(tmpdir)
-
-	bundleDir := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundleDir)
-	assert.NoError(err)
-
-	specFile := filepath.Join(bundleDir, specConfig)
-	assert.True(katautils.FileExists(specFile))
-}
-
-func TestCreateOCIConfig(t *testing.T) {
-	assert := assert.New(t)
-
-	tmpdir, err := ioutil.TempDir(testDir, "")
-	assert.NoError(err)
-	defer os.RemoveAll(tmpdir)
-
-	bundleDir := filepath.Join(tmpdir, "bundle")
-
-	err = createOCIConfig(bundleDir)
-	// ENOENT
-	assert.Error(err)
-
-	err = os.MkdirAll(bundleDir, testDirMode)
-	assert.NoError(err)
-
-	err = createOCIConfig(bundleDir)
-	assert.NoError(err)
-
-	specFile := filepath.Join(bundleDir, specConfig)
-	assert.True(katautils.FileExists(specFile))
-}
-
-func TestCreateRootfs(t *testing.T) {
-	assert := assert.New(t)
-
-	tmpdir, err := ioutil.TempDir(testDir, "")
-	assert.NoError(err)
-	defer os.RemoveAll(tmpdir)
-
-	rootfsDir := filepath.Join(tmpdir, "rootfs")
-	assert.False(katautils.FileExists(rootfsDir))
-
-	err = createRootfs(rootfsDir)
-	assert.NoError(err)
-
-	// non-comprehensive list of expected directories
-	expectedDirs := []string{"bin", "dev", "etc", "usr", "var"}
-
-	assert.True(katautils.FileExists(rootfsDir))
-
-	for _, dir := range expectedDirs {
-		dirPath := filepath.Join(rootfsDir, dir)
-		assert.True(katautils.FileExists(dirPath))
-	}
-}
-
 func TestMainUserWantsUsage(t *testing.T) {
 	assert := assert.New(t)
 
@@ -525,7 +259,7 @@ func TestMainBeforeSubCommands(t *testing.T) {
 func TestMainBeforeSubCommandsInvalidLogFile(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir(testDir, "")
+	tmpdir, err := ioutil.TempDir("", "katatest")
 	assert.NoError(err)
 	defer os.RemoveAll(tmpdir)
 
@@ -548,7 +282,7 @@ func TestMainBeforeSubCommandsInvalidLogFile(t *testing.T) {
 func TestMainBeforeSubCommandsInvalidLogFormat(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir(testDir, "")
+	tmpdir, err := ioutil.TempDir("", "katatest")
 	assert.NoError(err)
 	defer os.RemoveAll(tmpdir)
 
@@ -577,7 +311,7 @@ func TestMainBeforeSubCommandsInvalidLogFormat(t *testing.T) {
 func TestMainBeforeSubCommandsLoadConfigurationFail(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir(testDir, "")
+	tmpdir, err := ioutil.TempDir("", "katatest")
 	assert.NoError(err)
 	defer os.RemoveAll(tmpdir)
 
@@ -612,7 +346,7 @@ func TestMainBeforeSubCommandsLoadConfigurationFail(t *testing.T) {
 func TestMainBeforeSubCommandsShowCCConfigPaths(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir(testDir, "")
+	tmpdir, err := ioutil.TempDir("", "katatest")
 	assert.NoError(err)
 	defer os.RemoveAll(tmpdir)
 
@@ -676,7 +410,7 @@ func TestMainBeforeSubCommandsShowCCConfigPaths(t *testing.T) {
 func TestMainFatal(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir(testDir, "")
+	tmpdir, err := ioutil.TempDir("", "katatest")
 	assert.NoError(err)
 	defer os.RemoveAll(tmpdir)
 
@@ -900,7 +634,7 @@ func TestMainCreateRuntime(t *testing.T) {
 func TestMainVersionPrinter(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir("", "")
+	tmpdir, err := ioutil.TempDir("", "katatest")
 	assert.NoError(err)
 	defer os.RemoveAll(tmpdir)
 

src/runtime/cmd/kata-runtime/utils_test.go

@@ -18,7 +18,7 @@ import (
 )
 
 func TestFileExists(t *testing.T) {
-	dir, err := ioutil.TempDir(testDir, "")
+	dir, err := ioutil.TempDir("", "katatest")
 	if err != nil {
 		t.Fatal(err)
 	}

src/runtime/cmd/netmon/netmon.go

@@ -170,7 +170,7 @@ func newNetmon(params netmonParams) (*netmon, error) {
 
 func (n *netmon) cleanup() {
 	os.RemoveAll(n.storagePath)
-	n.netHandler.Delete()
+	n.netHandler.Close()
 	close(n.linkDoneCh)
 	close(n.rtDoneCh)
 }

src/runtime/cmd/netmon/netmon_test.go

@@ -348,7 +348,7 @@ func TestScanNetwork(t *testing.T) {
 	handler, err := netlink.NewHandle(netlinkFamily)
 	assert.Nil(t, err)
 	assert.NotNil(t, handler)
-	defer handler.Delete()
+	defer handler.Close()
 
 	idx, expected := testCreateDummyNetwork(t, handler)
 
@@ -480,7 +480,7 @@ func TestActionsCLI(t *testing.T) {
 	handler, err := netlink.NewHandle(netlinkFamily)
 	assert.Nil(t, err)
 	assert.NotNil(t, handler)
-	defer handler.Delete()
+	defer handler.Close()
 
 	n.netHandler = handler
 
@@ -569,7 +569,7 @@ func TestHandleRTMNewLink(t *testing.T) {
 	handler, err := netlink.NewHandle(netlinkFamily)
 	assert.Nil(t, err)
 	assert.NotNil(t, handler)
-	defer handler.Delete()
+	defer handler.Close()
 	n.netHandler = handler
 	err = n.handleRTMNewLink(ev)
 	assert.NotNil(t, err)
@@ -690,7 +690,7 @@ func TestHandleRouteEvent(t *testing.T) {
 	handler, err := netlink.NewHandle(netlinkFamily)
 	assert.Nil(t, err)
 	assert.NotNil(t, handler)
-	defer handler.Delete()
+	defer handler.Close()
 
 	n.netHandler = handler
 

src/runtime/config/configuration-acrn.toml.in

@@ -124,24 +124,17 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_ACRN@"
 
 # Enable agent tracing.
 #
-# If enabled, the default trace mode is "dynamic" and the
-# default trace type is "isolated". The trace mode and type are set
-# explicity with the `trace_type=` and `trace_mode=` options.
+# If enabled, the agent will generate OpenTelemetry trace spans.
 #
 # Notes:
 #
-# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly
-#   setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing`
-#   will NOT activate agent tracing.
-#
-# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for
-#   full details.
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
 #
 # (default: disabled)
 #enable_tracing = true
-#
-#trace_mode = "dynamic"
-#trace_type = "isolated"
 
 # Enable debug console.
 

src/runtime/config/configuration-clh.toml.in

@@ -105,8 +105,7 @@ virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@
 virtio_fs_cache = "@DEFVIRTIOFSCACHE@"
 
 # Block storage driver to be used for the hypervisor in case the container
-# rootfs is backed by a block device. This is virtio-scsi, virtio-blk
-# or nvdimm.
+# rootfs is backed by a block device. This is virtio-blk.
 block_device_driver = "virtio-blk"
 
 # Enable huge pages for VM RAM, default false
@@ -114,6 +113,9 @@ block_device_driver = "virtio-blk"
 # being allocated using huge pages.
 #enable_hugepages = true
 
+# Disable the 'seccomp' feature from Cloud Hypervisor, default false
+# disable_seccomp = true
+
 # This option changes the default hypervisor and kernel parameters
 # to enable debug output where available.
 #
@@ -144,24 +146,17 @@ block_device_driver = "virtio-blk"
 
 # Enable agent tracing.
 #
-# If enabled, the default trace mode is "dynamic" and the
-# default trace type is "isolated". The trace mode and type are set
-# explicity with the `trace_type=` and `trace_mode=` options.
+# If enabled, the agent will generate OpenTelemetry trace spans.
 #
 # Notes:
 #
-# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly
-#   setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing`
-#   will NOT activate agent tracing.
-#
-# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for
-#   full details.
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
 #
 # (default: disabled)
 #enable_tracing = true
-#
-#trace_mode = "dynamic"
-#trace_type = "isolated"
 
 # Enable debug console.
 
@@ -267,6 +262,27 @@ sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
 # These will not be exposed to the container workloads, and are only provided for potential guest services.
 sandbox_bind_mounts=@DEFBINDMOUNTS@
 
+# VFIO Mode
+# Determines how VFIO devices should be be presented to the container.
+# Options:
+#
+#  - vfio
+#    Matches behaviour of OCI runtimes (e.g. runc) as much as
+#    possible.  VFIO devices will appear in the container as VFIO
+#    character devices under /dev/vfio.  The exact names may differ
+#    from the host (they need to match the VM's IOMMU group numbers
+#    rather than the host's)
+#
+#  - guest-kernel
+#    This is a Kata-specific behaviour that's useful in certain cases.
+#    The VFIO device is managed by whatever driver in the VM kernel
+#    claims it.  This means it will appear as one or more device nodes
+#    or network interfaces depending on the nature of the device.
+#    Using this mode requires specially built workloads that know how
+#    to locate the relevant device interfaces within the VM.
+#
+vfio_mode="@DEFVFIOMODE@"
+
 # Enabled experimental feature list, format: ["a", "b"].
 # Experimental features are features not stable enough for production,
 # they may break compatibility, and are prepared for a big version bump.

src/runtime/config/configuration-fc.toml.in

@@ -246,24 +246,17 @@ valid_entropy_sources = @DEFVALIDENTROPYSOURCES@
 
 # Enable agent tracing.
 #
-# If enabled, the default trace mode is "dynamic" and the
-# default trace type is "isolated". The trace mode and type are set
-# explicity with the `trace_type=` and `trace_mode=` options.
+# If enabled, the agent will generate OpenTelemetry trace spans.
 #
 # Notes:
 #
-# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly
-#   setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing`
-#   will NOT activate agent tracing.
-#
-# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for
-#   full details.
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
 #
 # (default: disabled)
 #enable_tracing = true
-#
-#trace_mode = "dynamic"
-#trace_type = "isolated"
 
 # Comma separated list of kernel modules and their parameters.
 # These modules will be loaded in the guest kernel using modprobe(8).

src/runtime/config/configuration-qemu.toml.in

@@ -422,24 +422,17 @@ valid_entropy_sources = @DEFVALIDENTROPYSOURCES@
 
 # Enable agent tracing.
 #
-# If enabled, the default trace mode is "dynamic" and the
-# default trace type is "isolated". The trace mode and type are set
-# explicity with the `trace_type=` and `trace_mode=` options.
+# If enabled, the agent will generate OpenTelemetry trace spans.
 #
 # Notes:
 #
-# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly
-#   setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing`
-#   will NOT activate agent tracing.
-#
-# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for
-#   full details.
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
 #
 # (default: disabled)
 #enable_tracing = true
-#
-#trace_mode = "dynamic"
-#trace_type = "isolated"
 
 # Comma separated list of kernel modules and their parameters.
 # These modules will be loaded in the guest kernel using modprobe(8).
@@ -550,6 +543,27 @@ sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
 # These will not be exposed to the container workloads, and are only provided for potential guest services.
 sandbox_bind_mounts=@DEFBINDMOUNTS@
 
+# VFIO Mode
+# Determines how VFIO devices should be be presented to the container.
+# Options:
+#
+#  - vfio
+#    Matches behaviour of OCI runtimes (e.g. runc) as much as
+#    possible.  VFIO devices will appear in the container as VFIO
+#    character devices under /dev/vfio.  The exact names may differ
+#    from the host (they need to match the VM's IOMMU group numbers
+#    rather than the host's)
+#
+#  - guest-kernel
+#    This is a Kata-specific behaviour that's useful in certain cases.
+#    The VFIO device is managed by whatever driver in the VM kernel
+#    claims it.  This means it will appear as one or more device nodes
+#    or network interfaces depending on the nature of the device.
+#    Using this mode requires specially built workloads that know how
+#    to locate the relevant device interfaces within the VM.
+#
+vfio_mode="@DEFVFIOMODE@"
+
 # Enabled experimental feature list, format: ["a", "b"].
 # Experimental features are features not stable enough for production,
 # they may break compatibility, and are prepared for a big version bump.

src/runtime/data/completions/bash/kata-runtime

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Copyright (c) 2018 Intel Corporation
 #
@@ -53,39 +53,6 @@ _kata_subcmd_seen()
     echo ""
 }
 
-# Return 0 if the specified sub-command requires the name of an
-# *existing* container, else 1.
-_kata_subcmd_needs_existing_container()
-{
-    local subcmd="$1"
-    local cmd
-
-    for cmd in \
-	    'kata-check' \
-	    'kata-env' \
-	    'check' \
-	    'env' \
-	    'create' \
-	    'help' \
-	    'list' \
-	    'version'; do
-        [ "$cmd" = "$subcmd" ] && return 1
-    done
-
-    return 0
-}
-
-# Returns a list of container names
-_kata_get_containers()
-{
-    # Commands that manipulate containers need root privileges.
-    # If the user isn't running as root, don't attempt to obtain a list
-    # as it will result in an error.
-    [ $(id -u) -eq 0 ] || return
-
-    "$_kataruntime" list --quiet
-}
-
 _kata_bash_autocomplete() {
     COMPREPLY=()
 
@@ -104,32 +71,22 @@ _kata_bash_autocomplete() {
     local subcmd_seen=$(_kata_subcmd_seen)
 
     if [ -n "$subcmd_seen" ]; then
-	    _kata_subcmd_needs_existing_container "$subcmd_seen"
-	    local container_cmd=$?
-
-            if [ -n "$cur" ]; then
-                    # Complete with local options and maybe container names
-                    opts=$(_kata_get_subcmd_options "$subcmd_seen")
-                    [ $container_cmd -eq 0 ] && opts="$opts $(_kata_get_containers)"
-            elif [[ "${cur}" == -* ]]; then
-                    # Complete with local options
-                    opts=$(_kata_get_subcmd_options "$subcmd_seen")
-            else
-                    # Potentially complete with container names
-                    [ $container_cmd -eq 0 ] && opts="$(_kata_get_containers)"
-            fi
+        if [ -n "$cur" ]; then
+                # Complete with local options
+                opts=$(_kata_get_subcmd_options "$subcmd_seen")
+        fi
     else
-            if [ -n "$cur" ]; then
-                    # Complete with global options and subcmds
-                    opts="$opts $(_kata_get_global_options)"
-                    opts="$opts $(_kata_get_subcmds)"
-            elif [[ "${cur}" == -* ]]; then
-                    # Complete with global options
-                    opts=$(_kata_get_global_options)
-            else
-                    # Complete with subcmds
-                    opts=$(_kata_get_subcmds)
-            fi
+        if [ -n "$cur" ]; then
+                # Complete with global options and subcmds
+                opts="$opts $(_kata_get_global_options)"
+                opts="$opts $(_kata_get_subcmds)"
+        elif [[ "${cur}" == -* ]]; then
+                # Complete with global options
+                opts=$(_kata_get_global_options)
+        else
+                # Complete with subcmds
+                opts=$(_kata_get_subcmds)
+        fi
     fi
 
     [ -n "$opts" ] && COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )

src/runtime/go.mod

@@ -8,25 +8,28 @@ require (
 	github.com/blang/semver/v4 v4.0.0
 	github.com/containerd/cgroups v1.0.1
 	github.com/containerd/console v1.0.2
-	github.com/containerd/containerd v1.5.4
+	github.com/containerd/containerd v1.5.7
 	github.com/containerd/cri-containerd v1.11.1-0.20190125013620-4dd6735020f5
 	github.com/containerd/fifo v1.0.0
-	github.com/containerd/ttrpc v1.0.2
+	github.com/containerd/ttrpc v1.1.0
 	github.com/containerd/typeurl v1.0.2
 	github.com/containernetworking/plugins v0.9.1
+	github.com/coreos/go-systemd/v22 v22.3.2
 	github.com/cri-o/cri-o v1.0.0-rc2.0.20170928185954-3394b3b2d6af
+	github.com/fsnotify/fsnotify v1.4.9
 	github.com/go-ini/ini v1.28.2
 	github.com/go-openapi/errors v0.18.0
 	github.com/go-openapi/runtime v0.18.0
 	github.com/go-openapi/strfmt v0.18.0
 	github.com/go-openapi/swag v0.19.5
 	github.com/go-openapi/validate v0.18.0
+	github.com/godbus/dbus/v5 v5.0.4
 	github.com/gogo/protobuf v1.3.2
 	github.com/hashicorp/go-multierror v1.0.0
 	github.com/intel-go/cpuid v0.0.0-20210602155658-5747e5cec0d9
 	github.com/kata-containers/govmm v0.0.0-20210909155007-1b60b536f3c7
 	github.com/mdlayher/vsock v0.0.0-20191108225356-d9c65923cb8f
-	github.com/opencontainers/runc v1.0.1
+	github.com/opencontainers/runc v1.0.2
 	github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417
 	github.com/opencontainers/selinux v1.8.2
 	github.com/pkg/errors v0.9.1
@@ -37,24 +40,28 @@ require (
 	github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8
 	github.com/sirupsen/logrus v1.8.1
 	github.com/smartystreets/goconvey v1.6.4 // indirect
-	github.com/stretchr/testify v1.6.1
+	github.com/stretchr/testify v1.7.0
 	github.com/urfave/cli v1.22.2
-	github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852
+	github.com/vishvananda/netlink v1.1.1-0.20210924202909-187053b97868
 	github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae
-	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb
-	go.opentelemetry.io/otel v0.15.0
-	go.opentelemetry.io/otel/exporters/trace/jaeger v0.15.0
-	go.opentelemetry.io/otel/sdk v0.15.0
+	go.opencensus.io v0.23.0 // indirect
+	go.opentelemetry.io/otel v1.0.0
+	go.opentelemetry.io/otel/exporters/jaeger v1.0.0
+	go.opentelemetry.io/otel/sdk v1.0.0
+	go.opentelemetry.io/otel/trace v1.0.0
 	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110
-	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43
+	golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93
 	golang.org/x/sys v0.0.0-20210426230700-d19ff857e887
-	google.golang.org/grpc v1.33.2
+	golang.org/x/text v0.3.5 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb // indirect
+	google.golang.org/grpc v1.36.0
 	k8s.io/apimachinery v0.20.6
 	k8s.io/cri-api v0.20.6
 )
 
 replace (
-	github.com/containerd/containerd => github.com/containerd/containerd v1.5.4
+	github.com/containerd/containerd => github.com/containerd/containerd v1.5.8
 	github.com/opencontainers/runc => github.com/opencontainers/runc v1.0.1
 	github.com/uber-go/atomic => go.uber.org/atomic v1.5.1
 	google.golang.org/genproto => google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8

src/runtime/go.sum

@@ -45,7 +45,6 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/DataDog/sketches-go v0.0.1/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60=
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
@@ -53,8 +52,8 @@ github.com/Microsoft/go-winio v0.4.17 h1:iT12IBVClFevaf8PuVyi3UmZOVh4OqnaLxDTW2O
 github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
 github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600=
-github.com/Microsoft/hcsshim v0.8.18 h1:cYnKADiM1869gvBpos3YCteeT6sZLB48lB5dmMMs8Tg=
-github.com/Microsoft/hcsshim v0.8.18/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4=
+github.com/Microsoft/hcsshim v0.8.23 h1:47MSwtKGXet80aIn+7h4YI6fwPmwIghAnsx2aOUrG2M=
+github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
@@ -67,13 +66,10 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
-github.com/apache/thrift v0.13.0 h1:5hryIiq9gtn+MiLVn0wP37kb/uTeRZgN08WoCsAhIhI=
-github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -86,6 +82,7 @@ github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
+github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@@ -97,9 +94,11 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs=
 github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
+github.com/cilium/ebpf v0.6.2 h1:iHsfF/t4aW4heW2YKfeHrVPGdtYTL4C4KocpM8KTSnI=
 github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=
 github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss=
@@ -109,8 +108,8 @@ github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f2
 github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw=
 github.com/containerd/console v1.0.2 h1:Pi6D+aZXM+oUw1czuKgH5IJ+y0jhYcwBJfx5/Ghn9dE=
 github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ=
-github.com/containerd/containerd v1.5.4 h1:uPF0og3ByFzDnaStfiQj3fVGTEtaSNyU+bW7GR/nqGA=
-github.com/containerd/containerd v1.5.4/go.mod h1:sx18RgvW6ABJ4iYUw7Q5x7bgFOAB9B6G7+yO0XBc4zw=
+github.com/containerd/containerd v1.5.8 h1:NmkCC1/QxyZFBny8JogwLpOy2f+VEbO/f6bV2Mqtwuw=
+github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s=
 github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ=
 github.com/containerd/continuity v0.1.0 h1:UFRRY5JemiAhPZrr/uE0n8fMTLcZsUvySPr1+D7pgr8=
 github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM=
@@ -124,8 +123,9 @@ github.com/containerd/go-runc v1.0.0 h1:oU+lLv1ULm5taqgV/CJivypVODI4SUz1znWjv3nN
 github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok=
 github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms=
 github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
-github.com/containerd/ttrpc v1.0.2 h1:2/O3oTZN36q2xRolk0a2WWGgh7/Vf/liElg5hFYLX9U=
 github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y=
+github.com/containerd/ttrpc v1.1.0 h1:GbtyLRxb0gOLR0TYQWt3O6B0NvT8tMdorEHqIQo/lWI=
+github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ=
 github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg=
 github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY=
 github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s=
@@ -156,6 +156,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/cri-o/cri-o v1.0.0-rc2.0.20170928185954-3394b3b2d6af h1:H6nLV96F1LkWizYLQtrMtqJBrlJxnpjgisHsTsOS2HU=
 github.com/cri-o/cri-o v1.0.0-rc2.0.20170928185954-3394b3b2d6af/go.mod h1:POmDVglzQ2jWTlL9ZCfZ8d1QjLhmk0oB36O8T0oG75Y=
+github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg=
 github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
 github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
 github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
@@ -181,10 +182,12 @@ github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
+github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
@@ -297,9 +300,11 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -343,7 +348,7 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/intel-go/cpuid v0.0.0-20210602155658-5747e5cec0d9 h1:x9HFDMDCsaxTvC4X3o0ZN6mw99dT/wYnTItGwhBRmg0=
 github.com/intel-go/cpuid v0.0.0-20210602155658-5747e5cec0d9/go.mod h1:RmeVYf9XrPRbRc3XIx0gLYA8qOFvNoPOfaEZduRlEp4=
@@ -434,7 +439,6 @@ github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/
 github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 h1:3snG66yBm59tKhhSPQrQ/0bCrv1LQbKt40LnUPiUxdc=
 github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo=
 github.com/opencontainers/selinux v1.8.2 h1:c4ca10UMgRcvZ6h0K4HtS15UaVSBEaE+iln2LVpAuGc=
 github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=
 github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g=
@@ -483,6 +487,7 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8 h1:2c1EFnZHIPCW8qKWgHMH/fX2PkSabFc5mrVzfUNdg5U=
 github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
+github.com/seccomp/libseccomp-golang v0.9.1 h1:NJjM5DNFOs0s3kYE1WUOr6G8V97sdt46rlXTMfXGWBo=
 github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
@@ -520,8 +525,10 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -532,14 +539,12 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX
 github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo=
 github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
-github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 h1:cPXZWzzG0NllBLdjWoD1nDfaqu98YMv+OneaKc8sPOA=
 github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
+github.com/vishvananda/netlink v1.1.1-0.20210924202909-187053b97868 h1:FFT5/l13iFxg+2dzyoiXZPmMtoclsyBKnUqTEzYpDXw=
+github.com/vishvananda/netlink v1.1.1-0.20210924202909-187053b97868/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns=
 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
-github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
-github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
-github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -555,14 +560,17 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opentelemetry.io/otel v0.15.0 h1:CZFy2lPhxd4HlhZnYK8gRyDotksO3Ip9rBweY1vVYJw=
-go.opentelemetry.io/otel v0.15.0/go.mod h1:e4GKElweB8W2gWUqbghw0B8t5MCTccc9212eNHnOHwA=
-go.opentelemetry.io/otel/exporters/trace/jaeger v0.15.0 h1:OZY+lMaUJiJ6ls1dDtqKhSPJWEVNytLuRUrzuR852jc=
-go.opentelemetry.io/otel/exporters/trace/jaeger v0.15.0/go.mod h1:4DeFMzRzr2l5o/Lzh4GfOYEH+mnf7ZrEGDzzJEP6ta8=
-go.opentelemetry.io/otel/sdk v0.15.0 h1:Hf2dl1Ad9Hn03qjcAuAq51GP5Pv1SV5puIkS2nRhdd8=
-go.opentelemetry.io/otel/sdk v0.15.0/go.mod h1:Qudkwgq81OcA9GYVlbyZ62wkLieeS1eWxIL0ufxgwoc=
+go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.0.0 h1:qTTn6x71GVBvoafHK/yaRUmFzI4LcONZD0/kXxl5PHI=
+go.opentelemetry.io/otel v1.0.0/go.mod h1:AjRVh9A5/5DE7S+mZtTR6t8vpKKryam+0lREnfmS4cg=
+go.opentelemetry.io/otel/exporters/jaeger v1.0.0 h1:cLhx8llHw02h5JTqGqaRbYn+QVKHmrzD9vEbKnSPk5U=
+go.opentelemetry.io/otel/exporters/jaeger v1.0.0/go.mod h1:q10N1AolE1JjqKrFJK2tYw0iZpmX+HBaXBtuCzRnBGQ=
+go.opentelemetry.io/otel/sdk v1.0.0 h1:BNPMYUONPNbLneMttKSjQhOTlFLOD9U22HNG1KrIN2Y=
+go.opentelemetry.io/otel/sdk v1.0.0/go.mod h1:PCrDHlSy5x1kjezSdL37PhbFUMjrsLRshJ2zCzeXwbM=
+go.opentelemetry.io/otel/trace v1.0.0 h1:TSBr8GTEtKevYMG/2d21M989r5WJYVimhTHBKVEZuh4=
+go.opentelemetry.io/otel/trace v1.0.0/go.mod h1:PXTWqayeFUlJV1YDNhsJYB184+IvAH814St6o6ajzIs=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
@@ -651,8 +659,8 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 h1:ld7aEMNHoBnnDAX15v1T6z31v8HwR2A9FYOuAhWqkwc=
-golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93 h1:alLDrZkL34Y2bnGHfvC1CYBRBXCXgx8AC2vY4MRtYX4=
+golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -662,7 +670,6 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -710,7 +717,6 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -721,6 +727,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 h1:dXfMednGJh/SUUFjTLsWJz3P+TQt9qnR11GgeI3vWKs=
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -729,8 +736,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -781,7 +789,6 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -804,15 +811,14 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
 google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
 google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/api v0.32.0 h1:Le77IccnTqEa8ryp9wIpX5W3zYm7Gf9LhOp9PHcwFts=
-google.golang.org/api v0.32.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
@@ -828,9 +834,9 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa
 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
 google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.33.2 h1:EQyQC3sa8M+p6Ulc8yy9SWSS2GVwyRc83gAbG8lrl4o=
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.36.0 h1:o1bcQ6imQMIOpdrO3SWf2z5RV72WbDwdXuK0MDlc8As=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -840,8 +846,9 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

src/runtime/pkg/containerd-shim-v2/create.go

@@ -10,9 +10,6 @@ package containerdshim
 import (
 	"context"
 	"fmt"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/rootless"
-	"math/rand"
 	"os"
 	"os/user"
 	"path"
@@ -24,6 +21,8 @@ import (
 	"github.com/containerd/containerd/mount"
 	taskAPI "github.com/containerd/containerd/runtime/v2/task"
 	"github.com/containerd/typeurl"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/rootless"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 
@@ -40,6 +39,13 @@ import (
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 )
 
+type startManagementServerFunc func(s *service, ctx context.Context, ociSpec *specs.Spec)
+
+var defaultStartManagementServerFunc startManagementServerFunc = func(s *service, ctx context.Context, ociSpec *specs.Spec) {
+	go s.startManagementServer(ctx, ociSpec)
+	shimLog.Info("management server started")
+}
+
 func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*container, error) {
 	rootFs := vc.RootFs{}
 	if len(r.Rootfs) == 1 {
@@ -132,7 +138,9 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con
 		}
 		s.hpid = uint32(pid)
 
-		go s.startManagementServer(ctx, ociSpec)
+		if defaultStartManagementServerFunc != nil {
+			defaultStartManagementServerFunc(s, ctx, ociSpec)
+		}
 
 	case vc.PodContainer:
 		span, ctx := katatrace.Trace(s.ctx, shimLog, "create", shimTracingTags)
@@ -274,13 +282,15 @@ func doMount(mounts []*containerd_types.Mount, rootfs string) error {
 }
 
 func configureNonRootHypervisor(runtimeConfig *oci.RuntimeConfig) error {
-	userName, err := createVmmUser()
+	userName, err := utils.CreateVmmUser()
 	if err != nil {
 		return err
 	}
 	defer func() {
 		if err != nil {
-			removeVmmUser(userName)
+			if err2 := utils.RemoveVmmUser(userName); err2 != nil {
+				shimLog.WithField("userName", userName).WithError(err).Warn("failed to remove user")
+			}
 		}
 	}()
 
@@ -301,24 +311,26 @@ func configureNonRootHypervisor(runtimeConfig *oci.RuntimeConfig) error {
 	runtimeConfig.HypervisorConfig.Gid = uint32(gid)
 
 	userTmpDir := path.Join("/run/user/", fmt.Sprint(uid))
-	dir, err := os.Stat(userTmpDir)
-	if os.IsNotExist(err) {
-		if err = os.Mkdir(userTmpDir, vc.DirMode); err != nil {
-			return err
-		}
-		defer func() {
-			if err != nil {
-				if err = os.RemoveAll(userTmpDir); err != nil {
-					shimLog.WithField("userTmpDir", userTmpDir).WithError(err).Warn("failed to remove userTmpDir")
-				}
-			}
-		}()
-		if err = syscall.Chown(userTmpDir, uid, gid); err != nil {
+	_, err = os.Stat(userTmpDir)
+	// Clean up the directory created by the previous run
+	if !os.IsNotExist(err) {
+		if err = os.RemoveAll(userTmpDir); err != nil {
 			return err
 		}
 	}
-	if dir != nil && !dir.IsDir() {
-		return fmt.Errorf("%s is expected to be a directory", userTmpDir)
+
+	if err = os.Mkdir(userTmpDir, vc.DirMode); err != nil {
+		return err
+	}
+	defer func() {
+		if err != nil {
+			if err = os.RemoveAll(userTmpDir); err != nil {
+				shimLog.WithField("userTmpDir", userTmpDir).WithError(err).Warn("failed to remove userTmpDir")
+			}
+		}
+	}()
+	if err = syscall.Chown(userTmpDir, uid, gid); err != nil {
+		return err
 	}
 
 	if err := os.Setenv("XDG_RUNTIME_DIR", userTmpDir); err != nil {
@@ -336,48 +348,3 @@ func configureNonRootHypervisor(runtimeConfig *oci.RuntimeConfig) error {
 	}
 	return fmt.Errorf("failed to get the gid of /dev/kvm")
 }
-
-func createVmmUser() (string, error) {
-	var (
-		err      error
-		userName string
-	)
-
-	useraddPath, err := utils.FirstValidExecutable([]string{"/usr/sbin/useradd", "/sbin/useradd", "/bin/useradd"})
-	if err != nil {
-		return "", err
-	}
-	nologinPath, err := utils.FirstValidExecutable([]string{"/usr/sbin/nologin", "/sbin/nologin", "/bin/nologin"})
-	if err != nil {
-		return "", err
-	}
-
-	// Add retries to mitigate temporary errors and race conditions. For example, the user already exists
-	// or another instance of the runtime is also creating a user.
-	maxAttempt := 5
-	for i := 0; i < maxAttempt; i++ {
-		userName = fmt.Sprintf("kata-%v", rand.Intn(100000))
-		_, err = utils.RunCommand([]string{useraddPath, "-M", "-s", nologinPath, userName, "-c", "\"Kata Containers temporary hypervisor user\""})
-		if err == nil {
-			return userName, nil
-		}
-		shimLog.WithField("attempt", i+1).WithField("username", userName).
-			WithError(err).Warn("failed to add user, will try again")
-	}
-	return "", fmt.Errorf("could not create VMM user: %v", err)
-}
-
-func removeVmmUser(user string) {
-	userdelPath, err := utils.FirstValidExecutable([]string{"/usr/sbin/userdel", "/sbin/userdel", "/bin/userdel"})
-	if err != nil {
-		shimLog.WithField("username", user).WithError(err).Warn("failed to remove user")
-	}
-	// Add retries to mitigate temporary errors and race conditions.
-	for i := 0; i < 5; i++ {
-		_, err := utils.RunCommand([]string{userdelPath, "-f", user})
-		if err == nil {
-			return
-		}
-		shimLog.WithField("username", user).WithField("attempt", i+1).WithError(err).Warn("failed to remove user")
-	}
-}

src/runtime/pkg/containerd-shim-v2/create_test.go

@@ -12,7 +12,6 @@ import (
 	"io/ioutil"
 	"os"
 	"path"
-	"path/filepath"
 	"testing"
 
 	"github.com/containerd/containerd/namespaces"
@@ -23,7 +22,6 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
@@ -52,21 +50,12 @@ func TestCreateSandboxSuccess(t *testing.T) {
 		testingImpl.CreateSandboxFunc = nil
 	}()
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
-	defer os.RemoveAll(tmpdir)
+	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
+	// defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(katautils.FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -82,7 +71,7 @@ func TestCreateSandboxSuccess(t *testing.T) {
 	}
 
 	// Rewrite the file
-	err = writeOCIConfigFile(spec, ociConfigFile)
+	err = ktu.WriteOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
 	s := &service{
@@ -110,25 +99,16 @@ func TestCreateSandboxFail(t *testing.T) {
 
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(katautils.FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
-	err = writeOCIConfigFile(spec, ociConfigFile)
+	err = ktu.WriteOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
 	s := &service{
@@ -157,21 +137,12 @@ func TestCreateSandboxConfigFail(t *testing.T) {
 
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, _ := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(katautils.FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -216,21 +187,12 @@ func TestCreateContainerSuccess(t *testing.T) {
 		},
 	}
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(katautils.FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -240,7 +202,7 @@ func TestCreateContainerSuccess(t *testing.T) {
 	spec.Annotations[testSandboxIDAnnotation] = testSandboxID
 
 	// rewrite file
-	err = writeOCIConfigFile(spec, ociConfigFile)
+	err = ktu.WriteOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
 	s := &service{
@@ -265,21 +227,12 @@ func TestCreateContainerSuccess(t *testing.T) {
 func TestCreateContainerFail(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(katautils.FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -287,7 +240,7 @@ func TestCreateContainerFail(t *testing.T) {
 	spec.Annotations[testContainerTypeAnnotation] = testContainerTypeContainer
 	spec.Annotations[testSandboxIDAnnotation] = testSandboxID
 
-	err = writeOCIConfigFile(spec, ociConfigFile)
+	err = ktu.WriteOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
 	// doesn't create sandbox first
@@ -325,21 +278,12 @@ func TestCreateContainerConfigFail(t *testing.T) {
 		sandbox.CreateContainerFunc = nil
 	}()
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(katautils.FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -348,7 +292,7 @@ func TestCreateContainerConfigFail(t *testing.T) {
 	spec.Annotations[testContainerTypeAnnotation] = "errorType"
 	spec.Annotations[testSandboxIDAnnotation] = testSandboxID
 
-	err = writeOCIConfigFile(spec, ociConfigFile)
+	err = ktu.WriteOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
 	s := &service{

src/runtime/pkg/containerd-shim-v2/delete_test.go

@@ -7,14 +7,13 @@
 package containerdshim
 
 import (
-	"io/ioutil"
 	"os"
-	"path/filepath"
 	"testing"
 
 	taskAPI "github.com/containerd/containerd/runtime/v2/task"
 	"github.com/stretchr/testify/assert"
 
+	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/vcmock"
 )
@@ -26,7 +25,7 @@ func TestDeleteContainerSuccessAndFail(t *testing.T) {
 		MockID: testSandboxID,
 	}
 
-	rootPath, bundlePath := testConfigSetup(t)
+	rootPath, bundlePath, _ := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(rootPath)
 	_, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
@@ -43,19 +42,3 @@ func TestDeleteContainerSuccessAndFail(t *testing.T) {
 	s.containers[testContainerID], err = newContainer(s, reqCreate, "", nil, true)
 	assert.NoError(err)
 }
-
-func testConfigSetup(t *testing.T) (rootPath string, bundlePath string) {
-	assert := assert.New(t)
-
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
-
-	bundlePath = filepath.Join(tmpdir, "bundle")
-	err = os.MkdirAll(bundlePath, testDirMode)
-	assert.NoError(err)
-
-	err = createOCIConfig(bundlePath)
-	assert.NoError(err)
-
-	return tmpdir, bundlePath
-}

src/runtime/pkg/containerd-shim-v2/service.go

@@ -935,6 +935,10 @@ func (s *service) Shutdown(ctx context.Context, r *taskAPI.ShutdownRequest) (_ *
 	s.mu.Lock()
 	if len(s.containers) != 0 {
 		s.mu.Unlock()
+
+		span.End()
+		katatrace.StopTracing(s.rootCtx)
+
 		return empty, nil
 	}
 	s.mu.Unlock()

src/runtime/pkg/containerd-shim-v2/service_test.go

@@ -8,9 +8,7 @@ package containerdshim
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"os"
-	"path/filepath"
 	"strings"
 	"testing"
 
@@ -43,13 +41,9 @@ func TestServiceCreate(t *testing.T) {
 
 	assert := assert.New(t)
 
-	tmpdir, _ := ioutil.TempDir("", "")
+	tmpdir, bundleDir, _ := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
-	bundleDir := filepath.Join(tmpdir, "bundle")
-	err := makeOCIBundle(bundleDir)
-	assert.NoError(err)
-
 	ctx := context.Background()
 
 	s, err := newService("foo")

src/runtime/pkg/containerd-shim-v2/shim_management.go

@@ -183,8 +183,13 @@ func (s *service) mountPprofHandle(m *http.ServeMux, ociSpec *specs.Spec) {
 	m.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
 }
 
-// SocketAddress returns the address of the abstract domain socket for communicating with the
+// GetSandboxesStoragePath returns the storage path where sandboxes info are stored
+func GetSandboxesStoragePath() string {
+	return "/run/vc/sbs"
+}
+
+// SocketAddress returns the address of the unix domain socket for communicating with the
 // shim management endpoint
 func SocketAddress(id string) string {
-	return fmt.Sprintf("unix://%s", filepath.Join(string(filepath.Separator), "run", "vc", "sbs", id, "shim-monitor.sock"))
+	return fmt.Sprintf("unix://%s", filepath.Join(string(filepath.Separator), GetSandboxesStoragePath(), id, "shim-monitor.sock"))
 }

src/runtime/pkg/containerd-shim-v2/stream_test.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"io"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 	"syscall"
 	"testing"
@@ -24,6 +25,11 @@ func TestNewTtyIOFifoReopen(t *testing.T) {
 	var tty *ttyIO
 	assert := assert.New(t)
 	ctx := context.TODO()
+
+	testDir, err := ioutil.TempDir("", "kata-")
+	assert.NoError(err)
+	defer os.RemoveAll(testDir)
+
 	fifoPath, err := ioutil.TempDir(testDir, "fifo-path-")
 	assert.NoError(err)
 	stdout := filepath.Join(fifoPath, "stdout")
@@ -91,8 +97,6 @@ func TestNewTtyIOFifoReopen(t *testing.T) {
 }
 
 func TestIoCopy(t *testing.T) {
-	t.Skip("TestIoCopy is failing randonly, see https://github.com/kata-containers/kata-containers/issues/2042")
-
 	assert := assert.New(t)
 	ctx := context.TODO()
 
@@ -100,6 +104,10 @@ func TestIoCopy(t *testing.T) {
 	testBytes2 := []byte("Test2")
 	testBytes3 := []byte("Test3")
 
+	testDir, err := ioutil.TempDir("", "kata-")
+	assert.NoError(err)
+	defer os.RemoveAll(testDir)
+
 	fifoPath, err := ioutil.TempDir(testDir, "fifo-path-")
 	assert.NoError(err)
 	dstStdoutPath := filepath.Join(fifoPath, "dststdout")
@@ -229,6 +237,18 @@ func TestIoCopy(t *testing.T) {
 			}
 		}
 
+		// check everything works without closed pipes
+		checkFifoWrite(firstW, testBytes1, first)
+		checkFifoRead(firstR, testBytes1, first)
+
+		checkFifoWrite(secondW, testBytes2, second)
+		checkFifoRead(secondR, testBytes2, second)
+
+		if thirdW != nil {
+			checkFifoWrite(thirdW, testBytes3, third)
+			checkFifoRead(thirdR, testBytes3, third)
+		}
+
 		// write to each pipe, and close them immediately
 		// the ioCopy function should copy the data, then stop the corresponding thread
 		checkFifoWrite(firstW, testBytes1, first)

src/runtime/pkg/containerd-shim-v2/utils.go

@@ -112,13 +112,5 @@ func getAddress(ctx context.Context, bundlePath, address, id string) (string, er
 }
 
 func noNeedForOutput(detach bool, tty bool) bool {
-	if !detach {
-		return false
-	}
-
-	if !tty {
-		return false
-	}
-
-	return true
+	return detach && tty
 }

src/runtime/pkg/containerd-shim-v2/utils_test.go

@@ -7,43 +7,28 @@
 package containerdshim
 
 import (
-	"encoding/json"
-	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
-	sysExec "os/exec"
 	"path"
-	"path/filepath"
-	"strings"
+	"testing"
 
-	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
 
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/vcmock"
+	"github.com/pkg/errors"
 )
 
 const (
-	// specConf is the name of the file holding the containers configuration
-	specConf = "config.json"
-
 	TestID = "container_test"
 
-	testDirMode  = os.FileMode(0750)
 	testFileMode = os.FileMode(0640)
-	// testExeFileMode         = os.FileMode(0750)
-
-	// small docker image used to create root filesystems from
-	testDockerImage = "busybox"
 
 	testSandboxID   = "777-77-77777777"
 	testContainerID = "42"
-	testBundle      = "bundle"
 	testConsole     = "/dev/pts/888"
 
 	testContainerTypeAnnotation = "io.kubernetes.cri.container-type"
@@ -54,10 +39,7 @@ const (
 
 var (
 	// package variables set by calling TestMain()
-	testDir       = ""
-	testBundleDir = ""
-	tc            ktu.TestConstraint
-	ctrEngine     = katautils.CtrEngine{}
+	tc ktu.TestConstraint
 )
 
 // testingImpl is a concrete mock RVC implementation used for testing
@@ -70,98 +52,11 @@ func init() {
 	fmt.Printf("INFO: switching to fake virtcontainers implementation for testing\n")
 	vci = testingImpl
 
-	var err error
-
-	fmt.Printf("INFO: creating test directory\n")
-	testDir, err = ioutil.TempDir("", "shimV2-")
-	if err != nil {
-		panic(fmt.Sprintf("ERROR: failed to create test directory: %v", err))
-	}
-	fmt.Printf("INFO: test directory is %v\n", testDir)
-
-	var output string
-	for _, name := range katautils.DockerLikeCtrEngines {
-		fmt.Printf("INFO: checking for container engine: %s\n", name)
-
-		output, err = ctrEngine.Init(name)
-		if err == nil {
-			break
-		}
-	}
-
-	if ctrEngine.Name == "" {
-		panic(fmt.Sprintf("ERROR: Docker-like container engine not accessible to current user: %v (error %v)",
-			output, err))
-	}
-
-	// Do this now to avoid hitting the test timeout value due to
-	// slow network response.
-	fmt.Printf("INFO: ensuring required container image (%v) is available\n", testDockerImage)
-	// Only hit the network if the image doesn't exist locally
-	_, err = ctrEngine.Inspect(testDockerImage)
-	if err == nil {
-		fmt.Printf("INFO: container image %v already exists locally\n", testDockerImage)
-	} else {
-		fmt.Printf("INFO: pulling container image %v\n", testDockerImage)
-		_, err = ctrEngine.Pull(testDockerImage)
-		if err != nil {
-			panic(err)
-		}
-	}
-
-	testBundleDir = filepath.Join(testDir, testBundle)
-	err = os.MkdirAll(testBundleDir, testDirMode)
-	if err != nil {
-		panic(fmt.Sprintf("ERROR: failed to create bundle directory %v: %v", testBundleDir, err))
-	}
-
-	fmt.Printf("INFO: creating OCI bundle in %v for tests to use\n", testBundleDir)
-	err = realMakeOCIBundle(testBundleDir)
-	if err != nil {
-		panic(fmt.Sprintf("ERROR: failed to create OCI bundle: %v", err))
-	}
-
 	tc = ktu.NewTestConstraint(false)
-}
 
-// createOCIConfig creates an OCI configuration (spec) file in
-// the bundle directory specified (which must exist).
-func createOCIConfig(bundleDir string) error {
-	if bundleDir == "" {
-		return errors.New("BUG: Need bundle directory")
-	}
-
-	if !katautils.FileExists(bundleDir) {
-		return fmt.Errorf("BUG: Bundle directory %s does not exist", bundleDir)
-	}
-
-	var configCmd string
-
-	// Search for a suitable version of runc to use to generate
-	// the OCI config file.
-	for _, cmd := range []string{"docker-runc", "runc"} {
-		fullPath, err := sysExec.LookPath(cmd)
-		if err == nil {
-			configCmd = fullPath
-			break
-		}
-	}
-
-	if configCmd == "" {
-		return errors.New("Cannot find command to generate OCI config file")
-	}
-
-	_, err := utils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
-	if err != nil {
-		return err
-	}
-
-	specFile := filepath.Join(bundleDir, specConf)
-	if !katautils.FileExists(specFile) {
-		return fmt.Errorf("generated OCI config file does not exist: %v", specFile)
-	}
-
-	return nil
+	// disable shim management server.
+	// all tests are not using this, so just set it to nil
+	defaultStartManagementServerFunc = nil
 }
 
 func createEmptyFile(path string) (err error) {
@@ -214,115 +109,38 @@ func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConf
 	}, nil
 }
 
-// realMakeOCIBundle will create an OCI bundle (including the "config.json"
-// config file) in the directory specified (which must already exist).
-//
-// XXX: Note that tests should *NOT* call this function - they should
-// XXX: instead call makeOCIBundle().
-func realMakeOCIBundle(bundleDir string) error {
-	if bundleDir == "" {
-		return errors.New("BUG: Need bundle directory")
+func TestNoNeedForOutput(t *testing.T) {
+	assert := assert.New(t)
+
+	testCases := []struct {
+		detach bool
+		tty    bool
+		result bool
+	}{
+		{
+			detach: true,
+			tty:    true,
+			result: true,
+		},
+		{
+			detach: false,
+			tty:    true,
+			result: false,
+		},
+		{
+			detach: true,
+			tty:    false,
+			result: false,
+		},
+		{
+			detach: false,
+			tty:    false,
+			result: false,
+		},
 	}
 
-	if !katautils.FileExists(bundleDir) {
-		return fmt.Errorf("BUG: Bundle directory %v does not exist", bundleDir)
+	for i := range testCases {
+		result := noNeedForOutput(testCases[i].detach, testCases[i].tty)
+		assert.Equal(testCases[i].result, result)
 	}
-
-	err := createOCIConfig(bundleDir)
-	if err != nil {
-		return err
-	}
-
-	// Note the unusual parameter (a directory, not the config
-	// file to parse!)
-	spec, err := compatoci.ParseConfigJSON(bundleDir)
-	if err != nil {
-		return err
-	}
-
-	// Determine the rootfs directory name the OCI config refers to
-	ociRootPath := spec.Root.Path
-
-	rootfsDir := filepath.Join(bundleDir, ociRootPath)
-
-	if strings.HasPrefix(ociRootPath, "/") {
-		return fmt.Errorf("Cannot handle absolute rootfs as bundle must be unique to each test")
-	}
-
-	err = createRootfs(rootfsDir)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// Create an OCI bundle in the specified directory.
-//
-// Note that the directory will be created, but it's parent is expected to exist.
-//
-// This function works by copying the already-created test bundle. Ideally,
-// the bundle would be recreated for each test, but createRootfs() uses
-// docker which on some systems is too slow, resulting in the tests timing
-// out.
-func makeOCIBundle(bundleDir string) error {
-	from := testBundleDir
-	to := bundleDir
-
-	// only the basename of bundleDir needs to exist as bundleDir
-	// will get created by cp(1).
-	base := filepath.Dir(bundleDir)
-
-	for _, dir := range []string{from, base} {
-		if !katautils.FileExists(dir) {
-			return fmt.Errorf("BUG: directory %v should exist", dir)
-		}
-	}
-
-	output, err := utils.RunCommandFull([]string{"cp", "-a", from, to}, true)
-	if err != nil {
-		return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output)
-	}
-
-	return nil
-}
-
-// createRootfs creates a minimal root filesystem below the specified
-// directory.
-func createRootfs(dir string) error {
-	err := os.MkdirAll(dir, testDirMode)
-	if err != nil {
-		return err
-	}
-
-	container, err := ctrEngine.Create(testDockerImage)
-	if err != nil {
-		return err
-	}
-
-	err = ctrEngine.GetRootfs(container, dir)
-	if err != nil {
-		return err
-	}
-
-	// Clean up
-	_, err = ctrEngine.Rm(container)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func writeOCIConfigFile(spec specs.Spec, configPath string) error {
-	if configPath == "" {
-		return errors.New("BUG: need config file path")
-	}
-
-	bytes, err := json.MarshalIndent(spec, "", "\t")
-	if err != nil {
-		return err
-	}
-
-	return ioutil.WriteFile(configPath, bytes, testFileMode)
 }

src/runtime/pkg/kata-monitor/cri.go

@@ -8,15 +8,12 @@ package katamonitor
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"net"
 	"net/url"
-	"strings"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
-	"github.com/xeipuuv/gojsonpointer"
 	"google.golang.org/grpc"
 
 	pb "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
@@ -117,14 +114,12 @@ func parseEndpoint(endpoint string) (string, string, error) {
 	}
 }
 
-// getSandboxes get kata sandbox from the container engine.
-// this will be called only after monitor start.
-func (km *KataMonitor) getSandboxes() (map[string]struct{}, error) {
-
-	sandboxMap := make(map[string]struct{})
+// getSandboxes gets ready sandboxes from the container engine and returns an updated sandboxMap
+func (km *KataMonitor) getSandboxes(sandboxMap map[string]bool) (map[string]bool, error) {
+	newMap := make(map[string]bool)
 	runtimeClient, runtimeConn, err := getRuntimeClient(km.runtimeEndpoint)
 	if err != nil {
-		return sandboxMap, err
+		return newMap, err
 	}
 	defer closeConnection(runtimeConn)
 
@@ -140,61 +135,26 @@ func (km *KataMonitor) getSandboxes() (map[string]struct{}, error) {
 	monitorLog.Debugf("ListPodSandboxRequest: %v", request)
 	r, err := runtimeClient.ListPodSandbox(context.Background(), request)
 	if err != nil {
-		return sandboxMap, err
+		return newMap, err
 	}
 	monitorLog.Debugf("ListPodSandboxResponse: %v", r)
 
 	for _, pod := range r.Items {
-		request := &pb.PodSandboxStatusRequest{
-			PodSandboxId: pod.Id,
-			Verbose:      true,
-		}
-
-		r, err := runtimeClient.PodSandboxStatus(context.Background(), request)
-		if err != nil {
-			return sandboxMap, err
-		}
-
-		lowRuntime := ""
-		var res map[string]interface{}
-		if err := json.Unmarshal([]byte(r.Info["info"]), &res); err != nil {
-			monitorLog.WithError(err).WithField("pod", r).Error("failed to Unmarshal pod info")
-			continue
-		} else {
-			monitorLog.WithField("pod info", res).Debug("")
-
-			// get low level container runtime
-			// containerd stores the pod runtime in "/runtimeType" while CRI-O stores it the
-			// io.kubernetes.cri-o.RuntimeHandler annotation: check for both.
-			keys := []string{"/runtimeType", "/runtimeSpec/annotations/io.kubernetes.cri-o.RuntimeHandler"}
-			for _, key := range keys {
-				pointer, _ := gojsonpointer.NewJsonPointer(key)
-				rt, _, _ := pointer.Get(res)
-				if rt != nil {
-					if str, ok := rt.(string); ok {
-						lowRuntime = str
-						break
-					}
-				}
-			}
-		}
-
-		// If lowRuntime is empty something changed in containerd/CRI-O or we are dealing with an unknown container engine.
-		// Safest options is to add the POD in the list: we will be able to connect to the shim to retrieve the actual info
-		// only for kata PODs.
-		if lowRuntime == "" {
-			monitorLog.WithField("pod", r).Info("unable to retrieve the runtime type")
-			sandboxMap[pod.Id] = struct{}{}
+		// Use the cached data if available
+		if isKata, ok := sandboxMap[pod.Id]; ok {
+			newMap[pod.Id] = isKata
 			continue
 		}
 
+		// Check if a directory associated with the POD ID exist on the kata fs:
+		// if so we know that the POD is a kata one.
+		newMap[pod.Id] = checkSandboxFSExists(pod.Id)
 		monitorLog.WithFields(logrus.Fields{
-			"low runtime": lowRuntime,
+			"id":      pod.Id,
+			"is kata": newMap[pod.Id],
+			"pod":     pod,
 		}).Debug("")
-		if strings.Contains(lowRuntime, "kata") {
-			sandboxMap[pod.Id] = struct{}{}
-		}
 	}
 
-	return sandboxMap, nil
+	return newMap, nil
 }

src/runtime/pkg/kata-monitor/metrics.go

@@ -140,8 +140,8 @@ func encodeMetricFamily(mfs []*dto.MetricFamily, encoder expfmt.Encoder) error {
 
 // aggregateSandboxMetrics will get metrics from one sandbox and do some process
 func (km *KataMonitor) aggregateSandboxMetrics(encoder expfmt.Encoder) error {
-	// get all sandboxes from cache
-	sandboxes := km.sandboxCache.getAllSandboxes()
+	// get all kata sandboxes from cache
+	sandboxes := km.sandboxCache.getKataSandboxes()
 	// save running kata pods as a metrics.
 	runningShimCount.Set(float64(len(sandboxes)))
 
@@ -159,7 +159,7 @@ func (km *KataMonitor) aggregateSandboxMetrics(encoder expfmt.Encoder) error {
 	monitorLog.WithField("sandbox_count", len(sandboxes)).Debugf("sandboxes count")
 
 	// get metrics from sandbox's shim
-	for sandboxID := range sandboxes {
+	for _, sandboxID := range sandboxes {
 		wg.Add(1)
 		go func(sandboxID string, results chan<- []*dto.MetricFamily) {
 			sandboxMetrics, err := getParsedMetrics(sandboxID)

src/runtime/pkg/kata-monitor/monitor.go

@@ -9,19 +9,22 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"os"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/fsnotify/fsnotify"
 	"github.com/sirupsen/logrus"
 )
 
 var monitorLog = logrus.WithField("source", "kata-monitor")
 
 const (
-	RuntimeContainerd          = "containerd"
-	RuntimeCRIO                = "cri-o"
-	podCacheRefreshTimeSeconds = 15
+	RuntimeContainerd           = "containerd"
+	RuntimeCRIO                 = "cri-o"
+	fsMonitorRetryDelaySeconds  = 60
+	podCacheRefreshDelaySeconds = 5
 )
 
 // SetLogger sets the logger for katamonitor package.
@@ -50,7 +53,7 @@ func NewKataMonitor(runtimeEndpoint string) (*KataMonitor, error) {
 		runtimeEndpoint: runtimeEndpoint,
 		sandboxCache: &sandboxCache{
 			Mutex:     &sync.Mutex{},
-			sandboxes: make(map[string]struct{}),
+			sandboxes: make(map[string]bool),
 		},
 	}
 
@@ -64,15 +67,74 @@ func NewKataMonitor(runtimeEndpoint string) (*KataMonitor, error) {
 
 // startPodCacheUpdater will boot a thread to manage sandbox cache
 func (km *KataMonitor) startPodCacheUpdater() {
+	sbsWatcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		monitorLog.WithError(err).Fatal("failed to setup sandbox events watcher")
+		os.Exit(1)
+	}
+	defer sbsWatcher.Close()
 	for {
-		time.Sleep(podCacheRefreshTimeSeconds * time.Second)
-		sandboxes, err := km.getSandboxes()
+		err = sbsWatcher.Add(getSandboxFS())
 		if err != nil {
-			monitorLog.WithError(err).Error("failed to get sandboxes")
+			// if there are no kata pods (yet), the kata /run/vc directory may not be there: retry later
+			monitorLog.WithError(err).Warnf("cannot monitor %s, retry in %d sec.", getSandboxFS(), fsMonitorRetryDelaySeconds)
+			time.Sleep(fsMonitorRetryDelaySeconds * time.Second)
 			continue
 		}
-		monitorLog.WithField("count", len(sandboxes)).Debug("update sandboxes list")
-		km.sandboxCache.set(sandboxes)
+		monitorLog.Debugf("started fs monitoring @%s", getSandboxFS())
+		break
+	}
+	// we refresh the pod cache once if we get multiple add/delete pod events in a short time (< podCacheRefreshDelaySeconds)
+	cacheUpdateTimer := time.NewTimer(podCacheRefreshDelaySeconds * time.Second)
+	cacheUpdateTimerWasSet := false
+	for {
+		select {
+		case event, ok := <-sbsWatcher.Events:
+			if !ok {
+				monitorLog.WithError(err).Fatal("cannot watch sandboxes fs")
+				os.Exit(1)
+			}
+			monitorLog.WithField("event", event).Debug("got sandbox event")
+			switch event.Op {
+			case fsnotify.Create:
+				splitPath := strings.Split(event.Name, string(os.PathSeparator))
+				id := splitPath[len(splitPath)-1]
+				if !km.sandboxCache.putIfNotExists(id, true) {
+					monitorLog.WithField("pod", id).Warn(
+						"CREATE event but pod already present in the sandbox cache")
+				}
+				monitorLog.WithField("pod", id).Info("sandbox cache: added pod")
+
+			case fsnotify.Remove:
+				splitPath := strings.Split(event.Name, string(os.PathSeparator))
+				id := splitPath[len(splitPath)-1]
+				if !km.sandboxCache.deleteIfExists(id) {
+					monitorLog.WithField("pod", id).Warn(
+						"REMOVE event but pod was missing from the sandbox cache")
+				}
+				monitorLog.WithField("pod", id).Info("sandbox cache: removed pod")
+
+			default:
+				monitorLog.WithField("event", event).Warn("got unexpected fs event")
+			}
+
+			// While we process fs events directly to update the sandbox cache we need to sync with the
+			// container engine to ensure we are on sync with it: we can get out of sync in environments
+			// where kata workloads can be started by other processes than the container engine.
+			cacheUpdateTimerWasSet = cacheUpdateTimer.Reset(podCacheRefreshDelaySeconds * time.Second)
+			monitorLog.WithField("was reset", cacheUpdateTimerWasSet).Debugf(
+				"cache update timer fires in %d secs", podCacheRefreshDelaySeconds)
+
+		case <-cacheUpdateTimer.C:
+			sandboxes, err := km.getSandboxes(km.sandboxCache.getAllSandboxes())
+			if err != nil {
+				monitorLog.WithError(err).Error("failed to get sandboxes")
+				continue
+			}
+			monitorLog.WithField("count", len(sandboxes)).Info("synced sandbox cache with the container engine")
+			monitorLog.WithField("sandboxes", sandboxes).Debug("dump sandbox cache")
+			km.sandboxCache.set(sandboxes)
+		}
 	}
 }
 
@@ -95,20 +157,8 @@ func (km *KataMonitor) GetAgentURL(w http.ResponseWriter, r *http.Request) {
 
 // ListSandboxes list all sandboxes running in Kata
 func (km *KataMonitor) ListSandboxes(w http.ResponseWriter, r *http.Request) {
-	sandboxes := km.getSandboxList()
+	sandboxes := km.sandboxCache.getKataSandboxes()
 	for _, s := range sandboxes {
 		w.Write([]byte(fmt.Sprintf("%s\n", s)))
 	}
 }
-
-func (km *KataMonitor) getSandboxList() []string {
-	sn := km.sandboxCache.getAllSandboxes()
-	result := make([]string, len(sn))
-
-	i := 0
-	for k := range sn {
-		result[i] = k
-		i++
-	}
-	return result
-}

src/runtime/pkg/kata-monitor/sandbox_cache.go

@@ -11,15 +11,28 @@ import (
 
 type sandboxCache struct {
 	*sync.Mutex
-	sandboxes map[string]struct{}
+	// the bool value tracks if the pod is a kata one (true) or not (false)
+	sandboxes map[string]bool
 }
 
-func (sc *sandboxCache) getAllSandboxes() map[string]struct{} {
+func (sc *sandboxCache) getAllSandboxes() map[string]bool {
 	sc.Lock()
 	defer sc.Unlock()
 	return sc.sandboxes
 }
 
+func (sc *sandboxCache) getKataSandboxes() []string {
+	sc.Lock()
+	defer sc.Unlock()
+	var katasandboxes []string
+	for id, isKata := range sc.sandboxes {
+		if isKata {
+			katasandboxes = append(katasandboxes, id)
+		}
+	}
+	return katasandboxes
+}
+
 func (sc *sandboxCache) deleteIfExists(id string) bool {
 	sc.Lock()
 	defer sc.Unlock()
@@ -33,12 +46,12 @@ func (sc *sandboxCache) deleteIfExists(id string) bool {
 	return false
 }
 
-func (sc *sandboxCache) putIfNotExists(id string) bool {
+func (sc *sandboxCache) putIfNotExists(id string, value bool) bool {
 	sc.Lock()
 	defer sc.Unlock()
 
 	if _, found := sc.sandboxes[id]; !found {
-		sc.sandboxes[id] = struct{}{}
+		sc.sandboxes[id] = value
 		return true
 	}
 
@@ -46,7 +59,7 @@ func (sc *sandboxCache) putIfNotExists(id string) bool {
 	return false
 }
 
-func (sc *sandboxCache) set(sandboxes map[string]struct{}) {
+func (sc *sandboxCache) set(sandboxes map[string]bool) {
 	sc.Lock()
 	defer sc.Unlock()
 	sc.sandboxes = sandboxes

src/runtime/pkg/kata-monitor/sandbox_cache_test.go

@@ -16,10 +16,10 @@ func TestSandboxCache(t *testing.T) {
 	assert := assert.New(t)
 	sc := &sandboxCache{
 		Mutex:     &sync.Mutex{},
-		sandboxes: make(map[string]struct{}),
+		sandboxes: make(map[string]bool),
 	}
 
-	scMap := map[string]struct{}{"111": {}}
+	scMap := map[string]bool{"111": true}
 
 	sc.set(scMap)
 
@@ -28,12 +28,12 @@ func TestSandboxCache(t *testing.T) {
 
 	// put new item
 	id := "new-id"
-	b := sc.putIfNotExists(id)
+	b := sc.putIfNotExists(id, true)
 	assert.Equal(true, b)
 	assert.Equal(2, len(scMap))
 
 	// put key that alreay exists
-	b = sc.putIfNotExists(id)
+	b = sc.putIfNotExists(id, true)
 	assert.Equal(false, b)
 
 	b = sc.deleteIfExists(id)

src/runtime/pkg/kata-monitor/shim_client.go

@@ -10,6 +10,8 @@ import (
 	"io/ioutil"
 	"net"
 	"net/http"
+	"os"
+	"path/filepath"
 	"time"
 
 	cdshim "github.com/containerd/containerd/runtime/v2/shim"
@@ -37,6 +39,17 @@ func getSandboxIDFromReq(r *http.Request) (string, error) {
 	return "", fmt.Errorf("sandbox not found in %+v", r.URL.Query())
 }
 
+func getSandboxFS() string {
+	return shim.GetSandboxesStoragePath()
+}
+
+func checkSandboxFSExists(sandboxID string) bool {
+	sbsPath := filepath.Join(string(filepath.Separator), getSandboxFS(), sandboxID)
+	_, err := os.Stat(sbsPath)
+
+	return !os.IsNotExist(err)
+}
+
 // BuildShimClient builds and returns an http client for communicating with the provided sandbox
 func BuildShimClient(sandboxID string, timeout time.Duration) (*http.Client, error) {
 	return buildUnixSocketClient(shim.SocketAddress(sandboxID), timeout)

src/runtime/pkg/katatestutils/constraints.go

@@ -147,8 +147,15 @@ func getDistroDetails() (name, version string, err error) {
 //   centos: 3.10.0-957.12.1.el7.x86_64
 //   fedora: 5.0.9-200.fc29.x86_64
 //
+// For some self compiled kernel, the kernel version will be with "+" as its suffix
+// For example:
+//   5.12.0-rc4+
+// These kernel version can't be parsed by the current lib and lead to panic
+// therefore the '+' should be removed.
+//
 func fixKernelVersion(version string) string {
-	return strings.Replace(version, "_", "-", -1)
+	version = strings.Replace(version, "_", "-", -1)
+	return strings.Replace(version, "+", "", -1)
 }
 
 // handleDistroName checks that the current distro is compatible with

src/runtime/pkg/katatestutils/constraints_test.go

@@ -21,7 +21,6 @@ import (
 )
 
 const (
-	testFileMode    = os.FileMode(0640)
 	invalidOperator = 1234
 
 	skipUnknownDistroName = "skipping test as cannot determine distro name"

src/runtime/pkg/katatestutils/utils.go

@@ -7,8 +7,203 @@
 package katatestutils
 
 import (
+	"encoding/json"
+	"errors"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strconv"
+	"testing"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	testDirMode  = os.FileMode(0750)
+	testFileMode = os.FileMode(0640)
+
+	busyboxConfigJson = `
+{
+	"ociVersion": "1.0.1-dev",
+	"process": {
+		"terminal": true,
+		"user": {
+			"uid": 0,
+			"gid": 0
+		},
+		"args": [
+			"sh"
+		],
+		"env": [
+			"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+			"TERM=xterm"
+		],
+		"cwd": "/",
+		"capabilities": {
+			"bounding": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			],
+			"effective": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			],
+			"inheritable": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			],
+			"permitted": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			],
+			"ambient": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			]
+		},
+		"rlimits": [
+			{
+				"type": "RLIMIT_NOFILE",
+				"hard": 1024,
+				"soft": 1024
+			}
+		],
+		"noNewPrivileges": true
+	},
+	"root": {
+		"path": "rootfs",
+		"readonly": true
+	},
+	"hostname": "runc",
+	"mounts": [
+		{
+			"destination": "/proc",
+			"type": "proc",
+			"source": "proc"
+		},
+		{
+			"destination": "/dev",
+			"type": "tmpfs",
+			"source": "tmpfs",
+			"options": [
+				"nosuid",
+				"strictatime",
+				"mode=755",
+				"size=65536k"
+			]
+		},
+		{
+			"destination": "/dev/pts",
+			"type": "devpts",
+			"source": "devpts",
+			"options": [
+				"nosuid",
+				"noexec",
+				"newinstance",
+				"ptmxmode=0666",
+				"mode=0620",
+				"gid=5"
+			]
+		},
+		{
+			"destination": "/dev/shm",
+			"type": "tmpfs",
+			"source": "shm",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev",
+				"mode=1777",
+				"size=65536k"
+			]
+		},
+		{
+			"destination": "/dev/mqueue",
+			"type": "mqueue",
+			"source": "mqueue",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev"
+			]
+		},
+		{
+			"destination": "/sys",
+			"type": "sysfs",
+			"source": "sysfs",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev",
+				"ro"
+			]
+		},
+		{
+			"destination": "/sys/fs/cgroup",
+			"type": "cgroup",
+			"source": "cgroup",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev",
+				"relatime",
+				"ro"
+			]
+		}
+	],
+	"linux": {
+		"resources": {
+			"devices": [
+				{
+					"allow": false,
+					"access": "rwm"
+				}
+			]
+		},
+		"namespaces": [
+			{
+				"type": "pid"
+			},
+			{
+				"type": "network"
+			},
+			{
+				"type": "ipc"
+			},
+			{
+				"type": "uts"
+			},
+			{
+				"type": "mount"
+			}
+		],
+		"maskedPaths": [
+			"/proc/acpi",
+			"/proc/asound",
+			"/proc/kcore",
+			"/proc/keys",
+			"/proc/latency_stats",
+			"/proc/timer_list",
+			"/proc/timer_stats",
+			"/proc/sched_debug",
+			"/sys/firmware",
+			"/proc/scsi"
+		],
+		"readonlyPaths": [
+			"/proc/bus",
+			"/proc/fs",
+			"/proc/irq",
+			"/proc/sys",
+			"/proc/sysrq-trigger"
+		]
+	}
+}`
 )
 
 type RuntimeConfigOptions struct {
@@ -23,8 +218,6 @@ type RuntimeConfigOptions struct {
 	NetmonPath           string
 	LogPath              string
 	BlockDeviceDriver    string
-	AgentTraceMode       string
-	AgentTraceType       string
 	SharedFS             string
 	VirtioFSDaemon       string
 	JaegerEndpoint       string
@@ -137,8 +330,6 @@ func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string {
 	[agent.kata]
 	enable_debug = ` + strconv.FormatBool(config.AgentDebug) + `
 	enable_tracing = ` + strconv.FormatBool(config.AgentTrace) + `
-	trace_mode = "` + config.AgentTraceMode + `"` + `
-	trace_type = "` + config.AgentTraceType + `"` + `
 
 	[netmon]
 	path = "` + config.NetmonPath + `"
@@ -158,3 +349,34 @@ func IsInGitHubActions() bool {
 	// https://docs.github.com/en/actions/reference/environment-variables#default-environment-variables
 	return os.Getenv("GITHUB_ACTIONS") == "true"
 }
+
+func SetupOCIConfigFile(t *testing.T) (rootPath string, bundlePath, ociConfigFile string) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "katatest-")
+	assert.NoError(err)
+
+	bundlePath = filepath.Join(tmpdir, "bundle")
+	err = os.MkdirAll(bundlePath, testDirMode)
+	assert.NoError(err)
+
+	ociConfigFile = filepath.Join(bundlePath, "config.json")
+	err = ioutil.WriteFile(ociConfigFile, []byte(busyboxConfigJson), testFileMode)
+	assert.NoError(err)
+
+	return tmpdir, bundlePath, ociConfigFile
+}
+
+// WriteOCIConfigFile using spec to update OCI config file by path configPath
+func WriteOCIConfigFile(spec specs.Spec, configPath string) error {
+	if configPath == "" {
+		return errors.New("BUG: need config file path")
+	}
+
+	bytes, err := json.MarshalIndent(spec, "", "\t")
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(configPath, bytes, testFileMode)
+}

src/runtime/pkg/katautils/config-settings.go.in

@@ -87,6 +87,8 @@ const defaultTxRateLimiterMaxRate = uint64(0)
 const defaultConfidentialGuest = false
 const defaultGuestSwap = false
 const defaultRootlessHypervisor = false
+const defaultDisableSeccomp = false
+const defaultVfioMode = "guest-kernel"
 
 var defaultSGXEPCSize = int64(0)
 

src/runtime/pkg/katautils/config.go

@@ -135,6 +135,7 @@ type hypervisor struct {
 	ConfidentialGuest       bool     `toml:"confidential_guest"`
 	GuestSwap               bool     `toml:"enable_guest_swap"`
 	Rootless                bool     `toml:"rootless"`
+	DisableSeccomp          bool     `toml:"disable_seccomp"`
 }
 
 type runtime struct {
@@ -142,6 +143,7 @@ type runtime struct {
 	JaegerEndpoint      string   `toml:"jaeger_endpoint"`
 	JaegerUser          string   `toml:"jaeger_user"`
 	JaegerPassword      string   `toml:"jaeger_password"`
+	VfioMode            string   `toml:"vfio_mode"`
 	SandboxBindMounts   []string `toml:"sandbox_bind_mounts"`
 	Experimental        []string `toml:"experimental"`
 	Debug               bool     `toml:"enable_debug"`
@@ -153,8 +155,6 @@ type runtime struct {
 }
 
 type agent struct {
-	TraceMode           string   `toml:"trace_mode"`
-	TraceType           string   `toml:"trace_type"`
 	KernelModules       []string `toml:"kernel_modules"`
 	Debug               bool     `toml:"enable_debug"`
 	Tracing             bool     `toml:"enable_tracing"`
@@ -308,11 +308,24 @@ func (h hypervisor) GetEntropySource() string {
 	return h.EntropySource
 }
 
+// Current cpu number should not larger than defaultMaxVCPUs()
+func getCurrentCpuNum() uint32 {
+	var cpu uint32
+	h := hypervisor{}
+
+	cpu = uint32(goruntime.NumCPU())
+	if cpu > h.defaultMaxVCPUs() {
+		cpu = h.defaultMaxVCPUs()
+	}
+
+	return cpu
+}
+
 func (h hypervisor) defaultVCPUs() uint32 {
-	numCPUs := goruntime.NumCPU()
+	numCPUs := getCurrentCpuNum()
 
 	if h.NumVCPUs < 0 || h.NumVCPUs > int32(numCPUs) {
-		return uint32(numCPUs)
+		return numCPUs
 	}
 	if h.NumVCPUs == 0 { // or unspecified
 		return defaultVCPUCount
@@ -489,14 +502,6 @@ func (a agent) trace() bool {
 	return a.Tracing
 }
 
-func (a agent) traceMode() string {
-	return a.TraceMode
-}
-
-func (a agent) traceType() string {
-	return a.TraceType
-}
-
 func (a agent) kernelModules() []string {
 	return a.KernelModules
 }
@@ -875,6 +880,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		VirtioFSExtraArgs:       h.VirtioFSExtraArgs,
 		SGXEPCSize:              defaultSGXEPCSize,
 		EnableAnnotations:       h.EnableAnnotations,
+		DisableSeccomp:          h.DisableSeccomp,
 	}, nil
 }
 
@@ -929,8 +935,6 @@ func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oc
 			LongLiveConn:       true,
 			Debug:              agent.debug(),
 			Trace:              agent.trace(),
-			TraceMode:          agent.traceMode(),
-			TraceType:          agent.traceType(),
 			KernelModules:      agent.kernelModules(),
 			EnableDebugConsole: agent.debugConsoleEnabled(),
 			DialTimeout:        agent.dialTimout(),
@@ -976,10 +980,6 @@ func SetKernelParams(runtimeConfig *oci.RuntimeConfig) error {
 	}
 
 	// next, check for agent specific kernel params
-	err := vc.KataAgentSetDefaultTraceConfigOptions(&runtimeConfig.AgentConfig)
-	if err != nil {
-		return err
-	}
 
 	params := vc.KataAgentKernelParams(runtimeConfig.AgentConfig)
 
@@ -1072,6 +1072,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
 		ConfidentialGuest:       defaultConfidentialGuest,
 		GuestSwap:               defaultGuestSwap,
 		Rootless:                defaultRootlessHypervisor,
+		DisableSeccomp:          defaultDisableSeccomp,
 	}
 }
 
@@ -1081,6 +1082,11 @@ func initConfig() (config oci.RuntimeConfig, err error) {
 		return oci.RuntimeConfig{}, err
 	}
 
+	err = config.VfioMode.VFIOSetMode(defaultVfioMode)
+	if err != nil {
+		return oci.RuntimeConfig{}, err
+	}
+
 	config = oci.RuntimeConfig{
 		HypervisorType:   defaultHypervisor,
 		HypervisorConfig: GetDefaultHypervisorConfig(),
@@ -1127,6 +1133,14 @@ func LoadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat
 		}
 	}
 
+	if tomlConf.Runtime.VfioMode != "" {
+		err = config.VfioMode.VFIOSetMode(tomlConf.Runtime.VfioMode)
+
+		if err != nil {
+			return "", config, err
+		}
+	}
+
 	if !ignoreLogging {
 		err := handleSystemLog("", "")
 		if err != nil {

src/runtime/pkg/katautils/config_test.go

@@ -14,7 +14,6 @@ import (
 	"path"
 	"path/filepath"
 	"reflect"
-	goruntime "runtime"
 	"strings"
 	"syscall"
 	"testing"
@@ -156,7 +155,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		KernelParams:          vc.DeserializeParams(strings.Fields(kernelParams)),
 		HypervisorMachineType: machineType,
 		NumVCPUs:              defaultVCPUCount,
-		DefaultMaxVCPUs:       uint32(goruntime.NumCPU()),
+		DefaultMaxVCPUs:       getCurrentCpuNum(),
 		MemorySize:            defaultMemSize,
 		DisableBlockDeviceUse: disableBlockDevice,
 		BlockDeviceDriver:     defaultBlockDeviceDriver,
@@ -919,13 +918,13 @@ func TestNewClhHypervisorConfig(t *testing.T) {
 func TestHypervisorDefaults(t *testing.T) {
 	assert := assert.New(t)
 
-	numCPUs := goruntime.NumCPU()
+	numCPUs := getCurrentCpuNum()
 
 	h := hypervisor{}
 
 	assert.Equal(h.machineType(), defaultMachineType, "default hypervisor machine type wrong")
 	assert.Equal(h.defaultVCPUs(), defaultVCPUCount, "default vCPU number is wrong")
-	assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong")
+	assert.Equal(h.defaultMaxVCPUs(), numCPUs, "default max vCPU number is wrong")
 	assert.Equal(h.defaultMemSz(), defaultMemSize, "default memory size is wrong")
 
 	machineType := "foo"
@@ -934,23 +933,23 @@ func TestHypervisorDefaults(t *testing.T) {
 
 	// auto inferring
 	h.NumVCPUs = -1
-	assert.Equal(h.defaultVCPUs(), uint32(numCPUs), "default vCPU number is wrong")
+	assert.Equal(h.defaultVCPUs(), numCPUs, "default vCPU number is wrong")
 
 	h.NumVCPUs = 2
 	assert.Equal(h.defaultVCPUs(), uint32(2), "default vCPU number is wrong")
 
 	h.NumVCPUs = int32(numCPUs) + 1
-	assert.Equal(h.defaultVCPUs(), uint32(numCPUs), "default vCPU number is wrong")
+	assert.Equal(h.defaultVCPUs(), numCPUs, "default vCPU number is wrong")
 
 	h.DefaultMaxVCPUs = 2
 	assert.Equal(h.defaultMaxVCPUs(), uint32(2), "default max vCPU number is wrong")
 
-	h.DefaultMaxVCPUs = uint32(numCPUs) + 1
-	assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong")
+	h.DefaultMaxVCPUs = numCPUs + 1
+	assert.Equal(h.defaultMaxVCPUs(), numCPUs, "default max vCPU number is wrong")
 
 	maxvcpus := vc.MaxQemuVCPUs()
 	h.DefaultMaxVCPUs = maxvcpus + 1
-	assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong")
+	assert.Equal(h.defaultMaxVCPUs(), numCPUs, "default max vCPU number is wrong")
 
 	h.MemorySize = 1024
 	assert.Equal(h.defaultMemSz(), uint32(1024), "default memory size is wrong")
@@ -1152,9 +1151,6 @@ func TestAgentDefaults(t *testing.T) {
 
 	a.Tracing = true
 	assert.Equal(a.trace(), a.Tracing)
-
-	assert.Equal(a.traceMode(), a.TraceMode)
-	assert.Equal(a.traceType(), a.TraceType)
 }
 
 func TestGetDefaultConfigFilePaths(t *testing.T) {

src/runtime/pkg/katautils/container_engine.go

@@ -1,77 +0,0 @@
-// Copyright (c) 2019 SUSE LLC
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-package katautils
-
-import (
-	"os/exec"
-
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
-)
-
-type CtrEngine struct {
-	Name string
-}
-
-var (
-	DockerLikeCtrEngines = []string{"docker", "podman"}
-)
-
-func (e *CtrEngine) Init(name string) (string, error) {
-	var out string
-	out, err := utils.RunCommandFull([]string{name, "version"}, true)
-	if err != nil {
-		return out, err
-	}
-
-	e.Name = name
-	return out, nil
-}
-
-func (e *CtrEngine) Inspect(image string) (string, error) {
-	// Only hit the network if the image doesn't exist locally
-	return utils.RunCommand([]string{e.Name, "inspect", "--type=image", image})
-}
-
-func (e *CtrEngine) Pull(image string) (string, error) {
-	return utils.RunCommand([]string{e.Name, "pull", image})
-}
-
-func (e *CtrEngine) Create(image string) (string, error) {
-	return utils.RunCommand([]string{e.Name, "create", image})
-}
-
-func (e *CtrEngine) Rm(ctrID string) (string, error) {
-	return utils.RunCommand([]string{e.Name, "rm", ctrID})
-}
-
-func (e *CtrEngine) GetRootfs(ctrID string, dir string) error {
-	cmd1 := exec.Command(e.Name, "export", ctrID)
-	cmd2 := exec.Command("tar", "-C", dir, "-xvf", "-")
-
-	cmd1Stdout, err := cmd1.StdoutPipe()
-	if err != nil {
-		return err
-	}
-
-	cmd2.Stdin = cmd1Stdout
-
-	err = cmd2.Start()
-	if err != nil {
-		return err
-	}
-
-	err = cmd1.Run()
-	if err != nil {
-		return err
-	}
-
-	err = cmd2.Wait()
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

src/runtime/pkg/katautils/create.go

@@ -112,7 +112,7 @@ func SetEphemeralStorageType(ociSpec specs.Spec) specs.Spec {
 func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec specs.Spec, runtimeConfig oci.RuntimeConfig, rootFs vc.RootFs,
 	containerID, bundlePath, console string, disableOutput, systemdCgroup bool) (_ vc.VCSandbox, _ vc.Process, err error) {
 	span, ctx := katatrace.Trace(ctx, nil, "CreateSandbox", createTracingTags)
-	katatrace.AddTag(span, "container_id", containerID)
+	katatrace.AddTags(span, "container_id", containerID)
 	defer span.End()
 
 	sandboxConfig, err := oci.SandboxConfig(ociSpec, runtimeConfig, bundlePath, containerID, console, disableOutput, systemdCgroup)
@@ -167,7 +167,7 @@ func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec specs.Spec, runtimeCo
 
 	sid := sandbox.ID()
 	kataUtilsLogger = kataUtilsLogger.WithField("sandbox", sid)
-	katatrace.AddTag(span, "sandbox_id", sid)
+	katatrace.AddTags(span, "sandbox_id", sid)
 
 	containers := sandbox.GetAllContainers()
 	if len(containers) != 1 {
@@ -211,7 +211,7 @@ func CreateContainer(ctx context.Context, sandbox vc.VCSandbox, ociSpec specs.Sp
 	var c vc.VCContainer
 
 	span, ctx := katatrace.Trace(ctx, nil, "CreateContainer", createTracingTags)
-	katatrace.AddTag(span, "container_id", containerID)
+	katatrace.AddTags(span, "container_id", containerID)
 	defer span.End()
 
 	ociSpec = SetEphemeralStorageType(ociSpec)
@@ -237,7 +237,7 @@ func CreateContainer(ctx context.Context, sandbox vc.VCSandbox, ociSpec specs.Sp
 		return vc.Process{}, err
 	}
 
-	katatrace.AddTag(span, "sandbox_id", sandboxID)
+	katatrace.AddTags(span, "sandbox_id", sandboxID)
 
 	c, err = sandbox.CreateContainer(ctx, contConfig)
 	if err != nil {

src/runtime/pkg/katautils/create_test.go

@@ -8,7 +8,6 @@ package katautils
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -20,7 +19,6 @@ import (
 	"testing"
 
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
@@ -37,8 +35,6 @@ const (
 )
 
 var (
-	testBundleDir = ""
-
 	// testingImpl is a concrete mock RVC implementation used for testing
 	testingImpl = &vcmock.VCMock{}
 	// mock sandbox
@@ -53,49 +49,6 @@ func init() {
 	tc = ktu.NewTestConstraint(false)
 }
 
-func writeOCIConfigFile(spec specs.Spec, configPath string) error {
-	if configPath == "" {
-		return errors.New("BUG: need config file path")
-	}
-
-	bytes, err := json.MarshalIndent(spec, "", "\t")
-	if err != nil {
-		return err
-	}
-
-	return ioutil.WriteFile(configPath, bytes, testFileMode)
-}
-
-// Create an OCI bundle in the specified directory.
-//
-// Note that the directory will be created, but it's parent is expected to exist.
-//
-// This function works by copying the already-created test bundle. Ideally,
-// the bundle would be recreated for each test, but createRootfs() uses
-// docker which on some systems is too slow, resulting in the tests timing
-// out.
-func makeOCIBundle(bundleDir string) error {
-	from := testBundleDir
-	to := bundleDir
-
-	// only the basename of bundleDir needs to exist as bundleDir
-	// will get created by cp(1).
-	base := filepath.Dir(bundleDir)
-
-	for _, dir := range []string{from, base} {
-		if !FileExists(dir) {
-			return fmt.Errorf("BUG: directory %v should exist", dir)
-		}
-	}
-
-	output, err := utils.RunCommandFull([]string{"cp", "-a", from, to}, true)
-	if err != nil {
-		return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output)
-	}
-
-	return nil
-}
-
 // newTestRuntimeConfig creates a new RuntimeConfig
 func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConfig, error) {
 	if dir == "" {
@@ -262,21 +215,12 @@ func TestSetKernelParamsUserOptionTakesPriority(t *testing.T) {
 func TestCreateSandboxConfigFail(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, _ := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -305,21 +249,12 @@ func TestCreateSandboxFail(t *testing.T) {
 
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, _ := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -375,18 +310,9 @@ func TestCheckForFips(t *testing.T) {
 func TestCreateContainerContainerConfigFail(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -396,7 +322,7 @@ func TestCreateContainerContainerConfigFail(t *testing.T) {
 	spec.Annotations[testContainerTypeAnnotation] = containerType
 
 	// rewrite file
-	err = writeOCIConfigFile(spec, ociConfigFile)
+	err = ktu.WriteOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
 	rootFs := vc.RootFs{Mounted: true}
@@ -412,18 +338,9 @@ func TestCreateContainerContainerConfigFail(t *testing.T) {
 func TestCreateContainerFail(t *testing.T) {
 	assert := assert.New(t)
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -433,7 +350,7 @@ func TestCreateContainerFail(t *testing.T) {
 	spec.Annotations[testSandboxIDAnnotation] = testSandboxID
 
 	// rewrite file
-	err = writeOCIConfigFile(spec, ociConfigFile)
+	err = ktu.WriteOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
 	rootFs := vc.RootFs{Mounted: true}
@@ -456,18 +373,9 @@ func TestCreateContainer(t *testing.T) {
 		mockSandbox.CreateContainerFunc = nil
 	}()
 
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
+	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
 	defer os.RemoveAll(tmpdir)
 
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(FileExists(ociConfigFile))
-
 	spec, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 
@@ -477,7 +385,7 @@ func TestCreateContainer(t *testing.T) {
 	spec.Annotations[testSandboxIDAnnotation] = testSandboxID
 
 	// rewrite file
-	err = writeOCIConfigFile(spec, ociConfigFile)
+	err = ktu.WriteOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
 	rootFs := vc.RootFs{Mounted: true}

src/runtime/pkg/katautils/hook.go

@@ -35,11 +35,7 @@ func hookLogger() *logrus.Entry {
 func runHook(ctx context.Context, hook specs.Hook, cid, bundlePath string) error {
 	span, _ := katatrace.Trace(ctx, hookLogger(), "runHook", hookTracingTags)
 	defer span.End()
-
-	// FIXME
-	// span.LogFields(
-	// 	log.String("hook-name", hook.Path),
-	// 	log.String("hook-args", strings.Join(hook.Args, " ")))
+	katatrace.AddTags(span, "path", hook.Path, "args", hook.Args)
 
 	state := specs.State{
 		Pid:    syscall.Gettid(),
@@ -96,7 +92,7 @@ func runHook(ctx context.Context, hook specs.Hook, cid, bundlePath string) error
 
 func runHooks(ctx context.Context, hooks []specs.Hook, cid, bundlePath, hookType string) error {
 	span, ctx := katatrace.Trace(ctx, hookLogger(), "runHooks", hookTracingTags)
-	katatrace.AddTag(span, "type", hookType)
+	katatrace.AddTags(span, "type", hookType)
 	defer span.End()
 
 	for _, hook := range hooks {

src/runtime/pkg/katautils/katatrace/tracing.go

@@ -7,14 +7,16 @@ package katatrace
 
 import (
 	"context"
+	"encoding/json"
 
 	"github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel"
-	"go.opentelemetry.io/otel/exporters/trace/jaeger"
-	"go.opentelemetry.io/otel/label"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/jaeger"
 	"go.opentelemetry.io/otel/propagation"
-	export "go.opentelemetry.io/otel/sdk/export/trace"
+	"go.opentelemetry.io/otel/sdk/resource"
 	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
 	"go.opentelemetry.io/otel/trace"
 	otelTrace "go.opentelemetry.io/otel/trace"
 )
@@ -25,10 +27,10 @@ import (
 // https: //github.com/kata-containers/tests/blob/master/tracing/tracing-test.sh
 type kataSpanExporter struct{}
 
-var _ export.SpanExporter = (*kataSpanExporter)(nil)
+var _ sdktrace.SpanExporter = (*kataSpanExporter)(nil)
 
 // ExportSpans exports SpanData to Jaeger.
-func (e *kataSpanExporter) ExportSpans(ctx context.Context, spans []*export.SpanData) error {
+func (e *kataSpanExporter) ExportSpans(ctx context.Context, spans []sdktrace.ReadOnlySpan) error {
 	for _, span := range spans {
 		kataTraceLogger.Tracef("Reporting span %+v", span)
 	}
@@ -39,9 +41,9 @@ func (e *kataSpanExporter) Shutdown(ctx context.Context) error {
 	return nil
 }
 
-// tracerCloser contains a copy of the closer returned by createTracer() which
-// is used by stopTracing().
-var tracerCloser func()
+// tp is the trace provider created in CreateTracer() and used in StopTracing()
+// to flush and shutdown all spans.
+var tp *sdktrace.TracerProvider
 
 var kataTraceLogger = logrus.NewEntry(logrus.New())
 
@@ -61,10 +63,10 @@ type JaegerConfig struct {
 }
 
 // CreateTracer create a tracer
-func CreateTracer(name string, config *JaegerConfig) (func(), error) {
+func CreateTracer(name string, config *JaegerConfig) (*sdktrace.TracerProvider, error) {
 	if !tracing {
 		otel.SetTracerProvider(trace.NewNoopTracerProvider())
-		return func() {}, nil
+		return nil, nil
 	}
 
 	// build kata exporter to log reporting span records
@@ -76,37 +78,32 @@ func CreateTracer(name string, config *JaegerConfig) (func(), error) {
 		collectorEndpoint = "http://localhost:14268/api/traces"
 	}
 
-	jaegerExporter, err := jaeger.NewRawExporter(
-		jaeger.WithCollectorEndpoint(collectorEndpoint,
+	jaegerExporter, err := jaeger.New(
+		jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(collectorEndpoint),
 			jaeger.WithUsername(config.JaegerUser),
 			jaeger.WithPassword(config.JaegerPassword),
-		), jaeger.WithProcess(jaeger.Process{
-			ServiceName: name,
-			Tags: []label.KeyValue{
-				label.String("exporter", "jaeger"),
-				label.String("lib", "opentelemetry"),
-			},
-		}))
+		),
+	)
+
 	if err != nil {
 		return nil, err
 	}
 
 	// build tracer provider, that combining both jaeger exporter and kata exporter.
 	tp := sdktrace.NewTracerProvider(
-		sdktrace.WithConfig(
-			sdktrace.Config{
-				DefaultSampler: sdktrace.AlwaysSample(),
-			},
-		),
+		sdktrace.WithSampler(sdktrace.AlwaysSample()),
 		sdktrace.WithSyncer(kataExporter),
 		sdktrace.WithSyncer(jaegerExporter),
+		sdktrace.WithResource(resource.NewSchemaless(
+			semconv.ServiceNameKey.String(name),
+			attribute.String("exporter", "jaeger"),
+			attribute.String("lib", "opentelemetry"),
+		)),
 	)
 
-	tracerCloser = jaegerExporter.Flush
-
 	otel.SetTracerProvider(tp)
 	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))
-	return tracerCloser, nil
+	return tp, nil
 }
 
 // StopTracing ends all tracing, reporting the spans to the collector.
@@ -121,9 +118,8 @@ func StopTracing(ctx context.Context) {
 	}
 
 	// report all possible spans to the collector
-	if tracerCloser != nil {
-		tracerCloser()
-	}
+	tp.ForceFlush(ctx)
+	tp.Shutdown(ctx)
 }
 
 // Trace creates a new tracing span based on the specified name and parent context.
@@ -138,12 +134,12 @@ func Trace(parent context.Context, logger *logrus.Entry, name string, tags ...ma
 		parent = context.Background()
 	}
 
-	var otelTags []label.KeyValue
+	var otelTags []attribute.KeyValue
 	// do not append tags if tracing is disabled
 	if tracing {
 		for _, tagSet := range tags {
 			for k, v := range tagSet {
-				otelTags = append(otelTags, label.Key(k).String(v))
+				otelTags = append(otelTags, attribute.Key(k).String(v))
 			}
 		}
 	}
@@ -163,8 +159,63 @@ func Trace(parent context.Context, logger *logrus.Entry, name string, tags ...ma
 	return span, ctx
 }
 
-// AddTag adds an additional key-value pair to a tracing span. This can be used to
-// provide dynamic tags that are determined at runtime.
-func AddTag(span otelTrace.Span, key string, value interface{}) {
-	span.SetAttributes(label.Any(key, value))
+func addTag(span otelTrace.Span, key string, value interface{}) {
+	// do not append tags if tracing is disabled
+	if !tracing {
+		return
+	}
+	if value == nil {
+		span.SetAttributes(attribute.String(key, "nil"))
+		return
+	}
+
+	switch value := value.(type) {
+	case string:
+		span.SetAttributes(attribute.String(key, value))
+	case bool:
+		span.SetAttributes(attribute.Bool(key, value))
+	case int:
+		span.SetAttributes(attribute.Int(key, value))
+	case int8:
+		span.SetAttributes(attribute.Int(key, int(value)))
+	case int16:
+		span.SetAttributes(attribute.Int(key, int(value)))
+	case int64:
+		span.SetAttributes(attribute.Int64(key, value))
+	case float64:
+		span.SetAttributes(attribute.Float64(key, value))
+	default:
+		content, err := json.Marshal(value)
+		if content == nil && err == nil {
+			span.SetAttributes(attribute.String(key, "nil"))
+		} else if content != nil && err == nil {
+			span.SetAttributes(attribute.String(key, string(content)))
+		} else {
+			kataTraceLogger.WithField("type", "bug").Error("span attribute value error")
+		}
+	}
+}
+
+// AddTag adds additional key-value pairs to a tracing span. This can be used to provide
+// dynamic tags that are determined at runtime and tags with a non-string value.
+// Must have an even number of keyValues with keys being strings.
+func AddTags(span otelTrace.Span, keyValues ...interface{}) {
+	if !tracing {
+		return
+	}
+	if len(keyValues) < 2 {
+		kataTraceLogger.WithField("type", "bug").Error("not enough inputs for attributes")
+		return
+	} else if len(keyValues)%2 != 0 {
+		kataTraceLogger.WithField("type", "bug").Error("number of attribute keyValues is not even")
+		return
+	}
+	for i := 0; i < len(keyValues); i++ {
+		if key, ok := keyValues[i].(string); ok {
+			addTag(span, key, keyValues[i+1])
+		} else {
+			kataTraceLogger.WithField("type", "bug").Error("key in attributes is not a string")
+		}
+		i++
+	}
 }