Merge pull request #4385 from snir911/2.4.2-branch-bump

# Kata Containers 2.4.2

.github/workflows/commit-message-check.yaml

@@ -5,8 +5,6 @@ on:
       - opened
       - reopened
       - synchronize
-      - labeled
-      - unlabeled
 
 env:
   error_msg: |+
@@ -22,9 +20,15 @@ jobs:
     - name: Get PR Commits
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       id: 'get-pr-commits'
-      uses: tim-actions/get-pr-commits@v1.0.0
+      uses: tim-actions/get-pr-commits@v1.2.0
       with:
         token: ${{ secrets.GITHUB_TOKEN }}
+        # Filter out revert commits
+        # The format of a revert commit is as follows:
+        #
+        # Revert "<original-subject-line>"
+        #
+        filter_out_pattern: '^Revert "'
 
     - name: DCO Check
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}

.github/workflows/darwin-tests.yaml

@@ -0,0 +1,25 @@
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+
+name: Darwin tests
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.16.x, 1.17.x]
+        os: [macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Build utils
+      run: ./ci/darwin-test.sh

.github/workflows/kata-deploy-push.yaml

@@ -7,9 +7,9 @@ on:
       - edited
       - reopened
       - synchronize
-      - labeled
-      - unlabeled
-  push:
+    paths:
+      - tools/**
+      - versions.yaml
 
 jobs:
   build-asset:
@@ -18,7 +18,6 @@ jobs:
       matrix:
         asset:
           - kernel
-          - kernel-experimental
           - shim-v2
           - qemu
           - cloud-hypervisor

.github/workflows/kata-deploy-test.yaml

@@ -1,4 +1,5 @@
 on:
+  workflow_dispatch: # this is used to trigger the workflow on non-main branches
   issue_comment:
     types: [created, edited]
 
@@ -48,18 +49,16 @@ jobs:
           - rootfs-initrd
           - shim-v2
     steps:
-      # As Github action event `issue_comment` does not provide the right ref
-      # (commit/branch) to be tested, let's use this third part action to work
-      # this limitation around.
-      - name: resolve pr refs
-        id: refs
-        uses: kata-containers/resolve-pr-refs@v0.0.3
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-
+      - name: get-PR-ref
+        id: get-PR-ref
+        run: |
+            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
+            echo "reference for PR: " ${ref}
+            echo "##[set-output name=pr-ref;]${ref}"
       - uses: actions/checkout@v2
         with:
-          ref: ${{ steps.refs.outputs.base_ref }}
+          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
+
       - name: Install docker
         run: |
           curl -fsSL https://test.docker.com -o test-docker.sh
@@ -86,17 +85,15 @@ jobs:
     runs-on: ubuntu-latest
     needs: build-asset
     steps:
-      # As Github action event `issue_comment` does not provide the right ref
-      # (commit/branch) to be tested, let's use this third part action to work
-      # this limitation around.
-      - name: resolve pr refs
-        id: refs
-        uses: kata-containers/resolve-pr-refs@v0.0.3
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: get-PR-ref
+        id: get-PR-ref
+        run: |
+            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
+            echo "reference for PR: " ${ref}
+            echo "##[set-output name=pr-ref;]${ref}"
       - uses: actions/checkout@v2
         with:
-          ref: ${{ steps.refs.outputs.base_ref }}
+          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
       - name: get-artifacts
         uses: actions/download-artifact@v2
         with:
@@ -115,17 +112,15 @@ jobs:
     needs: create-kata-tarball
     runs-on: ubuntu-latest
     steps:
-      # As Github action event `issue_comment` does not provide the right ref
-      # (commit/branch) to be tested, let's use this third part action to work
-      # this limitation around.
-      - name: resolve pr refs
-        id: refs
-        uses: kata-containers/resolve-pr-refs@v0.0.3
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: get-PR-ref
+        id: get-PR-ref
+        run: |
+            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
+            echo "reference for PR: " ${ref}
+            echo "##[set-output name=pr-ref;]${ref}"
       - uses: actions/checkout@v2
         with:
-          ref: ${{ steps.refs.outputs.base_ref }}
+          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
       - name: get-kata-tarball
         uses: actions/download-artifact@v2
         with:
@@ -133,18 +128,14 @@ jobs:
       - name: build-and-push-kata-deploy-ci
         id: build-and-push-kata-deploy-ci
         run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          pushd $GITHUB_WORKSPACE
-          git checkout $tag
-          pkg_sha=$(git rev-parse HEAD)
-          popd
+          PR_SHA=$(git log --format=format:%H -n1)
           mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
-          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t quay.io/kata-containers/kata-deploy-ci:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
+          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t quay.io/kata-containers/kata-deploy-ci:$PR_SHA $GITHUB_WORKSPACE/tools/packaging/kata-deploy
           docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
-          docker push quay.io/kata-containers/kata-deploy-ci:$pkg_sha
+          docker push quay.io/kata-containers/kata-deploy-ci:$PR_SHA
           mkdir -p packaging/kata-deploy
           ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
-          echo "::set-output name=PKG_SHA::${pkg_sha}"
+          echo "::set-output name=PKG_SHA::${PR_SHA}"
       - name: test-kata-deploy-ci-in-aks
         uses: ./packaging/kata-deploy/action
         with:

.github/workflows/move-issues-to-in-progress.yaml

@@ -10,8 +10,6 @@ on:
     types:
       - opened
       - reopened
-      - labeled
-      - unlabeled
 
 jobs:
   move-linked-issues-to-in-progress:

.github/workflows/release.yaml

@@ -26,6 +26,7 @@ jobs:
 
       - name: Build ${{ matrix.asset }}
         run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh
           ./tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh --build="${KATA_ASSET}"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink
@@ -140,13 +141,10 @@ jobs:
       - uses: actions/checkout@v2
       - name: generate-and-upload-tarball
         run: |
-          pushd $GITHUB_WORKSPACE/src/agent
-          cargo vendor >> .cargo/config
-          popd
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
           tarball="kata-containers-$tag-vendor.tar.gz"
           pushd $GITHUB_WORKSPACE
-          tar -cvzf "${tarball}" src/agent/.cargo/config src/agent/vendor
+          bash -c "tools/packaging/release/generate_vendor.sh ${tarball}"
           GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}" 
           popd
 

.github/workflows/snap.yaml

@@ -6,8 +6,6 @@ on:
       - synchronize
       - reopened
       - edited
-      - labeled
-      - unlabeled
 
 jobs:
   test:

.github/workflows/static-checks.yaml

@@ -5,8 +5,6 @@ on:
       - edited
       - reopened
       - synchronize
-      - labeled
-      - unlabeled
 
 name: Static checks
 jobs:

.gitignore

@@ -9,4 +9,5 @@ src/agent/src/version.rs
 src/agent/kata-agent.service
 src/agent/protocols/src/*.rs
 !src/agent/protocols/src/lib.rs
+build
 

CONTRIBUTING.md

@@ -2,4 +2,4 @@
 
 ## This repo is part of [Kata Containers](https://katacontainers.io)
 
-For details on how to contribute to the Kata Containers project, please see the main [contributing document](https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md).
+For details on how to contribute to the Kata Containers project, please see the main [contributing document](https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md).

README.md

@@ -17,16 +17,73 @@ standard implementation of lightweight Virtual Machines (VMs) that feel and
 perform like containers, but provide the workload isolation and security
 advantages of VMs.
 
+## License
+
+The code is licensed under the Apache 2.0 license.
+See [the license file](LICENSE) for further details.
+
+## Platform support
+
+Kata Containers currently runs on 64-bit systems supporting the following
+technologies:
+
+| Architecture | Virtualization technology |
+|-|-|
+| `x86_64`, `amd64` | [Intel](https://www.intel.com) VT-x, AMD SVM |
+| `aarch64` ("`arm64`")| [ARM](https://www.arm.com) Hyp |
+| `ppc64le` | [IBM](https://www.ibm.com) Power |
+| `s390x` | [IBM](https://www.ibm.com) Z & LinuxONE SIE |
+
+### Hardware requirements
+
+The [Kata Containers runtime](src/runtime) provides a command to
+determine if your host system is capable of running and creating a
+Kata Container:
+
+```bash
+$ kata-runtime check
+```
+
+> **Notes:**
+>
+> - This command runs a number of checks including connecting to the
+>   network to determine if a newer release of Kata Containers is
+>   available on GitHub. If you do not wish this to check to run, add
+>   the `--no-network-checks` option.
+>
+> - By default, only a brief success / failure message is printed.
+>   If more details are needed, the `--verbose` flag can be used to display the
+>   list of all the checks performed.
+>
+> - If the command is run as the `root` user additional checks are
+>   run (including checking if another incompatible hypervisor is running).
+>   When running as `root`, network checks are automatically disabled.
+
 ## Getting started
 
 See the [installation documentation](docs/install).
 
 ## Documentation
 
-See the [official documentation](docs)
-(including [installation guides](docs/install),
-[the developer guide](docs/Developer-Guide.md),
-[design documents](docs/design) and more).
+See the [official documentation](docs) including:
+
+- [Installation guides](docs/install)
+- [Developer guide](docs/Developer-Guide.md)
+- [Design documents](docs/design)
+  - [Architecture overview](docs/design/architecture)
+
+## Configuration
+
+Kata Containers uses a single
+[configuration file](src/runtime/README.md#configuration)
+which contains a number of sections for various parts of the Kata
+Containers system including the [runtime](src/runtime), the
+[agent](src/agent) and the [hypervisor](#hypervisors).
+
+## Hypervisors
+
+See the [hypervisors document](docs/hypervisors.md) and the
+[Hypervisor specific configuration details](src/runtime/README.md#hypervisor-specific-configuration).
 
 ## Community
 
@@ -48,6 +105,8 @@ Please raise an issue
 
 ## Developers
 
+See the [developer guide](docs/Developer-Guide.md).
+
 ### Components
 
 ### Main components
@@ -84,8 +143,4 @@ the [components](#components) section for further details.
 
 ## Glossary of Terms
 
-See the [glossary of terms](Glossary.md) related to Kata Containers.
----
-
-[kernel]: https://www.kernel.org
-[github-katacontainers.io]: https://github.com/kata-containers/www.katacontainers.io
+See the [glossary of terms](https://github.com/kata-containers/kata-containers/wiki/Glossary) related to Kata Containers.

VERSION

@@ -1 +1 @@
-2.4.0-alpha2
+2.4.2

ci/darwin-test.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2022 Apple Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+cidir=$(dirname "$0")
+runtimedir=$cidir/../src/runtime
+
+build_working_packages() {
+	# working packages:
+	device_api=$runtimedir/virtcontainers/device/api
+	device_config=$runtimedir/virtcontainers/device/config
+	device_drivers=$runtimedir/virtcontainers/device/drivers
+	device_manager=$runtimedir/virtcontainers/device/manager
+	rc_pkg_dir=$runtimedir/pkg/resourcecontrol/
+	utils_pkg_dir=$runtimedir/virtcontainers/utils
+
+	# broken packages :( :
+	#katautils=$runtimedir/pkg/katautils
+	#oci=$runtimedir/pkg/oci
+	#vc=$runtimedir/virtcontainers
+
+	pkgs=(
+		"$device_api"
+		"$device_config"
+		"$device_drivers"
+		"$device_manager"
+		"$utils_pkg_dir"
+		"$rc_pkg_dir")
+	for pkg in "${pkgs[@]}"; do
+		echo building "$pkg"
+		pushd "$pkg" &>/dev/null
+		go build
+		go test
+		popd &>/dev/null
+	done
+}
+
+build_working_packages

ci/openshift-ci/images/Dockerfile.buildroot

@@ -4,7 +4,7 @@
 #
 # This is the build root image for Kata Containers on OpenShift CI.
 #
-FROM registry.centos.org/centos:8
+FROM quay.io/centos/centos:stream8
 
 RUN yum -y update && \
     yum -y install \

docs/Developer-Guide.md

@@ -212,11 +212,13 @@ $ sudo systemctl restart systemd-journald
 >
 > - You should only do this step if you are testing with the latest version of the agent.
 
-The rust-agent is built with a static linked `musl.` To configure this:
+The agent is built with a statically linked `musl.` The default `libc` used is `musl`, but on `ppc64le` and `s390x`, `gnu` should be used. To configure this:
 
 ```
-rustup target add x86_64-unknown-linux-musl
-sudo ln -s /usr/bin/g++ /bin/musl-g++
+$ export ARCH=$(uname -m)
+$ if [ "$ARCH" = "ppc64le" -o "$ARCH" = "s390x" ]; then export LIBC=gnu; else export LIBC=musl; fi
+$ [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
+$ rustup target add ${ARCH}-unknown-linux-${LIBC}
 ```
 
 To build the agent:

docs/Limitations.md

@@ -57,6 +57,13 @@ for advice on which repository to raise the issue against.
 
 This section lists items that might be possible to fix.
 
+## OCI CLI commands
+
+### Docker and Podman support
+Currently Kata Containers does not support Docker or Podman.
+
+See issue https://github.com/kata-containers/kata-containers/issues/722 for more information.
+
 ## Runtime commands
 
 ### checkpoint and restore
@@ -97,57 +104,12 @@ See issue https://github.com/clearcontainers/runtime/issues/341 and [the constra
 For CPUs resource management see
 [CPU constraints](design/vcpu-handling.md).
 
-### docker run and shared memory
-
-The runtime does not implement the `docker run --shm-size` command to
-set the size of the `/dev/shm tmpfs` within the container. It is possible to pass this configuration value into the VM container so the appropriate mount command happens at launch time.
-
-See issue https://github.com/kata-containers/kata-containers/issues/21 for more information.
-
 # Architectural limitations
 
 This section lists items that might not be fixed due to fundamental
 architectural differences between "soft containers" (i.e. traditional Linux*
 containers) and those based on VMs.
 
-## Networking limitations
-
-### Support for joining an existing VM network
-
-Docker supports the ability for containers to join another containers
-namespace with the `docker run --net=containers` syntax. This allows
-multiple containers to share a common network namespace and the network
-interfaces placed in the network namespace.  Kata Containers does not
-support network namespace sharing. If a Kata Container is setup to
-share the network namespace of a `runc` container, the runtime
-effectively takes over all the network interfaces assigned to the
-namespace and binds them to the VM. Consequently, the `runc` container loses
-its network connectivity.
-
-### docker --net=host
-
-Docker host network support (`docker --net=host run`) is not supported.
-It is not possible to directly access the host networking configuration
-from within the VM.
-
-The `--net=host` option can still be used with `runc` containers and
-inter-mixed with running Kata Containers, thus enabling use of `--net=host`
-when necessary.
-
-It should be noted, currently passing the `--net=host` option into a
-Kata Container may result in the Kata Container networking setup
-modifying, re-configuring and therefore possibly breaking the host
-networking setup. Do not use `--net=host` with Kata Containers.
-
-### docker run --link
-
-The runtime does not support the `docker run --link` command. This
-command is now deprecated by docker and we have no intention of adding support.
-Equivalent functionality can be achieved with the newer docker networking commands.
-
-See more documentation at
-[docs.docker.com](https://docs.docker.com/engine/userguide/networking/default_network/dockerlinks/).
-
 ## Storage limitations
 
 ### Kubernetes `volumeMounts.subPaths`
@@ -158,15 +120,11 @@ moment.
 See [this issue](https://github.com/kata-containers/runtime/issues/2812) for more details.
 [Another issue](https://github.com/kata-containers/kata-containers/issues/1728) focuses on the case of `emptyDir`.
 
-
 ## Host resource sharing
 
-### docker run --privileged
+### Privileged containers
 
 Privileged support in Kata is essentially different from `runc` containers.
-Kata does support `docker run --privileged` command, but in this case full access
-to the guest VM is provided in addition to some host access.
-
 The container runs with elevated capabilities within the guest and is granted
 access to guest devices instead of the host devices.
 This is also true with using `securityContext privileged=true` with Kubernetes.
@@ -176,17 +134,6 @@ The container may also be granted full access to a subset of host devices
 
 See [Privileged Kata Containers](how-to/privileged.md) for how to configure some of this behavior.
 
-# Miscellaneous
-
-This section lists limitations where the possible solutions are uncertain.
-
-## Docker --security-opt option partially supported
-
-The `--security-opt=` option used by Docker is partially supported.
-We only support `--security-opt=no-new-privileges` and `--security-opt seccomp=/path/to/seccomp/profile.json`
-option as of today.
-
-Note: The `--security-opt apparmor=your_profile` is not yet supported. See https://github.com/kata-containers/runtime/issues/707.
 # Appendices
 
 ## The constraints challenge

docs/README.md

@@ -21,14 +21,13 @@ See the [tracing documentation](tracing.md).
 * [Limitations](Limitations.md): differences and limitations compared with the default [Docker](https://www.docker.com/) runtime,
 [`runc`](https://github.com/opencontainers/runc).
 
-### Howto guides
+### How-to guides
 
-See the [howto documentation](how-to).
+See the [how-to documentation](how-to).
 
 ## Kata Use-Cases
 
 * [GPU Passthrough with Kata](./use-cases/GPU-passthrough-and-Kata.md)
-* [OpenStack Zun with Kata Containers](./use-cases/zun_kata.md)
 * [SR-IOV with Kata](./use-cases/using-SRIOV-and-kata.md)
 * [Intel QAT with Kata](./use-cases/using-Intel-QAT-and-kata.md)
 * [VPP with Kata](./use-cases/using-vpp-and-kata.md)
@@ -49,7 +48,7 @@ Documents that help to understand and contribute to Kata Containers.
 ### How to Contribute
 
 * [Developer Guide](Developer-Guide.md): Setup the Kata Containers developing environments
-* [How to contribute to Kata Containers](https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md)
+* [How to contribute to Kata Containers](https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md)
 * [Code of Conduct](../CODE_OF_CONDUCT.md)
 
 ## Help Writing a Code PR
@@ -59,6 +58,7 @@ Documents that help to understand and contribute to Kata Containers.
 ## Help Writing Unit Tests
 
 * [Unit Test Advice](Unit-Test-Advice.md)
+* [Unit testing presentation](presentations/unit-testing/kata-containers-unit-testing.md)
 
 ## Help Improving the Documents
 
@@ -73,6 +73,10 @@ Documents that help to understand and contribute to Kata Containers.
 * [Release strategy](Stable-Branch-Strategy.md)
 * [Release Process](Release-Process.md)
 
+## Presentations
+
+* [Presentations](presentations)
+
 ## Website Changes
 
 If you have a suggestion for how we can improve the

docs/Release-Process.md

@@ -4,11 +4,11 @@
 ## Requirements
 
 - [hub](https://github.com/github/hub)
-  * Using an [application token](https://github.com/settings/tokens) is required for hub.
+  * Using an [application token](https://github.com/settings/tokens) is required for hub (set to a GITHUB_TOKEN environment variable).
 
 - GitHub permissions to push tags and create releases in Kata repositories.
 
-- GPG configured to sign git tags. https://help.github.com/articles/generating-a-new-gpg-key/
+- GPG configured to sign git tags. https://docs.github.com/en/authentication/managing-commit-signature-verification/generating-a-new-gpg-key
 
 - You should configure your GitHub to use your ssh keys (to push to branches). See https://help.github.com/articles/adding-a-new-ssh-key-to-your-github-account/.
     * As an alternative, configure hub to push and fork with HTTPS, `git config --global hub.protocol https` (Not tested yet) *
@@ -48,6 +48,7 @@
 ### Merge all bump version Pull requests
 
   - The above step will create a GitHub pull request in the Kata projects. Trigger the CI using `/test` command on each bump Pull request.
+  - Trigger the `test-kata-deploy` workflow which is under the `Actions` tab on the repository GitHub page (make sure to select the correct branch and validate it passes).
   - Check any failures and fix if needed.
   - Work with the Kata approvers to verify that the CI works and the pull requests are merged.
 
@@ -64,7 +65,7 @@
 
 ### Check Git-hub Actions
 
-  We make use of [GitHub actions](https://github.com/features/actions) in this [file](https://github.com/kata-containers/kata-containers/blob/main/.github/workflows/release.yaml) in the `kata-containers/kata-containers` repository to build and upload release artifacts. This action is auto triggered with the above step when a new tag is pushed to the `kata-containers/kata-containers` repository.
+  We make use of [GitHub actions](https://github.com/features/actions) in this [file](../.github/workflows/release.yaml) in the `kata-containers/kata-containers` repository to build and upload release artifacts. This action is auto triggered with the above step when a new tag is pushed to the `kata-containers/kata-containers` repository.
 
   Check the [actions status page](https://github.com/kata-containers/kata-containers/actions) to verify all steps in the actions workflow have completed successfully. On success, a static tarball containing Kata release artifacts will be uploaded to the [Release page](https://github.com/kata-containers/kata-containers/releases).
 

docs/Unit-Test-Advice.md

@@ -337,7 +337,7 @@ will run if the correct type of user is detected and skipped if not.
 
 The main repository has the most comprehensive set of skip abilities. See:
 
-- https://github.com/kata-containers/kata-containers/tree/main/src/runtime/pkg/katatestutils
+- [`katatestutils`](../src/runtime/pkg/katatestutils)
 
 ### Run Rust tests as a different user
 

docs/code-pr-advice.md

@@ -154,7 +154,7 @@ func testFoo() error {
 ### Tracing
 
 Consider if the code needs to create a new
-[trace span](https://github.com/kata-containers/kata-containers/blob/main/docs/tracing.md).
+[trace span](./tracing.md).
 
 Ensure any new trace spans added to the code are completed.
 

docs/design/README.md

@@ -10,6 +10,7 @@ Kata Containers design documents:
 - [Host cgroups](host-cgroups.md)
 - [`Inotify` support](inotify.md)
 - [Metrics(Kata 2.0)](kata-2-0-metrics.md)
+- [Design for Kata Containers `Lazyload` ability with `nydus`](kata-nydus-design.md)
 
 ---
 

docs/design/arch-images/kata-nydus.drawio

@@ -0,0 +1 @@
+<mxfile host="app.diagrams.net" modified="2022-01-18T14:06:01.890Z" agent="5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36" etag="nId-8OV6FDjWTDgzqDu-" version="15.8.9" type="device"><diagram id="bkF_ZONM9sPFCpIYoGFl" name="Page-1">5Vtbj6M2GP01eUyEbW55nGSmM2q70mqnUnf6UrnBCdYSnIIzSfbX14AJYDsLSSDZUWdHGvxhDD7n8F1sdoTm6/1zgjfhJxaQaAStYD9CjyMIgQ1d8SezHAqL504LwyqhgexUGV7pdyKNlrRuaUDSRkfOWMTppmlcsDgmC96w4SRhu2a3JYuad93gFdEMrwsc6dY/acDDwupDr7K/ELoKyzuDcn5rXHaWM0lDHLBdzYSeRmieMMaLo/V+TqIMvBKX4rpfTpw9PlhCYt7lgpd4/J1SSP9++bR8Cb6A1de/XsZ2Mco7jrZywiFLuXxgfihR4GQv7jEL+ToSBiAOU56wb2TOIpYIS8xi0XO2pFGkmHBEV7FoLsRTEmGfvZOEU4HvgzyxpkGQ3Wa2Cyknrxu8yO65E2oStoRt44BkE7BES5+xBCEbk+xrJonAM2FrwpOD6CLP+o5kQ8rRsZ2ivavItWWXsMZrSSKWclodR64QFwcSdDMB8ycYj+f+Hw/+b3jxvN57e/vXsa8RoIHfBMEEU42XJYu5fIugfeSplC5QSBpBtHSyf/LKmr340ZgWZ9z858iHBr6BopN8INDkAwGdj6llIMSxh2JkamDEjbhEqEGN+++WlSfGaY76g+gA3c2+OimOVtnf+BBs03Ea400aMp69DHJY8ZTFyEW/H/AP+uC/D9aQNbFAkzjDiwQ8A3H+ULyVSrqCOARNxInQwjGNSRIMzth0OMacCYJN14csnTFnOkG+Tpo3GGnAQJqCJomDhyySZ1EkwmlKFzlKOOG6uYZr023WUBYTRDOBW3L4mp2cOGXzTV6ZNx738sqidWjEIBJoWYMWlFK2TRakg2DFTFaEt3kkndoab47JQ0pbQiLM6XvzeU1Eyjt8ZjR/W0rluErELD10OUQxT3lVPf9QBrIVV2+7ykAFDtpAua6O075Cauh6x97iH8ZpSNfjb5jj8TscxFn04Aocx2n3A65BUMM5AT0L7c+lwqFcqg8UHKEeAVGJdSOXdAYD0rle4tOTucvw4W8wrhyvyZU7NWQr0KB5dzCq3OupMqaZufcRVWnOzwfNVnxbiTlTg4tCP4h5/dPlXZin1KA7phxjkT3DRtZhTbxj+0Tikbc+k4SKCWWFdGHcU/61HF4cv1UJjWhVI2WNITIYdM/MxIOKStSEomtmosrNVVOcoTOTDosAncWl5LNWm6ykgirVvNX0dCMFdciBC0ruJjWkKAReKjWnZaCBpQZNRfLFUmu6sFYPdmdn1bXcuq9Xc1WFqClIV6mpA3nWjaV2aWlfl9oFkql5QgvYTYkC95Ioexd/Z/9MoVWLiJ39HWiJ0UOLEBpEeF6aDXxTmr3akrRzhv0zbZ9cl5grcdBxJL732j6BpqWDM/k1llHFNthHordZifn9EA6A4gmQYZXjtozraxxzoFFyaU2bB4hBalpggROpX1tRO9gaBNTXILLt6GX6IeH0O8KJBoNTXyOg6+zzAhGOPw6sSi3sGTZkgWlDdjhYTdXxmS7eMbn4NBSwBDQZZJ2s9OwRWfJ+qJmq+bxxq/yGxKAOteStNzc0t2BC6aZeodx1/d/LV0kdfeve8jXtB95ZvtNpO0i3VW+Hrbm2Iv70RjysL0DWS/xbrQkVL+e9qmzfP8H2uVW2Fhrs21bZyLTv2K9KykWd4wJkvx9rtK7HFFnIvZQCLNiXVFxVKt7kxmLRq47yo7g8mpmL63Mrahm4TtbTqXDjNF79nnd7tCvLF0leZmLi8mWUazYUFxIxwmyT4ZIj5czEr0Bznq1IOuJZ56INqrb4zbonfM5i8fiY5pojOOW7bO0okzzHHP+Tz1Sv4HvLiFzHLJ2adD3DZwrDxZRet7vO24MIcBoe43mP7qEQ9f3cg6VwrC6/dHUP6kYXALA//yCa1efuRffqPw2gp/8A</diagram></mxfile>

docs/design/kata-2-0-metrics.md

@@ -51,6 +51,7 @@ The `kata-monitor` management agent should be started on each node where the Kat
 > **Note**: a *node* running Kata containers will be either a single host system or a worker node belonging to a K8s cluster capable of running Kata pods.
 
 - Aggregate sandbox metrics running on the node, adding the `sandbox_id` label to them.
+- Attach the additional `cri_uid`, `cri_name` and `cri_namespace` labels to the sandbox metrics, tracking the `uid`, `name` and `namespace` Kubernetes pod metadata.
 - Expose a new Prometheus target, allowing all node metrics coming from the Kata shim to be collected by Prometheus indirectly. This simplifies the targets count in Prometheus and avoids exposing shim's metrics by `ip:port`.
 
 Only one `kata-monitor` process runs in each node.

docs/design/kata-nydus-design.md

@@ -0,0 +1,93 @@
+# Background
+
+[Research](https://www.usenix.org/conference/fast16/technical-sessions/presentation/harter) shows that time to take for pull operation accounts for 76% of container startup time but only 6.4% of that data is read. So if we can get data on demand (lazy load), it will speed up the container start. [`Nydus`](https://github.com/dragonflyoss/image-service) is a project which build image with new format and can get data on demand when container start.
+
+The following benchmarking result shows the performance improvement compared with the OCI image for the container cold startup elapsed time on containerd. As the OCI image size increases, the container startup time of using `nydus` image remains very short. [Click here](https://github.com/dragonflyoss/image-service/blob/master/docs/nydus-design.md) to see `nydus` design.
+
+![`nydus`-performance](arch-images/nydus-performance.png)
+
+## Proposal - Bring `lazyload` ability to Kata Containers
+
+`Nydusd` is a fuse/`virtiofs` daemon which is provided by `nydus` project and it supports `PassthroughFS` and [RAFS](https://github.com/dragonflyoss/image-service/blob/master/docs/nydus-design.md) (Registry Acceleration File System) natively, so in Kata Containers, we can use `nydusd` in place of `virtiofsd` and mount `nydus` image to guest in the meanwhile.
+
+The process of creating/starting Kata Containers with `virtiofsd`,
+
+1. When creating sandbox, the Kata Containers Containerd v2 [shim](https://github.com/kata-containers/kata-containers/blob/main/docs/design/architecture/README.md#runtime) will launch `virtiofsd` before VM starts and share directories with VM.
+2. When creating container, the Kata Containers Containerd v2 shim will mount rootfs to `kataShared`(/run/kata-containers/shared/sandboxes/\<SANDBOX\>/mounts/\<CONTAINER\>/rootfs), so it can be seen at the path `/run/kata-containers/shared/containers/shared/\<CONTAINER\>/rootfs` in the guest and used as container's rootfs.
+
+The process of creating/starting Kata Containers with `nydusd`,
+
+![kata-`nydus`](arch-images/kata-nydus.png)
+
+1. When creating sandbox, the Kata Containers Containerd v2 shim will launch `nydusd` daemon before VM starts.
+After VM starts, `kata-agent` will mount `virtiofs` at the path `/run/kata-containers/shared` and Kata Containers Containerd v2 shim mount `passthroughfs` filesystem to path `/run/kata-containers/shared/containers` when the VM starts.
+
+```bash
+# start nydusd
+$ sandbox_id=my-test-sandbox
+$ sudo /usr/local/bin/nydusd --log-level info --sock /run/vc/vm/${sandbox_id}/vhost-user-fs.sock --apisock /run/vc/vm/${sandbox_id}/api.sock
+```
+
+```bash
+# source: the host sharedir which will pass through to guest
+$ sudo curl -v --unix-socket /run/vc/vm/${sandbox_id}/api.sock \
+    -X POST "http://localhost/api/v1/mount?mountpoint=/containers" -H "accept: */*" \
+    -H "Content-Type: application/json" \
+    -d '{
+            "source":"/path/to/sharedir",
+            "fs_type":"passthrough_fs",
+            "config":""
+    }'
+```
+
+2. When creating normal container, the Kata Containers Containerd v2 shim send request to `nydusd` to mount `rafs` at the path `/run/kata-containers/shared/rafs/<container_id>/lowerdir` in guest.
+
+```bash
+# source: the metafile of nydus image
+# config: the config of this image
+$ sudo curl --unix-socket /run/vc/vm/${sandbox_id}/api.sock \
+    -X POST "http://localhost/api/v1/mount?mountpoint=/rafs/<container_id>/lowerdir" -H "accept: */*" \
+    -H "Content-Type: application/json" \
+    -d '{
+            "source":"/path/to/bootstrap",
+            "fs_type":"rafs",
+            "config":"config":"{\"device\":{\"backend\":{\"type\":\"localfs\",\"config\":{\"dir\":\"blobs\"}},\"cache\":{\"type\":\"blobcache\",\"config\":{\"work_dir\":\"cache\"}}},\"mode\":\"direct\",\"digest_validate\":true}",
+    }'
+```
+
+The Kata Containers Containerd v2 shim will also bind mount `snapshotdir` which `nydus-snapshotter` assigns to `sharedir`。
+So in guest, container rootfs=overlay(`lowerdir=rafs`, `upperdir=snapshotdir/fs`, `workdir=snapshotdir/work`)
+
+> how to transfer the `rafs` info from `nydus-snapshotter` to the Kata Containers Containerd v2 shim?
+
+By default, when creating `OCI` image container, `nydus-snapshotter` will return [`struct` Mount slice](https://github.com/containerd/containerd/blob/main/mount/mount.go#L21) below to containerd and containerd use them to mount rootfs
+
+```
+[
+    {
+        Type: "overlay",
+        Source: "overlay",
+        Options: [lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/<snapshot_A>/mnt,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/<snapshot_B>/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/<snapshot_B>/work],
+    }
+]
+```
+
+Then, we can append `rafs` info into `Options`, but if do this, containerd will mount failed, as containerd can not identify `rafs` info. Here, we can refer to [containerd mount helper](https://github.com/containerd/containerd/blob/main/mount/mount_linux.go#L42) and provide a binary called `nydus-overlayfs`. The `Mount` slice which `nydus-snapshotter` returned becomes
+
+```
+[
+    {
+        Type: "fuse.nydus-overlayfs",
+        Source: "overlay",
+        Options: [lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/<snapshot_A>/mnt,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/<snapshot_B>/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/<snapshot_B>/work,extraoption=base64({source:xxx,config:xxx,snapshotdir:xxx})],
+    }
+]
+```
+
+When containerd find `Type` is `fuse.nydus-overlayfs`,
+
+1. containerd will call `mount.fuse` command;
+2. in `mount.fuse`, it will call `nydus-overlayfs`.
+3. in `nydus-overlayfs`, it will ignore the `extraoption` and do the overlay mount.
+
+Finally, in the Kata Containers Containerd v2 shim, it parse `extraoption` and get the `rafs` info to mount the image in guest.

docs/how-to/README.md

@@ -37,3 +37,4 @@
 - [How to setup swap devices in guest kernel](how-to-setup-swap-devices-in-guest-kernel.md)
 - [How to run rootless vmm](how-to-run-rootless-vmm.md)
 - [How to run Docker with Kata Containers](how-to-run-docker-with-kata.md)
+- [How to run Kata Containers with `nydus`](how-to-use-virtio-fs-nydus-with-kata.md)

docs/how-to/how-to-import-kata-logs-with-fluentd.md

@@ -4,7 +4,7 @@
 
 This document describes how to import Kata Containers logs into [Fluentd](https://www.fluentd.org/),
 typically for importing into an
-Elastic/Fluentd/Kibana([EFK](https://github.com/kubernetes/kubernetes/tree/master/cluster/addons/fluentd-elasticsearch#running-efk-stack-in-production))
+Elastic/Fluentd/Kibana([EFK](https://github.com/kubernetes-sigs/instrumentation-addons/tree/master/fluentd-elasticsearch#running-efk-stack-in-production))
 or Elastic/Logstash/Kibana([ELK](https://www.elastic.co/elastic-stack)) stack.
 
 The majority of this document focusses on CRI-O based (classic) Kata runtime. Much of that information
@@ -257,7 +257,7 @@ go directly to a full Kata specific JSON format logfile test.
 
 Kata runtime has the ability to generate JSON logs directly, rather than its default `logfmt` format. Passing
 the `--log-format=json` argument to the Kata runtime enables this. The easiest way to pass in this extra
-parameter from a [Kata deploy](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy) installation
+parameter from a [Kata deploy](../../tools/packaging/kata-deploy) installation
 is to edit the `/opt/kata/bin/kata-qemu` shell script.
 
 At the same time, we will add the `--log=/var/log/kata-runtime.log` argument to store the Kata logs in their

docs/how-to/how-to-run-docker-with-kata.md

@@ -22,7 +22,7 @@ You can learn more about about Docker-in-Docker at the following links:
 - [`docker` image Docker Hub page](https://hub.docker.com/_/docker/) (this page lists the `-dind` releases)
 
 While normally DinD refers to running `docker` from inside a Docker container,
-Kata Containers 2.x allows only supported runtimes (such as [`containerd`](../install/container-manager/containerd/containerd-install.md)).
+Kata Containers 2.x allows only [supported runtimes][kata-2.x-supported-runtimes] (such as [`containerd`](../install/container-manager/containerd/containerd-install.md)).
 
 Running `docker` in a Kata Container implies creating Docker containers from inside a container managed by `containerd` (or another supported container manager), as illustrated below:
 
@@ -37,7 +37,7 @@ container manager -> Kata Containers shim     -> Docker Daemon -> Docker contain
 
 [OverlayFS]: https://www.kernel.org/doc/html/latest/filesystems/overlayfs.html
 [v2.0.0]: https://github.com/kata-containers/kata-containers/releases/tag/2.0.0
-[kata-2.x-supported-runtimes]: https://github.com/kata-containers/kata-containers/blob/5737b36a3513f4da11a9dc7301b0c97ea22a51cf/docs/install/container-manager/containerd/containerd-install.md
+[kata-2.x-supported-runtimes]: ../install/container-manager/containerd/containerd-install.md
 
 ## Why Docker in Kata Containers 2.x requires special measures
 

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -62,6 +62,8 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.file_mem_backend` (R) | string | file based memory backend root directory |
 | `io.katacontainers.config.hypervisor.firmware_hash` | string | container firmware SHA-512 hash value |
 | `io.katacontainers.config.hypervisor.firmware` | string | the guest firmware that will run the container VM |
+| `io.katacontainers.config.hypervisor.firmware_volume_hash` | string | container firmware volume SHA-512 hash value |
+| `io.katacontainers.config.hypervisor.firmware_volume` | string | the guest firmware volume that will be passed to the container VM |
 | `io.katacontainers.config.hypervisor.guest_hook_path` | string | the path within the VM that will be used for drop in hooks |
 | `io.katacontainers.config.hypervisor.hotplug_vfio_on_root_bus` | `boolean` | indicate if devices need to be hotplugged on the root bus instead of a bridge|
 | `io.katacontainers.config.hypervisor.hypervisor_hash` | string | container hypervisor binary SHA-512 hash value |

docs/how-to/how-to-use-virtio-fs-nydus-with-kata.md

@@ -0,0 +1,57 @@
+# Kata Containers with virtio-fs-nydus
+
+## Introduction
+
+Refer to [kata-`nydus`-design](../design/kata-nydus-design.md) for introduction and `nydus` has supported Kata Containers with hypervisor `QEMU` and `CLH` currently.
+
+## How to
+
+You can use Kata Containers with `nydus` as follows,
+
+1. Use [`nydus` latest branch](https://github.com/dragonflyoss/image-service);
+
+2. Deploy `nydus` environment as [`Nydus` Setup for Containerd Environment](https://github.com/dragonflyoss/image-service/blob/master/docs/containerd-env-setup.md);
+
+3. Start `nydus-snapshotter` with `enable_nydus_overlayfs` enabled;
+
+4. Use [kata-containers](https://github.com/kata-containers/kata-containers) `latest` branch to compile and build `kata-containers.img`;
+
+5. Update `configuration-qemu.toml` or `configuration-clh.toml`to include:
+
+```toml
+shared_fs = "virtio-fs-nydus"
+virtio_fs_daemon = "<nydusd binary path>"
+virtio_fs_extra_args = []
+```
+
+6. run `crictl run -r kata nydus-container.yaml nydus-sandbox.yaml`;
+
+The `nydus-sandbox.yaml` looks like below:
+
+```yaml
+metadata:
+  attempt: 1
+  name: nydus-sandbox
+  namespace: default
+log_directory: /tmp
+linux:
+  security_context:
+    namespace_options:
+      network: 2
+annotations:
+  "io.containerd.osfeature": "nydus.remoteimage.v1"
+```
+
+The `nydus-container.yaml` looks like below:
+
+```yaml
+metadata:
+  name: nydus-container
+image:
+  image: localhost:5000/ubuntu-nydus:latest
+command:
+  - /bin/sleep
+args:
+  - 600
+log_path: container.1.log
+```

docs/how-to/how-to-use-virtio-fs-with-kata.md

@@ -6,4 +6,4 @@ Container deployments utilize explicit or implicit file sharing between host fil
 
 As of the 2.0 release of Kata Containers, [virtio-fs](https://virtio-fs.gitlab.io/) is the default filesystem sharing mechanism.
 
-virtio-fs support works out of the box for `cloud-hypervisor` and `qemu`, when Kata Containers is deployed using `kata-deploy`. Learn more about `kata-deploy` and how to use `kata-deploy` in Kubernetes [here](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy#kubernetes-quick-start).
+virtio-fs support works out of the box for `cloud-hypervisor` and `qemu`, when Kata Containers is deployed using `kata-deploy`. Learn more about `kata-deploy` and how to use `kata-deploy` in Kubernetes [here](../../tools/packaging/kata-deploy/README.md#kubernetes-quick-start).

docs/how-to/run-kata-with-k8s.md

@@ -104,26 +104,69 @@ $ sudo kubeadm init --ignore-preflight-errors=all --cri-socket /run/containerd/c
 $ export KUBECONFIG=/etc/kubernetes/admin.conf
 ```
 
-You can force Kubelet to use Kata Containers by adding some `untrusted`
-annotation to your pod configuration. In our case, this ensures Kata
-Containers is the selected runtime to run the described workload.
+### Allow pods to run in the master node
 
-`nginx-untrusted.yaml`
-```yaml
-apiVersion: v1
-kind: Pod
+By default, the cluster will not schedule pods in the master node. To enable master node scheduling:
+```bash
+$ sudo -E kubectl taint nodes --all node-role.kubernetes.io/master-
+```
+
+### Create runtime class for Kata Containers
+
+Users can use [`RuntimeClass`](https://kubernetes.io/docs/concepts/containers/runtime-class/#runtime-class) to specify a different runtime for Pods.
+
+```bash
+$ cat > runtime.yaml <<EOF
+apiVersion: node.k8s.io/v1
+kind: RuntimeClass
 metadata:
-  name: nginx-untrusted
-  annotations:
-    io.kubernetes.cri.untrusted-workload: "true"
-spec:
-  containers:
+  name: kata
+handler: kata
+EOF
+
+$ sudo -E kubectl apply -f runtime.yaml
+```
+
+### Run pod in Kata Containers
+
+If a pod has the `runtimeClassName` set to `kata`, the CRI plugin runs the pod with the
+[Kata Containers runtime](../../src/runtime/README.md).
+
+- Create an pod configuration that using Kata Containers runtime
+
+  ```bash
+  $ cat << EOF | tee nginx-kata.yaml
+  apiVersion: v1
+  kind: Pod
+  metadata:
+    name: nginx-kata
+  spec:
+    runtimeClassName: kata
+    containers:
     - name: nginx
       image: nginx
-```
 
-Next, you run your pod:
-```
-$ sudo -E kubectl apply -f nginx-untrusted.yaml
-```
+  EOF
+  ```
 
+- Create the pod
+  ```bash
+  $ sudo -E kubectl apply -f nginx-kata.yaml
+  ```
+
+- Check pod is running
+
+  ```bash
+  $ sudo -E kubectl get pods
+  ```
+
+- Check hypervisor is running
+  ```bash
+  $ ps aux | grep qemu
+  ```
+
+### Delete created pod
+
+```bash
+$ sudo -E kubectl delete -f nginx-kata.yaml
+```

docs/install/minikube-installation-guide.md

@@ -6,7 +6,7 @@
 cluster locally. It creates a single node Kubernetes stack in a local VM.
 
 [Kata Containers](https://github.com/kata-containers) can be installed into a Minikube cluster using
-[`kata-deploy`](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy).
+[`kata-deploy`](../../tools/packaging/kata-deploy).
 
 This document details the pre-requisites, installation steps, and how to check
 the installation has been successful.
@@ -123,7 +123,7 @@ $ kubectl apply -f kata-deploy/base/kata-deploy.yaml
 This installs the Kata Containers components into `/opt/kata` inside the Minikube node. It can take
 a few minutes for the operation to complete. You can check the installation has worked by checking
 the status of the `kata-deploy` pod, which will be executing
-[this script](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy/scripts/kata-deploy.sh),
+[this script](../../tools/packaging/kata-deploy/scripts/kata-deploy.sh),
 and will be executing a `sleep infinity` once it has successfully completed its work.
 You can accomplish this by running the following:
 

docs/install/snap-installation-guide.md

@@ -39,8 +39,8 @@ can be used as runtime.
 
 Read the following documents to know how to run Kata Containers 2.x with `containerd`.
 
-* [How to use Kata Containers and Containerd](https://github.com/kata-containers/kata-containers/blob/main/docs/how-to/containerd-kata.md)
-* [Install Kata Containers with containerd](https://github.com/kata-containers/kata-containers/blob/main/docs/install/container-manager/containerd/containerd-install.md)
+* [How to use Kata Containers and Containerd](../how-to/containerd-kata.md)
+* [Install Kata Containers with containerd](./container-manager/containerd/containerd-install.md)
 
 
 ## Remove Kata Containers snap package

docs/presentations/README.md

@@ -0,0 +1,3 @@
+# Kata Containers presentations
+
+* [Unit testing](unit-testing)

docs/presentations/unit-testing/README.md

@@ -0,0 +1,14 @@
+# Kata Containers unit testing presentation
+
+## Markdown version
+
+See [the Kata Containers unit testing presentation](kata-containers-unit-testing.md).
+
+### To view as an HTML presentation
+
+```bash
+$ infile="kata-containers-unit-testing.md"
+$ outfile="/tmp/kata-containers-unit-testing.html"
+$ pandoc -s --metadata title="Kata Containers unit testing" -f markdown -t revealjs --highlight-style="zenburn" -i -o "$outfile" "$infile"
+$ xdg-open "file://$outfile"
+```

docs/presentations/unit-testing/kata-containers-unit-testing.md

@@ -0,0 +1,335 @@
+## Why write unit tests?
+
+- Catch regressions
+
+- Improve the code being tested
+
+  Structure, quality, security, performance, "shakes out" implicit
+  assumptions, _etc_
+
+- Extremely instructive
+
+  Once you've fully tested a single function, you'll understand that
+  code very well indeed.
+
+## Why write unit tests? (continued)
+
+- Fun!
+
+  Yes, really! Don't believe me? Try it! ;)
+
+## Run all Kata Containers agent unit tests
+
+As an example, to run all agent unit tests:
+
+```bash
+$ cd $GOPATH/src/github.com/kata-containers/kata-containers
+$ cd src/agent
+$ make test
+```
+
+## List all unit tests
+
+- Identify the full name of all the tests _in the current package_:
+
+  ```bash
+  $ cargo test -- --list
+  ```
+
+- Identify the full name of all tests in the `foo` "local crate"
+  (sub-directory containing another `Cargo.toml` file):
+
+  ```bash
+  $ cargo test -p "foo" -- --list
+  ```
+
+## Run a single unit test
+
+- Run a test in the current package in verbose mode:
+
+  ```bash
+  # Example 
+  $ test="config::tests::test_get_log_level"
+
+  $ cargo test "$test" -vv -- --exact --nocapture
+  ```
+
+## Test coverage setup
+
+```bash
+$ cargo install cargo-tarpaulin
+```
+
+## Show test coverage
+
+```bash
+$ cd $GOPATH/src/github.com/kata-containers/kata-containers/src/agent
+$ cargo -v tarpaulin --all-features --run-types AllTargets --count --force-clean -o Html
+$ xdg-open "file://$PWD/tarpaulin-report.html"
+```
+
+## Testability (part 1)
+
+- To be testable, a function should:
+  - Not be "too long" (say >100 lines).
+  - Not be "too complex" (say >3 levels of indentation).
+  - Should return a `Result` or an `Option` so error paths
+    can be tested.
+
+- If functions don't conform, they need to be reworked (refactored)
+  before writing tests.
+
+## Testability (part 2)
+
+- Some functions can't be fully tested.
+- However, you _can_ test the initial code that checks
+  the parameter values (test error paths only).
+
+## Writing new tests: General advice (part 1)
+
+- KISS: Keep It Simple Stupid
+
+  You don't get extra points for cryptic code.
+
+- DRY: Don't Repeat Yourself
+
+  Make use of existing facilities (don't "re-invert the wheel").
+
+- Read the [unit test advice document](https://github.com/kata-containers/kata-containers/blob/main/docs/Unit-Test-Advice.md)
+
+## Writing new tests: General advice (part 2)
+
+- Attack the function in all possible ways
+
+- Use the _table driven_ approach:
+  - Simple
+  - Compact
+  - Easy to debug
+  - Makes boundary analysis easy
+  - Encourages functions to be testable
+
+## Writing new tests: Specific advice (part 1)
+
+- Create a new "`tests`" module if necessary.
+- Give each test function a "`test_`" prefix.
+- Add the "`#[test]`" annotation on each test function.
+
+## Writing new tests: Specific advice (part 2)
+
+- If you need to `use` (import) packages for the tests,
+  _only do it in the `tests` module_:
+  ```rust
+  use some_test_pkg::{foo, bar}; // <-- Not here
+
+  #[cfg(test)]
+  mod tests {
+    use super::*;
+    use some_test_pkg:{foo, bar}; // <-- Put it here
+  }
+  ```
+
+## Writing new tests: Specific advice (part 3)
+
+- You can add test-specific dependencies in `Cargo.toml`:
+  ```toml
+  [dev-dependencies]
+  serial_test = "0.5.1"
+  ```
+
+## Writing new tests: Specific advice (part 4)
+
+- Don't add in lots of error handling code: let the test panic!
+  ```rust
+  // This will panic if the unwrap fails.
+  // - NOT acceptable generally for production code.
+  // - PERFECTLY acceptable for test code since:
+  //   - Keeps the test code simple.
+  //   - Rust will detect the panic and fail the test.
+  let result = func().unwrap();
+  ```
+
+## Debugging tests (part 1)
+
+- Comment out all tests in your `TestData` array apart from the failing test.
+
+- Add temporary `println!("FIXME: ...")` statements in the code.
+
+- Set `RUST_BACKTRACE=full` before running `cargo test`.
+
+## Debugging tests (part 2)
+
+- Use a debugger (not normally necessary though):
+  ```bash
+  # Disable optimisation
+  $ RUSTFLAGS="-C opt-level=0" cargo test --no-run
+
+  # Find the test binary
+  $ test_binary=$(find target/debug/deps | grep "kata_agent-[a-z0-9][a-z0-9]*$" | tail -1)
+
+  $ rust-gdb "$test_binary"
+  ```
+
+## Useful tips
+
+- Always start a test with a "clean environment":
+
+  Create new set of objects / files / directories / _etc_
+  for each test.
+
+- Mounts
+  - Linux allows mounts on top of existing mounts.
+  - Bind mounts and read-only mounts can be useful.
+
+## Gotchas (part 1)
+
+If a test runs successfully _most of the time_:
+
+- Review the test logic.
+
+- Add a `#[serial]` annotation on the test function
+  Requires the `serial_test` package in the `[dev-dependencies]`
+  section of `Cargo.toml`.
+
+  If this makes it work the test is probably sharing resources with
+  another task (thread).
+
+## Gotchas (part 2)
+
+If a test works locally but fails in the CI, consider the following
+attributes of each environment (local and CI):
+
+- The version of rust being used.
+- The hardware architecture.
+- Number (and spec) of the CPUs.
+
+## Gotchas (part 3)
+
+If in doubt, look at the
+["test artifacts" attached to the failing CI test](http://jenkins.katacontainers.io).
+
+## Before raising a PR
+
+- Remember to check that the test runs locally:
+  - As a non-privileged user.
+  - As the `root` user (carefully!)
+
+- Run the [static checker](https://github.com/kata-containers/tests/blob/main/.ci/static-checks.sh)
+  on your changes.
+
+  Checks formatting and many other things.
+
+## If in doubt
+
+- Ask for help! ;)
+
+## Quiz 1
+
+What's wrong with this function?
+
+```rust
+fn foo(config: &Config, path_prefix: String, container_id: String, pid: String) -> Result<()> {
+    let mut full_path = format!("{}/{}", path_prefix, container_id);
+
+    let _ = remove_recursively(&mut full_path);
+
+    write_number_to_file(pid, full_path);
+
+    Ok(())
+}
+```
+
+## Quiz 1: Answers (part 1)
+
+- No check that `path_prefix`, `container_id` and `pid` are not `""`.
+- No check that `path_prefix` is absolute.
+- No check that `container_id` does not contain slashes / contains only valid characters.
+- Result of `remove_recursively()` discarded.
+- `remove_recursively()` _may_ modify `full_path` without `foo()` knowing!
+
+## Quiz 1: Answers (part 2)
+
+- Why is `pid` not a numeric?
+- No check to ensure the PID is positive.
+- No check to recreate any directories in the original `path_prefix`.
+- `write_number_to_file()` could fail so why doesn't it return a value?
+- The `config` parameter is unused.
+
+## Quiz 1: What if...
+
+Imagine if the caller managed to do this:
+
+```rust
+foo(config, "", "sbin/init", r#"#!/bin/sh\n/sbin/reboot"#);
+```
+
+## Quiz 2
+
+What makes this function difficult to test?
+
+```rust
+fn get_user_id(username: String) -> i32 {
+    let line = grep_file(username, "/etc/passwd").unwrap();
+    let fields = line.split(':');
+
+    let uid = fields.nth(2).ok_or("failed").unwrap();
+
+    uid.parse::<i32>()
+}
+```
+
+## Quiz 2: Answers (part 1)
+
+- Unhelpful error message ("failed").
+
+- Panics on error! Return a `Result` instead!
+
+- UID's cannot be negative so function should return an unsigned
+  value.
+
+## Quiz 2: Answers (part 2)
+
+- Hard-coded filename.
+
+  This would be better:
+
+  ```rust
+  const PASSWD_DB: &str = "/etc/passwd";
+
+  // Test code can now pass valid and invalid files!
+  fn get_user_id(filename: String, username: String) -> i32 {
+    // ...
+  }
+
+  let id = get_user_id(PASSWD_DB, username);
+  ```
+
+## Quiz 3
+
+What's wrong with this test code?
+
+```rust
+let mut obj = Object::new();
+
+// Sanity check
+assert_eq!(obj.num, 0);
+assert_eq!(obj.wibble, false);
+
+// Test 1
+obj->foo_method(7);
+assert_eq!(obj.num, 7);
+
+// Test 2
+obj->bar_method(true);
+assert_eq!(obj.wibble, true);
+```
+
+## Quiz 3: Answers
+
+- The test code is "fragile":
+  - The 2nd test re-uses the object created in the first test.
+
+## Finally
+
+- [We need a GH action to run the unit tests](https://github.com/kata-containers/kata-containers/issues/2934)
+
+  Needs to fail PRs that decrease test coverage<br/> by "x%".

docs/tracing.md

@@ -203,11 +203,11 @@ is highly recommended. For working with the agent, you may also wish to
 [enable a debug console][setup-debug-console]
 to allow you to access the VM environment.
 
-[enable-full-debug]: https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md#enable-full-debug
+[enable-full-debug]: ./Developer-Guide.md#enable-full-debug
 [jaeger-all-in-one]: https://www.jaegertracing.io/docs/getting-started/
 [jaeger-tracing]: https://www.jaegertracing.io
 [opentelemetry]: https://opentelemetry.io
-[osbuilder]: https://github.com/kata-containers/kata-containers/blob/main/tools/osbuilder
-[setup-debug-console]: https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md#set-up-a-debug-console
+[osbuilder]: ../tools/osbuilder
+[setup-debug-console]: ./Developer-Guide.md#set-up-a-debug-console
 [trace-forwarder]: /src/tools/trace-forwarder
 [vsock]: https://wiki.qemu.org/Features/VirtioVsock

docs/use-cases/using-Intel-QAT-and-kata.md

@@ -231,7 +231,7 @@ $ cp ${GOPATH}/${LINUX_VER}/vmlinux ${KATA_KERNEL_LOCATION}/${KATA_KERNEL_NAME}
 These instructions build upon the OS builder instructions located in the 
 [Developer Guide](../Developer-Guide.md). At this point it is recommended that
 [Docker](https://docs.docker.com/engine/install/ubuntu/) is installed first, and
-then [Kata-deploy](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy)
+then [Kata-deploy](../../tools/packaging/kata-deploy)
 is use to install Kata. This will make sure that the correct `agent` version 
 is installed into the rootfs in the steps below.
 
@@ -419,11 +419,11 @@ You might need to disable Docker before initializing Kubernetes. Be aware
 that the OpenSSL container image built above will need to be exported from
 Docker and imported into containerd.
 
-If Kata is installed through [`kata-deploy`](https://github.com/kata-containers/kata-containers/blob/stable-2.0/tools/packaging/kata-deploy/README.md)
+If Kata is installed through [`kata-deploy`](../../tools/packaging/kata-deploy/README.md)
 there will be multiple `configuration.toml` files associated with different 
 hypervisors. Rather than add in the custom Kata kernel, Kata rootfs, and 
 kernel modules to each `configuration.toml` as the default, instead use
-[annotations](https://github.com/kata-containers/kata-containers/blob/stable-2.0/docs/how-to/how-to-load-kernel-modules-with-kata.md)
+[annotations](../how-to/how-to-load-kernel-modules-with-kata.md)
 in the Kubernetes YAML file to tell Kata which kernel and rootfs to use. The 
 easy way to do this is to use `kata-deploy` which will install the Kata binaries
 to `/opt` and properly configure the `/etc/containerd/config.toml` with annotation 

docs/use-cases/using-Intel-SGX-and-kata.md

@@ -17,24 +17,11 @@ CONFIG_X86_SGX_KVM=y
 ```
 
 * Kubernetes cluster configured with:
-   * [`kata-deploy`](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy) based Kata Containers installation
+   * [`kata-deploy`](../../tools/packaging/kata-deploy) based Kata Containers installation
    * [Intel SGX Kubernetes device plugin](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/main/cmd/sgx_plugin#deploying-with-pre-built-images)
 
 > Note: Kata Containers supports creating VM sandboxes with Intel® SGX enabled
-> using [cloud-hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor/) VMM only. QEMU support is waiting to get the
-> Intel SGX enabled QEMU upstream release.
-
-## Installation
-
-### Kata Containers Guest Kernel
-
-Follow the instructions to [setup](../../tools/packaging/kernel/README.md#setup-kernel-source-code) and [build](../../tools/packaging/kernel/README.md#build-the-kernel) the experimental guest kernel. Then, install as:
-
-```sh
-$ sudo cp kata-linux-experimental-*/vmlinux /opt/kata/share/kata-containers/vmlinux.sgx
-$ sudo sed -i 's|vmlinux.container|vmlinux.sgx|g' \
-  /opt/kata/share/defaults/kata-containers/configuration-clh.toml
-```
+> using [cloud-hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor/) and [QEMU](https://www.qemu.org/) VMMs only.
 
 ### Kata Containers Configuration
 
@@ -48,6 +35,8 @@ to the `sandbox` are: `["io.katacontainers.*", "sgx.intel.com/epc"]`.
 
 With the following sample job deployed using `kubectl apply -f`:
 
+> Note: Change the `runtimeClassName` option accordingly, only `kata-clh` and `kata-qemu` support Intel® SGX.
+
 ```yaml
 apiVersion: batch/v1
 kind: Job

docs/use-cases/zun_kata.md

@@ -1,121 +0,0 @@
-# OpenStack Zun DevStack working with Kata Containers
-
-## Introduction
-
-This guide describes how to get Kata Containers to work with OpenStack Zun
-using DevStack on Ubuntu 16.04. Running DevStack with this guide will setup
-Docker and Clear Containers 2.0, which you replace with Kata Containers.
-Currently, the instructions are based on the following links:
-
-- https://docs.openstack.org/zun/latest/contributor/quickstart.html
-
-- https://docs.openstack.org/zun/latest/admin/clear-containers.html
-
-## Install Git to use with DevStack
-
-```sh
-$ sudo apt install git
-```
-
-## Setup OpenStack DevStack
-The following commands will sync DevStack from GitHub, create your
-`local.conf` file, assign your host IP to this file, enable Clear
-Containers, start DevStack, and set the environment variables to use
-`zun` on the command line.
-
-```sh
-$ sudo mkdir -p /opt/stack
-$ sudo chown $USER /opt/stack
-$ git clone https://github.com/openstack-dev/devstack /opt/stack/devstack
-$ HOST_IP="$(ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1 -d'/')"
-$ git clone https://github.com/openstack/zun /opt/stack/zun
-$ cat /opt/stack/zun/devstack/local.conf.sample \
-$     | sed "s/HOST_IP=.*/HOST_IP=$HOST_IP/" \
-$     > /opt/stack/devstack/local.conf
-$ sed -i "s/KURYR_CAPABILITY_SCOPE=.*/KURYR_CAPABILITY_SCOPE=local/" /opt/stack/devstack/local.conf
-$ echo "ENABLE_CLEAR_CONTAINER=true" >> /opt/stack/devstack/local.conf
-$ echo "enable_plugin zun-ui https://git.openstack.org/openstack/zun-ui" >> /opt/stack/devstack/local.conf
-$ /opt/stack/devstack/stack.sh
-$ source /opt/stack/devstack/openrc admin admin
-```
-
-The previous commands start OpenStack DevStack with Zun support. You can test
-it using `runc` as shown by the following commands to make sure everything
-installed correctly and is working.
-
-```sh
-$ zun run --name test cirros ping -c 4 8.8.8.8
-$ zun list
-$ zun logs test
-$ zun delete test
-```
-
-## Install Kata Containers
-
-Follow [these instructions](../install/README.md)
-to install the Kata Containers components.
-
-## Update Docker with new Kata Containers runtime
-
-The following commands replace the Clear Containers 2.x runtime setup with
-DevStack, with Kata Containers:
-
-```sh
-$ sudo sed -i 's/"cor"/"kata-runtime"/' /etc/docker/daemon.json
-$ sudo sed -i 's/"\/usr\/bin\/cc-oci-runtime"/"\/usr\/bin\/kata-runtime"/' /etc/docker/daemon.json
-$ sudo systemctl daemon-reload
-$ sudo systemctl restart docker
-```
-
-## Test that everything works in both Docker and OpenStack Zun
-
-```sh
-$ sudo docker run -ti --runtime kata-runtime busybox sh
-$ zun run --name kata --runtime kata-runtime cirros ping -c 4 8.8.8.8
-$ zun list
-$ zun logs kata
-$ zun delete kata
-```
-
-## Stop DevStack and clean up system (Optional)
-
-```sh
-$ /opt/stack/devstack/unstack.sh
-$ /opt/stack/devstack/clean.sh
-```
-
-## Restart DevStack and reset CC 2.x runtime to `kata-runtime`
-
-Run the following commands if you already setup Kata Containers and want to
-restart DevStack:
-
-```sh
-$ /opt/stack/devstack/unstack.sh
-$ /opt/stack/devstack/clean.sh
-$ /opt/stack/devstack/stack.sh
-$ source /opt/stack/devstack/openrc admin admin
-$ sudo sed -i 's/"cor"/"kata-runtime"/' /etc/docker/daemon.json
-$ sudo sed -i 's/"\/usr\/bin\/cc-oci-runtime"/"\/usr\/bin\/kata-runtime"/' /etc/docker/daemon.json
-$ sudo systemctl daemon-reload
-$ sudo systemctl restart docker
-```
-
-![Kata Zun image 1](./images/kata-zun1.png)
-
-Figure 1: Create a BusyBox container image
-
-![Kata Zun image 2](./images/kata-zun2.png)
-
-Figure 2: Select `kata-runtime` to use
-
-![Kata Zun image 3](./images/kata-zun3.png)
-
-Figure 3: Two BusyBox containers successfully launched
-
-![Kata Zun image 4](./images/kata-zun4.png)
-
-Figure 4: Test connectivity between Kata Containers
-
-![Kata Zun image 5](./images/kata-zun5.png)
-
-Figure 5: CLI for Zun

snap/snapcraft.yaml

@@ -204,14 +204,7 @@ parts:
       kernel_dir_prefix="kata-linux-"
 
       # Setup and build kernel
-      if [ "$(uname -m)" = "x86_64" ]; then
-        kernel_version="$(${yq} r $versions_file assets.kernel-experimental.tag)"
-        kernel_version=${kernel_version#v}
-        kernel_dir_prefix="kata-linux-experimental-"
-        ./build-kernel.sh -e -v ${kernel_version} -d setup
-      else
-        ./build-kernel.sh -v ${kernel_version} -d setup
-      fi
+      ./build-kernel.sh -v ${kernel_version} -d setup
       cd ${kernel_dir_prefix}*
       make -j $(($(nproc)-1)) EXTRAVERSION=".container"
 
@@ -262,34 +255,21 @@ parts:
       kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
 
       versions_file="${kata_dir}/versions.yaml"
-      # arch-specific definition
-      case "$(uname -m)" in
-        "aarch64")
-          branch="$(${yq} r ${versions_file} assets.hypervisor.qemu.architecture.aarch64.version)"
-          url="$(${yq} r ${versions_file} assets.hypervisor.qemu.url)"
-          commit="$(${yq} r ${versions_file} assets.hypervisor.qemu.architecture.aarch64.commit)"
-          patches_dir="${kata_dir}/tools/packaging/qemu/patches/$(echo ${branch} | sed -e 's/.[[:digit:]]*$//' -e 's/^v//').x"
-          patches_version_dir="${kata_dir}/tools/packaging/qemu/patches/tag_patches/${branch}"
-        ;;
-
-        *)
-          branch="$(${yq} r ${versions_file} assets.hypervisor.qemu.version)"
-          url="$(${yq} r ${versions_file} assets.hypervisor.qemu.url)"
-          commit=""
-          patches_dir="${kata_dir}/tools/packaging/qemu/patches/$(echo ${branch} | sed -e 's/.[[:digit:]]*$//' -e 's/^v//').x"
-          patches_version_dir="${kata_dir}/tools/packaging/qemu/patches/tag_patches/${branch}"
-        ;;
-      esac
+      branch="$(${yq} r ${versions_file} assets.hypervisor.qemu.version)"
+      url="$(${yq} r ${versions_file} assets.hypervisor.qemu.url)"
+      commit=""
+      patches_dir="${kata_dir}/tools/packaging/qemu/patches/$(echo ${branch} | sed -e 's/.[[:digit:]]*$//' -e 's/^v//').x"
+      patches_version_dir="${kata_dir}/tools/packaging/qemu/patches/tag_patches/${branch}"
 
       # download source
       qemu_dir=${SNAPCRAFT_STAGE}/qemu
       rm -rf "${qemu_dir}"
-      git clone --branch ${branch} --single-branch ${url} "${qemu_dir}"
+      git clone --depth 1 --branch ${branch} --single-branch ${url} "${qemu_dir}"
       cd ${qemu_dir}
       [ -z "${commit}" ] || git checkout ${commit}
 
-      [ -n "$(ls -A ui/keycodemapdb)" ] || git clone https://github.com/qemu/keycodemapdb ui/keycodemapdb/
-      [ -n "$(ls -A capstone)" ] || git clone https://github.com/qemu/capstone capstone
+      [ -n "$(ls -A ui/keycodemapdb)" ] || git clone --depth 1 https://github.com/qemu/keycodemapdb ui/keycodemapdb/
+      [ -n "$(ls -A capstone)" ] || git clone --depth 1 https://github.com/qemu/capstone capstone
 
       # Apply branch patches
       [ -d "${patches_version_dir}" ] || mkdir "${patches_version_dir}"
@@ -340,17 +320,23 @@ parts:
     plugin: nil
     after: [godeps]
     override-build: |
-      export GOPATH=${SNAPCRAFT_STAGE}/gopath
-      yq=${SNAPCRAFT_STAGE}/yq
-      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
-      versions_file="${kata_dir}/versions.yaml"
-      version="$(${yq} r ${versions_file} assets.hypervisor.cloud_hypervisor.version)"
-      url="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/${version}"
-      curl -L ${url}/cloud-hypervisor-static -o cloud-hypervisor
-      curl -LO ${url}/clh-remote
+      arch=$(uname -m)
+      if [ "{$arch}" == "aarch64" ] || [ "${arch}" == "x64_64" ]; then 
+          sudo apt-get -y update
+          sudo apt-get -y install ca-certificates curl gnupg lsb-release
+          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get -y update
+          sudo apt-get -y install docker-ce docker-ce-cli containerd.io
+          sudo systemctl start docker.socket
 
-      install -D cloud-hypervisor ${SNAPCRAFT_PART_INSTALL}/usr/bin/cloud-hypervisor
-      install -D clh-remote ${SNAPCRAFT_PART_INSTALL}/usr/bin/clh-remote
+          export GOPATH=${SNAPCRAFT_STAGE}/gopath
+          kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
+          cd ${kata_dir}
+          sudo -E NO_TTY=true make cloud-hypervisor-tarball
+          tar xvJpf build/kata-static-cloud-hypervisor.tar.xz -C /tmp/
+          install -D /tmp/opt/kata/bin/cloud-hypervisor ${SNAPCRAFT_PART_INSTALL}/usr/bin/cloud-hypervisor
+      fi
 
 apps:
   runtime:

src/agent/Cargo.lock

@@ -214,6 +214,12 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
+
 [[package]]
 name = "crc32fast"
 version = "1.3.0"
@@ -233,6 +239,30 @@ dependencies = [
  "crossbeam-utils",
 ]
 
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
+dependencies = [
+ "cfg-if 1.0.0",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97242a70df9b89a65d0b6df3c4bf5b9ce03c5b7309019777fbde37e7537f8762"
+dependencies = [
+ "cfg-if 1.0.0",
+ "crossbeam-utils",
+ "lazy_static",
+ "memoffset",
+ "scopeguard",
+]
+
 [[package]]
 name = "crossbeam-utils"
 version = "0.8.5"
@@ -565,6 +595,7 @@ dependencies = [
  "slog",
  "slog-scope",
  "slog-stdlog",
+ "sysinfo",
  "tempfile",
  "thiserror",
  "tokio",
@@ -1238,6 +1269,31 @@ dependencies = [
  "rand_core",
 ]
 
+[[package]]
+name = "rayon"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90"
+dependencies = [
+ "autocfg",
+ "crossbeam-deque",
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e"
+dependencies = [
+ "crossbeam-channel",
+ "crossbeam-deque",
+ "crossbeam-utils",
+ "lazy_static",
+ "num_cpus",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.2.10"
@@ -1518,6 +1574,21 @@ dependencies = [
  "unicode-xid",
 ]
 
+[[package]]
+name = "sysinfo"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e757000a4bed2b1be9be65a3f418b9696adf30bb419214c73997422de73a591"
+dependencies = [
+ "cfg-if 1.0.0",
+ "core-foundation-sys",
+ "libc",
+ "ntapi",
+ "once_cell",
+ "rayon",
+ "winapi",
+]
+
 [[package]]
 name = "take_mut"
 version = "0.2.2"

src/agent/Cargo.toml

@@ -20,6 +20,7 @@ scopeguard = "1.0.0"
 thiserror = "1.0.26"
 regex = "1.5.4"
 serial_test = "0.5.1"
+sysinfo = "0.23.0"
 
 # Async helpers
 async-trait = "0.1.42"

src/agent/Makefile

@@ -98,6 +98,8 @@ define INSTALL_FILE
 	install -D -m 644 $1 $(DESTDIR)$2/$1 || exit 1;
 endef
 
+.DEFAULT_GOAL := default
+
 ##TARGET default: build code
 default: $(TARGET) show-header
 
@@ -116,17 +118,6 @@ $(GENERATED_FILES): %: %.in
 optimize: $(SOURCES) | show-summary show-header
 	@RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
 
-##TARGET clippy: run clippy linter
-clippy: $(GENERATED_CODE)
-	cargo clippy --all-targets --all-features --release \
-		-- \
-		-Aclippy::redundant_allocation \
-		-D warnings
-
-format:
-	cargo fmt -- --check
-
-
 ##TARGET install: install agent
 install: install-services
 	@install -D $(TARGET_PATH) $(DESTDIR)/$(BINDIR)/$(TARGET)
@@ -146,7 +137,7 @@ test:
 	@cargo test --all --target $(TRIPLE) $(EXTRA_RUSTFEATURES) -- --nocapture
 
 ##TARGET check: run test
-check: clippy format
+check: $(GENERATED_FILES) standard_rust_check
 
 ##TARGET run: build and run agent
 run:

src/agent/rustjail/src/cgroups/fs/mod.rs

@@ -391,7 +391,7 @@ fn set_memory_resources(cg: &cgroups::Cgroup, memory: &LinuxMemory, update: bool
 
     if let Some(swappiness) = memory.swappiness {
         if (0..=100).contains(&swappiness) {
-            mem_controller.set_swappiness(swappiness as u64)?;
+            mem_controller.set_swappiness(swappiness)?;
         } else {
             return Err(anyhow!(
                 "invalid value:{}. valid memory swappiness range is 0-100",
@@ -590,9 +590,9 @@ fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> SingularPtrField<CpuUsage> {
 
         let h = lines_to_map(&cpuacct.stat);
         let usage_in_usermode =
-            (((*h.get("user").unwrap() * NANO_PER_SECOND) as f64) / *CLOCK_TICKS) as u64;
+            (((*h.get("user").unwrap_or(&0) * NANO_PER_SECOND) as f64) / *CLOCK_TICKS) as u64;
         let usage_in_kernelmode =
-            (((*h.get("system").unwrap() * NANO_PER_SECOND) as f64) / *CLOCK_TICKS) as u64;
+            (((*h.get("system").unwrap_or(&0) * NANO_PER_SECOND) as f64) / *CLOCK_TICKS) as u64;
 
         let total_usage = cpuacct.usage;
 
@@ -623,9 +623,9 @@ fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> SingularPtrField<CpuUsage> {
     let cpu_controller: &CpuController = get_controller_or_return_singular_none!(cg);
     let stat = cpu_controller.cpu().stat;
     let h = lines_to_map(&stat);
-    let usage_in_usermode = *h.get("user_usec").unwrap();
-    let usage_in_kernelmode = *h.get("system_usec").unwrap();
-    let total_usage = *h.get("usage_usec").unwrap();
+    let usage_in_usermode = *h.get("user_usec").unwrap_or(&0);
+    let usage_in_kernelmode = *h.get("system_usec").unwrap_or(&0);
+    let total_usage = *h.get("usage_usec").unwrap_or(&0);
     let percpu_usage = vec![];
 
     SingularPtrField::some(CpuUsage {
@@ -905,13 +905,7 @@ pub fn get_paths() -> Result<HashMap<String, String>> {
 
         let keys: Vec<&str> = fl[1].split(',').collect();
         for key in &keys {
-            // this is a workaround, cgroup file are using `name=systemd`,
-            // but if file system the name is `systemd`
-            if *key == "name=systemd" {
-                m.insert("systemd".to_string(), fl[2].to_string());
-            } else {
-                m.insert(key.to_string(), fl[2].to_string());
-            }
+            m.insert(key.to_string(), fl[2].to_string());
         }
     }
     Ok(m)

src/agent/rustjail/src/container.rs

@@ -215,7 +215,6 @@ pub trait BaseContainer {
     async fn start(&mut self, p: Process) -> Result<()>;
     async fn run(&mut self, p: Process) -> Result<()>;
     async fn destroy(&mut self) -> Result<()>;
-    fn signal(&self, sig: Signal, all: bool) -> Result<()>;
     fn exec(&mut self) -> Result<()>;
 }
 
@@ -1033,7 +1032,19 @@ impl BaseContainer for LinuxContainer {
         let st = self.oci_state()?;
 
         for pid in self.processes.keys() {
-            signal::kill(Pid::from_raw(*pid), Some(Signal::SIGKILL))?;
+            match signal::kill(Pid::from_raw(*pid), Some(Signal::SIGKILL)) {
+                Err(Errno::ESRCH) => {
+                    info!(
+                        self.logger,
+                        "kill encounters ESRCH, pid: {}, container: {}",
+                        pid,
+                        self.id.clone()
+                    );
+                    continue;
+                }
+                Err(err) => return Err(anyhow!(err)),
+                Ok(_) => continue,
+            }
         }
 
         if spec.hooks.is_some() {
@@ -1057,18 +1068,6 @@ impl BaseContainer for LinuxContainer {
         Ok(())
     }
 
-    fn signal(&self, sig: Signal, all: bool) -> Result<()> {
-        if all {
-            for pid in self.processes.keys() {
-                signal::kill(Pid::from_raw(*pid), Some(sig))?;
-            }
-        }
-
-        signal::kill(Pid::from_raw(self.init_process_pid), Some(sig))?;
-
-        Ok(())
-    }
-
     fn exec(&mut self) -> Result<()> {
         let fifo = format!("{}/{}", &self.root, EXEC_FIFO_FILENAME);
         let fd = fcntl::open(fifo.as_str(), OFlag::O_WRONLY, Mode::from_bits_truncate(0))?;
@@ -1482,15 +1481,15 @@ async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
         return Err(anyhow!(nix::Error::EINVAL));
     }
 
-    let args = h.args.clone();
-    let env: HashMap<String, String> = h
-        .env
-        .iter()
-        .map(|e| {
-            let v: Vec<&str> = e.split('=').collect();
-            (v[0].to_string(), v[1].to_string())
-        })
-        .collect();
+    let mut args = h.args.clone();
+    // the hook.args[0] is the hook binary name which shouldn't be included
+    // in the Command.args
+    if args.len() > 1 {
+        args.remove(0);
+    }
+
+    // all invalid envs will be omitted, only valid envs will be passed to hook.
+    let env: HashMap<&str, &str> = h.env.iter().filter_map(|e| valid_env(e)).collect();
 
     // Avoid the exit signal to be reaped by the global reaper.
     let _wait_locker = WAIT_PID_LOCKER.lock().await;
@@ -1501,8 +1500,7 @@ async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
         .stdin(Stdio::piped())
         .stdout(Stdio::piped())
         .stderr(Stdio::piped())
-        .spawn()
-        .unwrap();
+        .spawn()?;
 
     // default timeout 10s
     let mut timeout: u64 = 10;
@@ -1518,27 +1516,39 @@ async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
     let path = h.path.clone();
 
     let join_handle = tokio::spawn(async move {
-        child
-            .stdin
-            .as_mut()
-            .unwrap()
-            .write_all(state.as_bytes())
-            .await
-            .unwrap();
+        if let Some(mut stdin) = child.stdin.take() {
+            match stdin.write_all(state.as_bytes()).await {
+                Ok(_) => {}
+                Err(e) => {
+                    info!(logger, "write to child stdin failed: {:?}", e);
+                }
+            }
+        }
 
-        // Close stdin so that hook program could receive EOF
-        child.stdin.take();
+        // read something from stdout and stderr for debug
+        if let Some(stdout) = child.stdout.as_mut() {
+            let mut out = String::new();
+            match stdout.read_to_string(&mut out).await {
+                Ok(_) => {
+                    info!(logger, "child stdout: {}", out.as_str());
+                }
+                Err(e) => {
+                    info!(logger, "read from child stdout failed: {:?}", e);
+                }
+            }
+        }
 
-        // read something from stdout for debug
-        let mut out = String::new();
-        child
-            .stdout
-            .as_mut()
-            .unwrap()
-            .read_to_string(&mut out)
-            .await
-            .unwrap();
-        info!(logger, "child stdout: {}", out.as_str());
+        let mut err = String::new();
+        if let Some(stderr) = child.stderr.as_mut() {
+            match stderr.read_to_string(&mut err).await {
+                Ok(_) => {
+                    info!(logger, "child stderr: {}", err.as_str());
+                }
+                Err(e) => {
+                    info!(logger, "read from child stderr failed: {:?}", e);
+                }
+            }
+        }
 
         match child.wait().await {
             Ok(exit) => {
@@ -1547,7 +1557,10 @@ async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
                     .ok_or_else(|| anyhow!("hook exit status has no status code"))?;
 
                 if code != 0 {
-                    error!(logger, "hook {} exit status is {}", &path, code);
+                    error!(
+                        logger,
+                        "hook {} exit status is {}, error message is {}", &path, code, err
+                    );
                     return Err(anyhow!(nix::Error::UnknownErrno));
                 }
 
@@ -1624,13 +1637,47 @@ mod tests {
 
     #[tokio::test]
     async fn test_execute_hook() {
-        let xargs = which("xargs").await;
+        let temp_file = "/tmp/test_execute_hook";
+
+        let touch = which("touch").await;
+
+        defer!(fs::remove_file(temp_file).unwrap(););
+        let invalid_str = vec![97, b'\0', 98];
+        let invalid_string = std::str::from_utf8(&invalid_str).unwrap();
+        let invalid_env = format!("{}=value", invalid_string);
 
         execute_hook(
             &slog_scope::logger(),
             &Hook {
-                path: xargs,
-                args: vec![],
+                path: touch,
+                args: vec!["touch".to_string(), temp_file.to_string()],
+                env: vec![invalid_env],
+                timeout: Some(10),
+            },
+            &OCIState {
+                version: "1.2.3".to_string(),
+                id: "321".to_string(),
+                status: ContainerState::Running,
+                pid: 2,
+                bundle: "".to_string(),
+                annotations: Default::default(),
+            },
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(Path::new(&temp_file).exists(), true);
+    }
+
+    #[tokio::test]
+    async fn test_execute_hook_with_error() {
+        let ls = which("ls").await;
+
+        let res = execute_hook(
+            &slog_scope::logger(),
+            &Hook {
+                path: ls,
+                args: vec!["ls".to_string(), "/tmp/not-exist".to_string()],
                 env: vec![],
                 timeout: None,
             },
@@ -1643,8 +1690,13 @@ mod tests {
                 annotations: Default::default(),
             },
         )
-        .await
-        .unwrap()
+        .await;
+
+        let expected_err = nix::Error::UnknownErrno;
+        assert_eq!(
+            res.unwrap_err().downcast::<nix::Error>().unwrap(),
+            expected_err
+        );
     }
 
     #[tokio::test]
@@ -1655,7 +1707,7 @@ mod tests {
             &slog_scope::logger(),
             &Hook {
                 path: sleep,
-                args: vec!["2".to_string()],
+                args: vec!["sleep".to_string(), "2".to_string()],
                 env: vec![],
                 timeout: Some(1),
             },
@@ -1996,14 +2048,6 @@ mod tests {
         assert!(ret.is_ok(), "Expecting Ok, Got {:?}", ret);
     }
 
-    #[test]
-    fn test_linuxcontainer_signal() {
-        let ret = new_linux_container_and_then(|c: LinuxContainer| {
-            c.signal(nix::sys::signal::SIGCONT, true)
-        });
-        assert!(ret.is_ok(), "Expecting Ok, Got {:?}", ret);
-    }
-
     #[test]
     fn test_linuxcontainer_exec() {
         let ret = new_linux_container_and_then(|mut c: LinuxContainer| c.exec());

src/agent/rustjail/src/lib.rs

@@ -265,7 +265,7 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
             swap: Some(mem.Swap),
             kernel: Some(mem.Kernel),
             kernel_tcp: Some(mem.KernelTCP),
-            swappiness: Some(mem.Swappiness as i64),
+            swappiness: Some(mem.Swappiness),
             disable_oom_killer: Some(mem.DisableOOMKiller),
         })
     } else {

src/agent/rustjail/src/process.rs

@@ -8,8 +8,8 @@ use std::fs::File;
 use std::os::unix::io::RawFd;
 use tokio::sync::mpsc::Sender;
 
+use nix::errno::Errno;
 use nix::fcntl::{fcntl, FcntlArg, OFlag};
-use nix::sys::signal::{self, Signal};
 use nix::sys::wait::{self, WaitStatus};
 use nix::unistd::{self, Pid};
 use nix::Result;
@@ -80,7 +80,7 @@ pub struct Process {
 pub trait ProcessOperations {
     fn pid(&self) -> Pid;
     fn wait(&self) -> Result<WaitStatus>;
-    fn signal(&self, sig: Signal) -> Result<()>;
+    fn signal(&self, sig: libc::c_int) -> Result<()>;
 }
 
 impl ProcessOperations for Process {
@@ -92,8 +92,10 @@ impl ProcessOperations for Process {
         wait::waitpid(Some(self.pid()), None)
     }
 
-    fn signal(&self, sig: Signal) -> Result<()> {
-        signal::kill(self.pid(), Some(sig))
+    fn signal(&self, sig: libc::c_int) -> Result<()> {
+        let res = unsafe { libc::kill(self.pid().into(), sig) };
+
+        Errno::result(res).map(drop)
     }
 }
 
@@ -281,6 +283,6 @@ mod tests {
         // signal to every process in the process
         // group of the calling process.
         process.pid = 0;
-        assert!(process.signal(Signal::SIGCONT).is_ok());
+        assert!(process.signal(libc::SIGCONT).is_ok());
     }
 }

src/agent/samples/configuration-all-endpoints.toml

@@ -25,6 +25,7 @@ allowed = [
         "ReadStreamRequest",
         "RemoveContainerRequest",
         "ReseedRandomDevRequest",
+        "ResizeVolumeRequest",
         "ResumeContainerRequest",
         "SetGuestDateTimeRequest",
         "SignalProcessRequest",
@@ -34,6 +35,7 @@ allowed = [
         "UpdateContainerRequest",
         "UpdateInterfaceRequest",
         "UpdateRoutesRequest",
+        "VolumeStatsRequest",
         "WaitProcessRequest",
         "WriteStreamRequest"
 ]

src/agent/src/config.rs

@@ -432,6 +432,8 @@ fn get_container_pipe_size(param: &str) -> Result<i32> {
 
 #[cfg(test)]
 mod tests {
+    use crate::assert_result;
+
     use super::*;
     use anyhow::anyhow;
     use std::fs::File;
@@ -439,32 +441,6 @@ mod tests {
     use std::time;
     use tempfile::tempdir;
 
-    // Parameters:
-    //
-    // 1: expected Result
-    // 2: actual Result
-    // 3: string used to identify the test on error
-    macro_rules! assert_result {
-        ($expected_result:expr, $actual_result:expr, $msg:expr) => {
-            if $expected_result.is_ok() {
-                let expected_level = $expected_result.as_ref().unwrap();
-                let actual_level = $actual_result.unwrap();
-                assert!(*expected_level == actual_level, "{}", $msg);
-            } else {
-                let expected_error = $expected_result.as_ref().unwrap_err();
-                let expected_error_msg = format!("{:?}", expected_error);
-
-                if let Err(actual_error) = $actual_result {
-                    let actual_error_msg = format!("{:?}", actual_error);
-
-                    assert!(expected_error_msg == actual_error_msg, "{}", $msg);
-                } else {
-                    assert!(expected_error_msg == "expected error, got OK", "{}", $msg);
-                }
-            }
-        };
-    }
-
     #[test]
     fn test_new() {
         let config: AgentConfig = Default::default();

src/agent/src/device.rs

@@ -51,6 +51,8 @@ pub const DRIVER_VFIO_GK_TYPE: &str = "vfio-gk";
 // VFIO device to be bound to vfio-pci and made available inside the
 // container as a VFIO device node
 pub const DRIVER_VFIO_TYPE: &str = "vfio";
+pub const DRIVER_OVERLAYFS_TYPE: &str = "overlayfs";
+pub const FS_TYPE_HUGETLB: &str = "hugetlbfs";
 
 #[instrument]
 pub fn online_device(path: &str) -> Result<()> {
@@ -592,38 +594,38 @@ fn update_spec_devices(spec: &mut Spec, mut updates: HashMap<&str, DevUpdate>) -
     Ok(())
 }
 
-// update_spec_pci PCI addresses in the OCI spec to be guest addresses
-// instead of host addresses.  It is given a map of (host address =>
-// guest address)
+// update_env_pci alters PCI addresses in a set of environment
+// variables to be correct for the VM instead of the host.  It is
+// given a map of (host address => guest address)
 #[instrument]
-fn update_spec_pci(spec: &mut Spec, updates: HashMap<pci::Address, pci::Address>) -> Result<()> {
-    // Correct PCI addresses in the environment
-    if let Some(process) = spec.process.as_mut() {
-        for envvar in process.env.iter_mut() {
-            let eqpos = envvar
-                .find('=')
-                .ok_or_else(|| anyhow!("Malformed OCI env entry {:?}", envvar))?;
+pub fn update_env_pci(
+    env: &mut [String],
+    pcimap: &HashMap<pci::Address, pci::Address>,
+) -> Result<()> {
+    for envvar in env {
+        let eqpos = envvar
+            .find('=')
+            .ok_or_else(|| anyhow!("Malformed OCI env entry {:?}", envvar))?;
 
-            let (name, eqval) = envvar.split_at(eqpos);
-            let val = &eqval[1..];
+        let (name, eqval) = envvar.split_at(eqpos);
+        let val = &eqval[1..];
 
-            if !name.starts_with("PCIDEVICE_") {
-                continue;
-            }
-
-            let mut guest_addrs = Vec::<String>::new();
-
-            for host_addr in val.split(',') {
-                let host_addr = pci::Address::from_str(host_addr)
-                    .with_context(|| format!("Can't parse {} environment variable", name))?;
-                let guest_addr = updates
-                    .get(&host_addr)
-                    .ok_or_else(|| anyhow!("Unable to translate host PCI address {}", host_addr))?;
-                guest_addrs.push(format!("{}", guest_addr));
-            }
-
-            envvar.replace_range(eqpos + 1.., guest_addrs.join(",").as_str());
+        if !name.starts_with("PCIDEVICE_") {
+            continue;
         }
+
+        let mut guest_addrs = Vec::<String>::new();
+
+        for host_addr in val.split(',') {
+            let host_addr = pci::Address::from_str(host_addr)
+                .with_context(|| format!("Can't parse {} environment variable", name))?;
+            let guest_addr = pcimap
+                .get(&host_addr)
+                .ok_or_else(|| anyhow!("Unable to translate host PCI address {}", host_addr))?;
+            guest_addrs.push(format!("{}", guest_addr));
+        }
+
+        envvar.replace_range(eqpos + 1.., guest_addrs.join(",").as_str());
     }
 
     Ok(())
@@ -768,7 +770,6 @@ pub async fn add_devices(
     sandbox: &Arc<Mutex<Sandbox>>,
 ) -> Result<()> {
     let mut dev_updates = HashMap::<&str, DevUpdate>::with_capacity(devices.len());
-    let mut pci_updates = HashMap::<pci::Address, pci::Address>::new();
 
     for device in devices.iter() {
         let update = add_device(device, sandbox).await?;
@@ -783,8 +784,9 @@ pub async fn add_devices(
                 ));
             }
 
+            let mut sb = sandbox.lock().await;
             for (host, guest) in update.pci {
-                if let Some(other_guest) = pci_updates.insert(host, guest) {
+                if let Some(other_guest) = sb.pcimap.insert(host, guest) {
                     return Err(anyhow!(
                         "Conflicting guest address for host device {} ({} versus {})",
                         host,
@@ -796,6 +798,9 @@ pub async fn add_devices(
         }
     }
 
+    if let Some(process) = spec.process.as_mut() {
+        update_env_pci(&mut process.env, &sandbox.lock().await.pcimap)?
+    }
     update_spec_devices(spec, dev_updates)
 }
 
@@ -860,7 +865,7 @@ pub fn update_device_cgroup(spec: &mut Spec) -> Result<()> {
 mod tests {
     use super::*;
     use crate::uevent::spawn_test_watcher;
-    use oci::{Linux, Process};
+    use oci::Linux;
     use std::iter::FromIterator;
     use tempfile::tempdir;
 
@@ -1199,7 +1204,7 @@ mod tests {
     }
 
     #[test]
-    fn test_update_spec_pci() {
+    fn test_update_env_pci() {
         let example_map = [
             // Each is a host,guest pair of pci addresses
             ("0000:1a:01.0", "0000:01:01.0"),
@@ -1209,17 +1214,11 @@ mod tests {
             ("0000:01:01.0", "ffff:02:1f.7"),
         ];
 
-        let mut spec = Spec {
-            process: Some(Process {
-                env: vec![
-                    "PCIDEVICE_x=0000:1a:01.0,0000:1b:02.0".to_string(),
-                    "PCIDEVICE_y=0000:01:01.0".to_string(),
-                    "NOTAPCIDEVICE_blah=abcd:ef:01.0".to_string(),
-                ],
-                ..Process::default()
-            }),
-            ..Spec::default()
-        };
+        let mut env = vec![
+            "PCIDEVICE_x=0000:1a:01.0,0000:1b:02.0".to_string(),
+            "PCIDEVICE_y=0000:01:01.0".to_string(),
+            "NOTAPCIDEVICE_blah=abcd:ef:01.0".to_string(),
+        ];
 
         let pci_fixups = example_map
             .iter()
@@ -1231,10 +1230,9 @@ mod tests {
             })
             .collect();
 
-        let res = update_spec_pci(&mut spec, pci_fixups);
+        let res = update_env_pci(&mut env, &pci_fixups);
         assert!(res.is_ok());
 
-        let env = &spec.process.as_ref().unwrap().env;
         assert_eq!(env[0], "PCIDEVICE_x=0000:01:01.0,0000:01:02.0");
         assert_eq!(env[1], "PCIDEVICE_y=ffff:02:1f.7");
         assert_eq!(env[2], "NOTAPCIDEVICE_blah=abcd:ef:01.0");

src/agent/src/mount.rs

@@ -5,8 +5,8 @@
 
 use std::collections::HashMap;
 use std::fs;
-use std::fs::File;
-use std::io::{BufRead, BufReader};
+use std::fs::{File, OpenOptions};
+use std::io::{BufRead, BufReader, Write};
 use std::iter;
 use std::os::unix::fs::{MetadataExt, PermissionsExt};
 use std::path::Path;
@@ -16,19 +16,20 @@ use std::sync::Arc;
 use tokio::sync::Mutex;
 
 use nix::mount::MsFlags;
-use nix::unistd::Gid;
+use nix::unistd::{Gid, Uid};
 
 use regex::Regex;
 
 use crate::device::{
     get_scsi_device_name, get_virtio_blk_pci_device_name, online_device, wait_for_pmem_device,
     DRIVER_9P_TYPE, DRIVER_BLK_CCW_TYPE, DRIVER_BLK_TYPE, DRIVER_EPHEMERAL_TYPE, DRIVER_LOCAL_TYPE,
-    DRIVER_MMIO_BLK_TYPE, DRIVER_NVDIMM_TYPE, DRIVER_SCSI_TYPE, DRIVER_VIRTIOFS_TYPE,
-    DRIVER_WATCHABLE_BIND_TYPE,
+    DRIVER_MMIO_BLK_TYPE, DRIVER_NVDIMM_TYPE, DRIVER_OVERLAYFS_TYPE, DRIVER_SCSI_TYPE,
+    DRIVER_VIRTIOFS_TYPE, DRIVER_WATCHABLE_BIND_TYPE, FS_TYPE_HUGETLB,
 };
 use crate::linux_abi::*;
 use crate::pci;
 use crate::protocols::agent::Storage;
+use crate::protocols::types::FSGroupChangePolicy;
 use crate::Sandbox;
 #[cfg(target_arch = "s390x")]
 use crate::{ccw, device::get_virtio_blk_ccw_device_name};
@@ -37,12 +38,17 @@ use slog::Logger;
 use tracing::instrument;
 
 pub const TYPE_ROOTFS: &str = "rootfs";
-
+const SYS_FS_HUGEPAGES_PREFIX: &str = "/sys/kernel/mm/hugepages";
 pub const MOUNT_GUEST_TAG: &str = "kataShared";
 
 // Allocating an FSGroup that owns the pod's volumes
 const FS_GID: &str = "fsgid";
 
+const RW_MASK: u32 = 0o660;
+const RO_MASK: u32 = 0o440;
+const EXEC_MASK: u32 = 0o110;
+const MODE_SETGID: u32 = 0o2000;
+
 #[rustfmt::skip]
 lazy_static! {
     pub static ref FLAGS: HashMap<&'static str, (bool, MsFlags)> = {
@@ -130,6 +136,7 @@ pub const STORAGE_HANDLER_LIST: &[&str] = &[
     DRIVER_9P_TYPE,
     DRIVER_VIRTIOFS_TYPE,
     DRIVER_EPHEMERAL_TYPE,
+    DRIVER_OVERLAYFS_TYPE,
     DRIVER_MMIO_BLK_TYPE,
     DRIVER_LOCAL_TYPE,
     DRIVER_SCSI_TYPE,
@@ -192,13 +199,12 @@ async fn ephemeral_storage_handler(
     storage: &Storage,
     sandbox: Arc<Mutex<Sandbox>>,
 ) -> Result<String> {
-    let mut sb = sandbox.lock().await;
-    let new_storage = sb.set_sandbox_storage(&storage.mount_point);
-
-    if !new_storage {
-        return Ok("".to_string());
+    // hugetlbfs
+    if storage.fstype == FS_TYPE_HUGETLB {
+        return handle_hugetlbfs_storage(logger, storage).await;
     }
 
+    // normal ephemeral storage
     fs::create_dir_all(Path::new(&storage.mount_point))?;
 
     // By now we only support one option field: "fsGroup" which
@@ -222,7 +228,7 @@ async fn ephemeral_storage_handler(
             let meta = fs::metadata(&storage.mount_point)?;
             let mut permission = meta.permissions();
 
-            let o_mode = meta.mode() | 0o2000;
+            let o_mode = meta.mode() | MODE_SETGID;
             permission.set_mode(o_mode);
             fs::set_permissions(&storage.mount_point, permission)?;
         }
@@ -233,19 +239,21 @@ async fn ephemeral_storage_handler(
     Ok("".to_string())
 }
 
+#[instrument]
+async fn overlayfs_storage_handler(
+    logger: &Logger,
+    storage: &Storage,
+    _sandbox: Arc<Mutex<Sandbox>>,
+) -> Result<String> {
+    common_storage_handler(logger, storage)
+}
+
 #[instrument]
 async fn local_storage_handler(
     _logger: &Logger,
     storage: &Storage,
     sandbox: Arc<Mutex<Sandbox>>,
 ) -> Result<String> {
-    let mut sb = sandbox.lock().await;
-    let new_storage = sb.set_sandbox_storage(&storage.mount_point);
-
-    if !new_storage {
-        return Ok("".to_string());
-    }
-
     fs::create_dir_all(&storage.mount_point).context(format!(
         "failed to create dir all {:?}",
         &storage.mount_point
@@ -270,7 +278,7 @@ async fn local_storage_handler(
 
         if need_set_fsgid {
             // set SetGid mode mask.
-            o_mode |= 0o2000;
+            o_mode |= MODE_SETGID;
         }
         permission.set_mode(o_mode);
 
@@ -289,12 +297,116 @@ async fn virtio9p_storage_handler(
     common_storage_handler(logger, storage)
 }
 
+#[instrument]
+async fn handle_hugetlbfs_storage(logger: &Logger, storage: &Storage) -> Result<String> {
+    info!(logger, "handle hugetlbfs storage");
+    // Allocate hugepages before mount
+    // /sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages
+    // /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
+    // options eg "pagesize=2097152,size=524288000"(2M, 500M)
+    allocate_hugepages(logger, &storage.options.to_vec()).context("allocate hugepages")?;
+
+    common_storage_handler(logger, storage)?;
+
+    // hugetlbfs return empty string as ephemeral_storage_handler do.
+    // this is a sandbox level storage, but not a container-level mount.
+    Ok("".to_string())
+}
+
+// Allocate hugepages by writing to sysfs
+fn allocate_hugepages(logger: &Logger, options: &[String]) -> Result<()> {
+    info!(logger, "mounting hugePages storage options: {:?}", options);
+
+    let (pagesize, size) = get_pagesize_and_size_from_option(options)
+        .context(format!("parse mount options: {:?}", &options))?;
+
+    info!(
+        logger,
+        "allocate hugepages. pageSize: {}, size: {}", pagesize, size
+    );
+
+    // sysfs entry is always of the form hugepages-${pagesize}kB
+    // Ref: https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt
+    let path = Path::new(SYS_FS_HUGEPAGES_PREFIX)
+        .join(format!("hugepages-{}kB", pagesize / 1024))
+        .join("nr_hugepages");
+
+    // write numpages to nr_hugepages file.
+    let numpages = format!("{}", size / pagesize);
+    info!(logger, "write {} pages to {:?}", &numpages, &path);
+
+    let mut file = OpenOptions::new()
+        .write(true)
+        .open(&path)
+        .context(format!("open nr_hugepages directory {:?}", &path))?;
+
+    file.write_all(numpages.as_bytes())
+        .context(format!("write nr_hugepages failed: {:?}", &path))?;
+
+    // Even if the write succeeds, the kernel isn't guaranteed to be
+    // able to allocate all the pages we requested.  Verify that it
+    // did.
+    let verify = fs::read_to_string(&path).context(format!("reading {:?}", &path))?;
+    let allocated = verify
+        .trim_end()
+        .parse::<u64>()
+        .map_err(|_| anyhow!("Unexpected text {:?} in {:?}", &verify, &path))?;
+    if allocated != size / pagesize {
+        return Err(anyhow!(
+            "Only allocated {} of {} hugepages of size {}",
+            allocated,
+            numpages,
+            pagesize
+        ));
+    }
+
+    Ok(())
+}
+
+// Parse filesystem options string to retrieve hugepage details
+// options eg "pagesize=2048,size=107374182"
+fn get_pagesize_and_size_from_option(options: &[String]) -> Result<(u64, u64)> {
+    let mut pagesize_str: Option<&str> = None;
+    let mut size_str: Option<&str> = None;
+
+    for option in options {
+        let vars: Vec<&str> = option.trim().split(',').collect();
+
+        for var in vars {
+            if let Some(stripped) = var.strip_prefix("pagesize=") {
+                pagesize_str = Some(stripped);
+            } else if let Some(stripped) = var.strip_prefix("size=") {
+                size_str = Some(stripped);
+            }
+
+            if pagesize_str.is_some() && size_str.is_some() {
+                break;
+            }
+        }
+    }
+
+    if pagesize_str.is_none() || size_str.is_none() {
+        return Err(anyhow!("no pagesize/size options found"));
+    }
+
+    let pagesize = pagesize_str
+        .unwrap()
+        .parse::<u64>()
+        .context(format!("parse pagesize: {:?}", &pagesize_str))?;
+    let size = size_str
+        .unwrap()
+        .parse::<u64>()
+        .context(format!("parse size: {:?}", &pagesize_str))?;
+
+    Ok((pagesize, size))
+}
+
 // virtiommio_blk_storage_handler handles the storage for mmio blk driver.
 #[instrument]
 async fn virtiommio_blk_storage_handler(
     logger: &Logger,
     storage: &Storage,
-    _sandbox: Arc<Mutex<Sandbox>>,
+    sandbox: Arc<Mutex<Sandbox>>,
 ) -> Result<String> {
     //The source path is VmPath
     common_storage_handler(logger, storage)
@@ -383,7 +495,9 @@ fn common_storage_handler(logger: &Logger, storage: &Storage) -> Result<String>
     // Mount the storage device.
     let mount_point = storage.mount_point.to_string();
 
-    mount_storage(logger, storage).and(Ok(mount_point))
+    mount_storage(logger, storage)?;
+    set_ownership(logger, storage)?;
+    Ok(mount_point)
 }
 
 // nvdimm_storage_handler handles the storage for NVDIMM driver.
@@ -467,6 +581,91 @@ fn mount_storage(logger: &Logger, storage: &Storage) -> Result<()> {
     )
 }
 
+#[instrument]
+pub fn set_ownership(logger: &Logger, storage: &Storage) -> Result<()> {
+    let logger = logger.new(o!("subsystem" => "mount", "fn" => "set_ownership"));
+
+    // If fsGroup is not set, skip performing ownership change
+    if storage.fs_group.is_none() {
+        return Ok(());
+    }
+    let fs_group = storage.get_fs_group();
+
+    let mut read_only = false;
+    let opts_vec: Vec<String> = storage.options.to_vec();
+    if opts_vec.contains(&String::from("ro")) {
+        read_only = true;
+    }
+
+    let mount_path = Path::new(&storage.mount_point);
+    let metadata = mount_path.metadata().map_err(|err| {
+        error!(logger, "failed to obtain metadata for mount path";
+            "mount-path" => mount_path.to_str(),
+            "error" => err.to_string(),
+        );
+        err
+    })?;
+
+    if fs_group.group_change_policy == FSGroupChangePolicy::OnRootMismatch
+        && metadata.gid() == fs_group.group_id
+    {
+        let mut mask = if read_only { RO_MASK } else { RW_MASK };
+        mask |= EXEC_MASK;
+
+        // With fsGroup change policy to OnRootMismatch, if the current
+        // gid of the mount path root directory matches the desired gid
+        // and the current permission of mount path root directory is correct,
+        // then ownership change will be skipped.
+        let current_mode = metadata.permissions().mode();
+        if (mask & current_mode == mask) && (current_mode & MODE_SETGID != 0) {
+            info!(logger, "skipping ownership change for volume";
+                "mount-path" => mount_path.to_str(),
+                "fs-group" => fs_group.group_id.to_string(),
+            );
+            return Ok(());
+        }
+    }
+
+    info!(logger, "performing recursive ownership change";
+        "mount-path" => mount_path.to_str(),
+        "fs-group" => fs_group.group_id.to_string(),
+    );
+    recursive_ownership_change(
+        mount_path,
+        None,
+        Some(Gid::from_raw(fs_group.group_id)),
+        read_only,
+    )
+}
+
+#[instrument]
+pub fn recursive_ownership_change(
+    path: &Path,
+    uid: Option<Uid>,
+    gid: Option<Gid>,
+    read_only: bool,
+) -> Result<()> {
+    let mut mask = if read_only { RO_MASK } else { RW_MASK };
+    if path.is_dir() {
+        for entry in fs::read_dir(&path)? {
+            recursive_ownership_change(entry?.path().as_path(), uid, gid, read_only)?;
+        }
+        mask |= EXEC_MASK;
+        mask |= MODE_SETGID;
+    }
+    nix::unistd::chown(path, uid, gid)?;
+
+    if gid.is_some() {
+        let metadata = path.metadata()?;
+        let mut permission = metadata.permissions();
+        let target_mode = metadata.mode() | mask;
+        permission.set_mode(target_mode);
+        fs::set_permissions(path, permission)?;
+    }
+
+    Ok(())
+}
+
 /// Looks for `mount_point` entry in the /proc/mounts.
 #[instrument]
 pub fn is_mounted(mount_point: &str) -> Result<bool> {
@@ -534,6 +733,14 @@ pub async fn add_storages(
             "subsystem" => "storage",
             "storage-type" => handler_name.to_owned()));
 
+        {
+            let mut sb = sandbox.lock().await;
+            let new_storage = sb.set_sandbox_storage(&storage.mount_point);
+            if !new_storage {
+                continue;
+            }
+        }
+
         let res = match handler_name.as_str() {
             DRIVER_BLK_TYPE => virtio_blk_storage_handler(&logger, &storage, sandbox.clone()).await,
             DRIVER_BLK_CCW_TYPE => {
@@ -546,6 +753,9 @@ pub async fn add_storages(
             DRIVER_EPHEMERAL_TYPE => {
                 ephemeral_storage_handler(&logger, &storage, sandbox.clone()).await
             }
+            DRIVER_OVERLAYFS_TYPE => {
+                overlayfs_storage_handler(&logger, &storage, sandbox.clone()).await
+            }
             DRIVER_MMIO_BLK_TYPE => {
                 virtiommio_blk_storage_handler(&logger, &storage, sandbox.clone()).await
             }
@@ -808,6 +1018,8 @@ fn parse_options(option_list: Vec<String>) -> HashMap<String, String> {
 mod tests {
     use super::*;
     use crate::{skip_if_not_root, skip_loop_if_not_root, skip_loop_if_root};
+    use protobuf::RepeatedField;
+    use protocols::agent::FSGroup;
     use std::fs::File;
     use std::fs::OpenOptions;
     use std::io::Write;
@@ -1379,4 +1591,268 @@ mod tests {
 
         assert!(testfile.is_file());
     }
+
+    #[test]
+    fn test_get_pagesize_and_size_from_option() {
+        let expected_pagesize = 2048;
+        let expected_size = 107374182;
+        let expected = (expected_pagesize, expected_size);
+
+        let data = vec![
+            // (input, expected, is_ok)
+            ("size-1=107374182,pagesize-1=2048", expected, false),
+            ("size-1=107374182,pagesize=2048", expected, false),
+            ("size=107374182,pagesize-1=2048", expected, false),
+            ("size=107374182,pagesize=abc", expected, false),
+            ("size=abc,pagesize=2048", expected, false),
+            ("size=,pagesize=2048", expected, false),
+            ("size=107374182,pagesize=", expected, false),
+            ("size=107374182,pagesize=2048", expected, true),
+            ("pagesize=2048,size=107374182", expected, true),
+            ("foo=bar,pagesize=2048,size=107374182", expected, true),
+            (
+                "foo=bar,pagesize=2048,foo1=bar1,size=107374182",
+                expected,
+                true,
+            ),
+            (
+                "pagesize=2048,foo1=bar1,foo=bar,size=107374182",
+                expected,
+                true,
+            ),
+            (
+                "foo=bar,pagesize=2048,foo1=bar1,size=107374182,foo2=bar2",
+                expected,
+                true,
+            ),
+            (
+                "foo=bar,size=107374182,foo1=bar1,pagesize=2048",
+                expected,
+                true,
+            ),
+        ];
+
+        for case in data {
+            let input = case.0;
+            let r = get_pagesize_and_size_from_option(&[input.to_string()]);
+
+            let is_ok = case.2;
+            if is_ok {
+                let expected = case.1;
+                let (pagesize, size) = r.unwrap();
+                assert_eq!(expected.0, pagesize);
+                assert_eq!(expected.1, size);
+            } else {
+                assert!(r.is_err());
+            }
+        }
+    }
+
+    #[test]
+    fn test_set_ownership() {
+        skip_if_not_root!();
+
+        let logger = slog::Logger::root(slog::Discard, o!());
+
+        #[derive(Debug)]
+        struct TestData<'a> {
+            mount_path: &'a str,
+            fs_group: Option<FSGroup>,
+            read_only: bool,
+            expected_group_id: u32,
+            expected_permission: u32,
+        }
+
+        let tests = &[
+            TestData {
+                mount_path: "foo",
+                fs_group: None,
+                read_only: false,
+                expected_group_id: 0,
+                expected_permission: 0,
+            },
+            TestData {
+                mount_path: "rw_mount",
+                fs_group: Some(FSGroup {
+                    group_id: 3000,
+                    group_change_policy: FSGroupChangePolicy::Always,
+                    unknown_fields: Default::default(),
+                    cached_size: Default::default(),
+                }),
+                read_only: false,
+                expected_group_id: 3000,
+                expected_permission: RW_MASK | EXEC_MASK | MODE_SETGID,
+            },
+            TestData {
+                mount_path: "ro_mount",
+                fs_group: Some(FSGroup {
+                    group_id: 3000,
+                    group_change_policy: FSGroupChangePolicy::OnRootMismatch,
+                    unknown_fields: Default::default(),
+                    cached_size: Default::default(),
+                }),
+                read_only: true,
+                expected_group_id: 3000,
+                expected_permission: RO_MASK | EXEC_MASK | MODE_SETGID,
+            },
+        ];
+
+        let tempdir = tempdir().expect("failed to create tmpdir");
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let mount_dir = tempdir.path().join(d.mount_path);
+            fs::create_dir(&mount_dir)
+                .unwrap_or_else(|_| panic!("{}: failed to create root directory", msg));
+
+            let directory_mode = mount_dir.as_path().metadata().unwrap().permissions().mode();
+            let mut storage_data = Storage::new();
+            if d.read_only {
+                storage_data.set_options(RepeatedField::from_slice(&[
+                    "foo".to_string(),
+                    "ro".to_string(),
+                ]));
+            }
+            if let Some(fs_group) = d.fs_group.clone() {
+                storage_data.set_fs_group(fs_group);
+            }
+            storage_data.mount_point = mount_dir.clone().into_os_string().into_string().unwrap();
+
+            let result = set_ownership(&logger, &storage_data);
+            assert!(result.is_ok());
+
+            assert_eq!(
+                mount_dir.as_path().metadata().unwrap().gid(),
+                d.expected_group_id
+            );
+            assert_eq!(
+                mount_dir.as_path().metadata().unwrap().permissions().mode(),
+                (directory_mode | d.expected_permission)
+            );
+        }
+    }
+
+    #[test]
+    fn test_recursive_ownership_change() {
+        skip_if_not_root!();
+
+        const COUNT: usize = 5;
+
+        #[derive(Debug)]
+        struct TestData<'a> {
+            // Directory where the recursive ownership change should be performed on
+            path: &'a str,
+
+            // User ID for ownership change
+            uid: u32,
+
+            // Group ID for ownership change
+            gid: u32,
+
+            // Set when the permission should be read-only
+            read_only: bool,
+
+            // The expected permission of all directories after ownership change
+            expected_permission_directory: u32,
+
+            // The expected permission of all files after ownership change
+            expected_permission_file: u32,
+        }
+
+        let tests = &[
+            TestData {
+                path: "no_gid_change",
+                uid: 0,
+                gid: 0,
+                read_only: false,
+                expected_permission_directory: 0,
+                expected_permission_file: 0,
+            },
+            TestData {
+                path: "rw_gid_change",
+                uid: 0,
+                gid: 3000,
+                read_only: false,
+                expected_permission_directory: RW_MASK | EXEC_MASK | MODE_SETGID,
+                expected_permission_file: RW_MASK,
+            },
+            TestData {
+                path: "ro_gid_change",
+                uid: 0,
+                gid: 3000,
+                read_only: true,
+                expected_permission_directory: RO_MASK | EXEC_MASK | MODE_SETGID,
+                expected_permission_file: RO_MASK,
+            },
+        ];
+
+        let tempdir = tempdir().expect("failed to create tmpdir");
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let mount_dir = tempdir.path().join(d.path);
+            fs::create_dir(&mount_dir)
+                .unwrap_or_else(|_| panic!("{}: failed to create root directory", msg));
+
+            let directory_mode = mount_dir.as_path().metadata().unwrap().permissions().mode();
+            let mut file_mode: u32 = 0;
+
+            // create testing directories and files
+            for n in 1..COUNT {
+                let nest_dir = mount_dir.join(format!("nested{}", n));
+                fs::create_dir(&nest_dir)
+                    .unwrap_or_else(|_| panic!("{}: failed to create nest directory", msg));
+
+                for f in 1..COUNT {
+                    let filename = nest_dir.join(format!("file{}", f));
+                    File::create(&filename)
+                        .unwrap_or_else(|_| panic!("{}: failed to create file", msg));
+                    file_mode = filename.as_path().metadata().unwrap().permissions().mode();
+                }
+            }
+
+            let uid = if d.uid > 0 {
+                Some(Uid::from_raw(d.uid))
+            } else {
+                None
+            };
+            let gid = if d.gid > 0 {
+                Some(Gid::from_raw(d.gid))
+            } else {
+                None
+            };
+            let result = recursive_ownership_change(&mount_dir, uid, gid, d.read_only);
+
+            assert!(result.is_ok());
+
+            assert_eq!(mount_dir.as_path().metadata().unwrap().gid(), d.gid);
+            assert_eq!(
+                mount_dir.as_path().metadata().unwrap().permissions().mode(),
+                (directory_mode | d.expected_permission_directory)
+            );
+
+            for n in 1..COUNT {
+                let nest_dir = mount_dir.join(format!("nested{}", n));
+                for f in 1..COUNT {
+                    let filename = nest_dir.join(format!("file{}", f));
+                    let file = Path::new(&filename);
+
+                    assert_eq!(file.metadata().unwrap().gid(), d.gid);
+                    assert_eq!(
+                        file.metadata().unwrap().permissions().mode(),
+                        (file_mode | d.expected_permission_file)
+                    );
+                }
+
+                let dir = Path::new(&nest_dir);
+                assert_eq!(dir.metadata().unwrap().gid(), d.gid);
+                assert_eq!(
+                    dir.metadata().unwrap().permissions().mode(),
+                    (directory_mode | d.expected_permission_directory)
+                );
+            }
+        }
+    }
 }

src/agent/src/sandbox.rs

@@ -8,6 +8,7 @@ use crate::mount::{get_mount_fs_type, remove_mounts, TYPE_ROOTFS};
 use crate::namespace::Namespace;
 use crate::netlink::Handle;
 use crate::network::Network;
+use crate::pci;
 use crate::uevent::{Uevent, UeventMatcher};
 use crate::watcher::BindWatcher;
 use anyhow::{anyhow, Context, Result};
@@ -31,6 +32,8 @@ use tokio::sync::oneshot;
 use tokio::sync::Mutex;
 use tracing::instrument;
 
+pub const ERR_INVALID_CONTAINER_ID: &str = "Invalid container id";
+
 type UeventWatcher = (Box<dyn UeventMatcher>, oneshot::Sender<Uevent>);
 
 #[derive(Debug)]
@@ -56,6 +59,7 @@ pub struct Sandbox {
     pub event_rx: Arc<Mutex<Receiver<String>>>,
     pub event_tx: Option<Sender<String>>,
     pub bind_watcher: BindWatcher,
+    pub pcimap: HashMap<pci::Address, pci::Address>,
 }
 
 impl Sandbox {
@@ -88,6 +92,7 @@ impl Sandbox {
             event_rx,
             event_tx: Some(tx),
             bind_watcher: BindWatcher::new(),
+            pcimap: HashMap::new(),
         })
     }
 
@@ -146,7 +151,12 @@ impl Sandbox {
     pub fn remove_sandbox_storage(&self, path: &str) -> Result<()> {
         let mounts = vec![path.to_string()];
         remove_mounts(&mounts)?;
-        fs::remove_dir_all(path).context(format!("failed to remove dir {:?}", path))?;
+        // "remove_dir" will fail if the mount point is backed by a read-only filesystem.
+        // This is the case with the device mapper snapshotter, where we mount the block device directly
+        // at the underlying sandbox path which was provided from the base RO kataShared path from the host.
+        if let Err(err) = fs::remove_dir(path) {
+            warn!(self.logger, "failed to remove dir {}, {:?}", path, err);
+        }
         Ok(())
     }
 
@@ -229,7 +239,7 @@ impl Sandbox {
     pub fn find_container_process(&mut self, cid: &str, eid: &str) -> Result<&mut Process> {
         let ctr = self
             .get_container(cid)
-            .ok_or_else(|| anyhow!("Invalid container id"))?;
+            .ok_or_else(|| anyhow!(ERR_INVALID_CONTAINER_ID))?;
 
         if eid.is_empty() {
             return ctr
@@ -548,7 +558,7 @@ mod tests {
 
         assert!(
             s.remove_sandbox_storage(srcdir_path).is_err(),
-            "Expect Err as the directory i not a mountpoint"
+            "Expect Err as the directory is not a mountpoint"
         );
 
         assert!(s.remove_sandbox_storage("").is_err());
@@ -559,19 +569,8 @@ mod tests {
             .remove_sandbox_storage(invalid_dir.to_str().unwrap())
             .is_err());
 
-        // Now, create a double mount as this guarantees the directory cannot
-        // be deleted after the first umount.
-        for _i in 0..2 {
-            assert!(bind_mount(srcdir_path, destdir_path, &logger).is_ok());
-        }
+        assert!(bind_mount(srcdir_path, destdir_path, &logger).is_ok());
 
-        assert!(
-            s.remove_sandbox_storage(destdir_path).is_err(),
-            "Expect fail as deletion cannot happen due to the second mount."
-        );
-
-        // This time it should work as the previous two calls have undone the double
-        // mount.
         assert!(s.remove_sandbox_storage(destdir_path).is_ok());
     }
 
@@ -583,7 +582,6 @@ mod tests {
         let logger = slog::Logger::root(slog::Discard, o!());
         let mut s = Sandbox::new(&logger).unwrap();
 
-        // FIX: This test fails, not sure why yet.
         assert!(
             s.unset_and_remove_sandbox_storage("/tmp/testEphePath")
                 .is_err(),

src/agent/src/test_utils.rs

@@ -53,4 +53,29 @@ mod test_utils {
             }
         };
     }
+
+    // Parameters:
+    //
+    // 1: expected Result
+    // 2: actual Result
+    // 3: string used to identify the test on error
+    #[macro_export]
+    macro_rules! assert_result {
+        ($expected_result:expr, $actual_result:expr, $msg:expr) => {
+            if $expected_result.is_ok() {
+                let expected_value = $expected_result.as_ref().unwrap();
+                let actual_value = $actual_result.unwrap();
+                assert!(*expected_value == actual_value, "{}", $msg);
+            } else {
+                assert!($actual_result.is_err(), "{}", $msg);
+
+                let expected_error = $expected_result.as_ref().unwrap_err();
+                let expected_error_msg = format!("{:?}", expected_error);
+
+                let actual_error_msg = format!("{:?}", $actual_result.unwrap_err());
+
+                assert!(expected_error_msg == actual_error_msg, "{}", $msg);
+            }
+        };
+    }
 }

src/agent/src/watcher.rs

@@ -6,6 +6,7 @@
 #![allow(unknown_lints)]
 
 use std::collections::HashMap;
+use std::os::unix::fs::MetadataExt;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::SystemTime;
@@ -13,6 +14,7 @@ use std::time::SystemTime;
 use anyhow::{ensure, Context, Result};
 use async_recursion::async_recursion;
 use nix::mount::{umount, MsFlags};
+use nix::unistd::{Gid, Uid};
 use slog::{debug, error, info, warn, Logger};
 use thiserror::Error;
 use tokio::fs;
@@ -80,7 +82,8 @@ impl Drop for Storage {
 }
 
 async fn copy(from: impl AsRef<Path>, to: impl AsRef<Path>) -> Result<()> {
-    if fs::symlink_metadata(&from).await?.file_type().is_symlink() {
+    let metadata = fs::symlink_metadata(&from).await?;
+    if metadata.file_type().is_symlink() {
         // if source is a symlink, create new symlink with same link source. If
         // the symlink exists, remove and create new one:
         if fs::symlink_metadata(&to).await.is_ok() {
@@ -88,8 +91,15 @@ async fn copy(from: impl AsRef<Path>, to: impl AsRef<Path>) -> Result<()> {
         }
         fs::symlink(fs::read_link(&from).await?, &to).await?;
     } else {
-        fs::copy(from, to).await?;
+        fs::copy(&from, &to).await?;
     }
+    // preserve the source uid and gid to the destination.
+    nix::unistd::chown(
+        to.as_ref(),
+        Some(Uid::from_raw(metadata.uid())),
+        Some(Gid::from_raw(metadata.gid())),
+    )?;
+
     Ok(())
 }
 
@@ -106,14 +116,29 @@ impl Storage {
 
     async fn update_target(&self, logger: &Logger, source_path: impl AsRef<Path>) -> Result<()> {
         let source_file_path = source_path.as_ref();
+        let metadata = source_file_path.symlink_metadata()?;
 
         // if we are creating a directory: just create it, nothing more to do
-        if source_file_path.symlink_metadata()?.file_type().is_dir() {
+        if metadata.file_type().is_dir() {
             let dest_file_path = self.make_target_path(&source_file_path)?;
 
             fs::create_dir_all(&dest_file_path)
                 .await
                 .with_context(|| format!("Unable to mkdir all for {}", dest_file_path.display()))?;
+            // set the directory permissions to match the source directory permissions
+            fs::set_permissions(&dest_file_path, metadata.permissions())
+                .await
+                .with_context(|| {
+                    format!("Unable to set permissions for {}", dest_file_path.display())
+                })?;
+            // preserve the source directory uid and gid to the destination.
+            nix::unistd::chown(
+                &dest_file_path,
+                Some(Uid::from_raw(metadata.uid())),
+                Some(Gid::from_raw(metadata.gid())),
+            )
+            .with_context(|| format!("Unable to set ownership for {}", dest_file_path.display()))?;
+
             return Ok(());
         }
 
@@ -504,6 +529,7 @@ mod tests {
     use super::*;
     use crate::mount::is_mounted;
     use crate::skip_if_not_root;
+    use nix::unistd::{Gid, Uid};
     use std::fs;
     use std::thread;
 
@@ -895,20 +921,28 @@ mod tests {
 
     #[tokio::test]
     async fn test_copy() {
+        skip_if_not_root!();
+
         // prepare tmp src/destination
         let source_dir = tempfile::tempdir().unwrap();
         let dest_dir = tempfile::tempdir().unwrap();
+        let uid = Uid::from_raw(10);
+        let gid = Gid::from_raw(200);
 
         // verify copy of a regular file
         let src_file = source_dir.path().join("file.txt");
         let dst_file = dest_dir.path().join("file.txt");
         fs::write(&src_file, "foo").unwrap();
+        nix::unistd::chown(&src_file, Some(uid), Some(gid)).unwrap();
+
         copy(&src_file, &dst_file).await.unwrap();
         // verify destination:
-        assert!(!fs::symlink_metadata(dst_file)
+        assert!(!fs::symlink_metadata(&dst_file)
             .unwrap()
             .file_type()
             .is_symlink());
+        assert_eq!(fs::metadata(&dst_file).unwrap().uid(), uid.as_raw());
+        assert_eq!(fs::metadata(&dst_file).unwrap().gid(), gid.as_raw());
 
         // verify copy of a symlink
         let src_symlink_file = source_dir.path().join("symlink_file.txt");
@@ -916,7 +950,7 @@ mod tests {
         tokio::fs::symlink(&src_file, &src_symlink_file)
             .await
             .unwrap();
-        copy(src_symlink_file, &dst_symlink_file).await.unwrap();
+        copy(&src_symlink_file, &dst_symlink_file).await.unwrap();
         // verify destination:
         assert!(fs::symlink_metadata(&dst_symlink_file)
             .unwrap()
@@ -924,6 +958,8 @@ mod tests {
             .is_symlink());
         assert_eq!(fs::read_link(&dst_symlink_file).unwrap(), src_file);
         assert_eq!(fs::read_to_string(&dst_symlink_file).unwrap(), "foo");
+        assert_ne!(fs::metadata(&dst_symlink_file).unwrap().uid(), uid.as_raw());
+        assert_ne!(fs::metadata(&dst_symlink_file).unwrap().gid(), gid.as_raw());
     }
 
     #[tokio::test]
@@ -1069,6 +1105,8 @@ mod tests {
 
     #[tokio::test]
     async fn watch_directory() {
+        skip_if_not_root!();
+
         // Prepare source directory:
         // ./tmp/1.txt
         // ./tmp/A/B/2.txt
@@ -1079,7 +1117,9 @@ mod tests {
 
         // A/C is an empty directory
         let empty_dir = "A/C";
-        fs::create_dir_all(source_dir.path().join(empty_dir)).unwrap();
+        let path = source_dir.path().join(empty_dir);
+        fs::create_dir_all(&path).unwrap();
+        nix::unistd::chown(&path, Some(Uid::from_raw(10)), Some(Gid::from_raw(200))).unwrap();
 
         // delay 20 ms between writes to files in order to ensure filesystem timestamps are unique
         thread::sleep(Duration::from_millis(20));
@@ -1123,7 +1163,9 @@ mod tests {
 
         // create another empty directory A/C/D
         let empty_dir = "A/C/D";
-        fs::create_dir_all(source_dir.path().join(empty_dir)).unwrap();
+        let path = source_dir.path().join(empty_dir);
+        fs::create_dir_all(&path).unwrap();
+        nix::unistd::chown(&path, Some(Uid::from_raw(10)), Some(Gid::from_raw(200))).unwrap();
         assert_eq!(entry.scan(&logger).await.unwrap(), 1);
         assert!(dest_dir.path().join(empty_dir).exists());
     }

src/libs/oci/src/lib.rs

@@ -381,7 +381,7 @@ pub struct LinuxMemory {
     #[serde(default, skip_serializing_if = "Option::is_none", rename = "kernelTCP")]
     pub kernel_tcp: Option<i64>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub swappiness: Option<i64>,
+    pub swappiness: Option<u64>,
     #[serde(
         default,
         skip_serializing_if = "Option::is_none",

src/libs/protocols/build.rs

@@ -95,6 +95,7 @@ fn real_main() -> Result<(), std::io::Error> {
 
     let protos = vec![
         "protos/agent.proto",
+        "protos/csi.proto",
         "protos/google/protobuf/empty.proto",
         "protos/health.proto",
         "protos/oci.proto",

src/libs/protocols/hack/update-generated-proto.sh

@@ -51,6 +51,8 @@ generate_go_sources() {
 --gogottrpc_out=plugins=ttrpc+fieldpath,\
 import_path=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc,\
 \
+Mgithub.com/kata-containers/kata-containers/src/libs/protocols/protos/csi.proto=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc,\
+\
 Mgithub.com/kata-containers/kata-containers/src/libs/protocols/protos/types.proto=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols,\
 \
 Mgithub.com/kata-containers/kata-containers/src/libs/protocols/protos/oci.proto=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc,\
@@ -69,7 +71,7 @@ if [ "$(basename $(pwd))" != "agent" ]; then
 fi
 
 # Protocol buffer files required to generate golang/rust bindings.
-proto_files_list=(agent.proto health.proto oci.proto types.proto)
+proto_files_list=(agent.proto csi.proto health.proto oci.proto types.proto)
 
 if [ "$1" = "" ]; then
     show_usage "${proto_files_list[@]}"

src/libs/protocols/protos/agent.proto

@@ -12,6 +12,7 @@ option go_package = "github.com/kata-containers/kata-containers/src/runtime/virt
 package grpc;
 
 import "oci.proto";
+import "csi.proto";
 import "types.proto";
 
 import "google/protobuf/empty.proto";
@@ -65,6 +66,8 @@ service AgentService {
 	rpc CopyFile(CopyFileRequest) returns (google.protobuf.Empty);
 	rpc GetOOMEvent(GetOOMEventRequest) returns (OOMEvent);
 	rpc AddSwap(AddSwapRequest) returns (google.protobuf.Empty);
+	rpc GetVolumeStats(VolumeStatsRequest) returns (VolumeStatsResponse);
+	rpc ResizeVolume(ResizeVolumeRequest) returns (google.protobuf.Empty);
 }
 
 message CreateContainerRequest {
@@ -396,6 +399,17 @@ message SetGuestDateTimeRequest {
 	int64 Usec = 2;
 }
 
+// FSGroup consists of the group id and group ownership change policy
+// that a volume should have its ownership changed to.
+message FSGroup {
+	// GroupID is the ID that the group ownership of the
+	// files in the mounted volume will need to be changed to.
+	uint32 group_id = 2;
+	// GroupChangePolicy specifies the policy for applying group id
+	// ownership change on a mounted volume.
+	types.FSGroupChangePolicy group_change_policy = 3;
+}
+
 // Storage represents both the rootfs of the container, and any volume that
 // could have been defined through the Mount list of the OCI specification.
 message Storage {
@@ -419,11 +433,14 @@ message Storage {
 	// device, "9p" for shared filesystem, or "tmpfs" for shared /dev/shm.
 	string fstype = 4;
 	// Options describes the additional options that might be needed to
-	// mount properly the storage filesytem.
+	// mount properly the storage filesystem.
 	repeated string options = 5;
 	// MountPoint refers to the path where the storage should be mounted
 	// inside the VM.
 	string mount_point = 6;
+	// FSGroup consists of the group ID and group ownership change policy
+	// that the mounted volume must have its group ID changed to when specified.
+	FSGroup fs_group = 7;
 }
 
 // Device represents only the devices that could have been defined through the
@@ -505,3 +522,14 @@ message GetMetricsRequest {}
 message Metrics {
 	string metrics = 1;
 }
+
+message VolumeStatsRequest {
+	// The volume path on the guest outside the container
+	string volume_guest_path = 1;
+}
+
+message ResizeVolumeRequest {
+	// Full VM guest path of the volume (outside the container)
+	string volume_guest_path = 1;
+	uint64 size = 2;
+}

src/libs/protocols/protos/csi.proto

@@ -0,0 +1,60 @@
+// Copyright (c) 2022 Databricks Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+syntax = "proto3";
+option go_package = "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc";
+
+package grpc;
+import "gogo/protobuf/gogoproto/gogo.proto";
+
+option (gogoproto.equal_all) = true;
+option (gogoproto.populate_all) = true;
+option (gogoproto.testgen_all) = true;
+option (gogoproto.benchgen_all) = true;
+
+// This should be kept in sync with CSI NodeGetVolumeStatsResponse (https://github.com/container-storage-interface/spec/blob/v1.5.0/csi.proto)
+message VolumeStatsResponse {
+  // This field is OPTIONAL.
+  repeated VolumeUsage usage = 1;
+  // Information about the current condition of the volume.
+  // This field is OPTIONAL.
+  // This field MUST be specified if the VOLUME_CONDITION node
+  // capability is supported.
+  VolumeCondition volume_condition = 2;
+}
+message VolumeUsage {
+  enum Unit {
+    UNKNOWN = 0;
+    BYTES = 1;
+    INODES = 2;
+  }
+  // The available capacity in specified Unit. This field is OPTIONAL.
+  // The value of this field MUST NOT be negative.
+  uint64 available = 1;
+
+  // The total capacity in specified Unit. This field is REQUIRED.
+  // The value of this field MUST NOT be negative.
+  uint64 total = 2;
+
+  // The used capacity in specified Unit. This field is OPTIONAL.
+  // The value of this field MUST NOT be negative.
+  uint64 used = 3;
+
+  // Units by which values are measured. This field is REQUIRED.
+  Unit unit = 4;
+}
+
+// VolumeCondition represents the current condition of a volume.
+message VolumeCondition {
+
+  // Normal volumes are available for use and operating optimally.
+  // An abnormal volume does not meet these criteria.
+  // This field is REQUIRED.
+  bool abnormal = 1;
+
+  // The message describing the condition of the volume.
+  // This field is REQUIRED.
+  string message = 2;
+}

src/libs/protocols/protos/oci.proto

@@ -6,11 +6,9 @@
 //
 
 syntax = "proto3";
-
 option go_package = "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc";
 
 package grpc;
-
 import "gogo/protobuf/gogoproto/gogo.proto";
 
 option (gogoproto.equal_all) = true;

src/libs/protocols/protos/types.proto

@@ -16,6 +16,15 @@ enum IPFamily {
 	v6 = 1;
 }
 
+// FSGroupChangePolicy defines the policy for applying group id ownership change on a mounted volume.
+enum FSGroupChangePolicy {
+	// Always indicates that the volume ownership will always be changed.
+	Always = 0;
+	// OnRootMismatch indicates that the volume ownership will be changed only
+	// when the ownership of the root directory does not match with the expected group id for the volume.
+	OnRootMismatch = 1;
+}
+
 message IPAddress {
 	IPFamily family = 1;
 	string address = 2;

src/libs/protocols/src/lib.rs

@@ -7,6 +7,7 @@
 
 pub mod agent;
 pub mod agent_ttrpc;
+pub mod csi;
 pub mod empty;
 pub mod health;
 pub mod health_ttrpc;

src/runtime/Makefile

@@ -1,5 +1,6 @@
 #
 # Copyright (c) 2018-2019 Intel Corporation
+# Copyright (c) 2021 Adobe Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -8,20 +9,19 @@ SKIP_GO_VERSION_CHECK=
 include golang.mk
 
 #Get ARCH.
-ifneq (,$(golang_version_raw))
-    GOARCH=$(shell go env GOARCH)
-    ifeq ($(ARCH),)
-        ARCH = $(GOARCH)
-    endif
-else
-    ARCH = $(shell uname -m)
-    ifeq ($(ARCH),x86_64)
-        ARCH = amd64
-    endif
-    ifeq ($(ARCH),aarch64)
-        ARCH = arm64
+ifeq ($(ARCH),)
+    ifneq (,$(golang_version_raw))
+        override ARCH = $(shell go env GOARCH)
+    else
+        override ARCH = $(shell uname -m)
     endif
 endif
+ifeq ($(ARCH),x86_64)
+    override ARCH = amd64
+endif
+ifeq ($(ARCH),aarch64)
+    override ARCH = arm64
+endif
 
 ARCH_DIR = arch
 ARCH_FILE_SUFFIX = -options.mk
@@ -29,8 +29,15 @@ ARCH_FILE = $(ARCH_DIR)/$(ARCH)$(ARCH_FILE_SUFFIX)
 ARCH_FILES = $(wildcard arch/*$(ARCH_FILE_SUFFIX))
 ALL_ARCHES = $(patsubst $(ARCH_DIR)/%$(ARCH_FILE_SUFFIX),%,$(ARCH_FILES))
 
-# Load architecture-dependent settings
-include $(ARCH_FILE)
+# Build as safely as possible
+export CGO_CPPFLAGS = -D_FORTIFY_SOURCE=2 -fstack-protector
+
+ifeq (,$(realpath $(ARCH_FILE)))
+    $(error "ERROR: invalid architecture: '$(ARCH)'")
+else
+    # Load architecture-dependent settings
+    include $(ARCH_FILE)
+endif
 
 PROJECT_TYPE = kata
 PROJECT_NAME = Kata Containers
@@ -104,6 +111,7 @@ KERNELDIR := $(PKGDATADIR)
 
 IMAGEPATH := $(PKGDATADIR)/$(IMAGENAME)
 FIRMWAREPATH :=
+FIRMWAREVOLUMEPATH :=
 
 # Name of default configuration file the runtime will use.
 CONFIG_FILE = configuration.toml
@@ -154,14 +162,18 @@ DEFMEMSLOTS := 10
 DEFBRIDGES := 1
 DEFENABLEANNOTATIONS := []
 DEFDISABLEGUESTSECCOMP := true
+DEFDISABLEGUESTEMPTYDIR := false
 #Default experimental features enabled
 DEFAULTEXPFEATURES := []
 
+DEFDISABLESELINUX := false
+
 #Default entropy source
 DEFENTROPYSOURCE := /dev/urandom
 DEFVALIDENTROPYSOURCES := [\"/dev/urandom\",\"/dev/random\",\"\"]
 
 DEFDISABLEBLOCK := false
+DEFSHAREDFS_CLH_VIRTIOFS := virtio-fs
 DEFSHAREDFS_QEMU_VIRTIOFS := virtio-fs
 DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/kata-qemu/virtiofsd
 DEFVALIDVIRTIOFSDAEMONPATHS := [\"$(DEFVIRTIOFSDAEMON)\"]
@@ -174,7 +186,7 @@ DEFVIRTIOFSCACHE ?= auto
 #
 # see `virtiofsd -h` for possible options.
 # Make sure you quote args.
-DEFVIRTIOFSEXTRAARGS ?= [\"--thread-pool-size=1\"]
+DEFVIRTIOFSEXTRAARGS ?= [\"--thread-pool-size=1\", \"-o\", \"announce_submounts\"]
 DEFENABLEIOTHREADS := false
 DEFENABLEVHOSTUSERSTORE := false
 DEFVHOSTUSERSTOREPATH := $(PKGRUNDIR)/vhost-user
@@ -191,9 +203,6 @@ DEFSTATICRESOURCEMGMT ?= false
 
 DEFBINDMOUNTS := []
 
-# Features
-FEATURE_SELINUX ?= check
-
 SED = sed
 
 CLI_DIR = cmd
@@ -393,6 +402,7 @@ USER_VARS += KERNELPATH_CLH
 USER_VARS += KERNELPATH_FC
 USER_VARS += KERNELVIRTIOFSPATH
 USER_VARS += FIRMWAREPATH
+USER_VARS += FIRMWAREVOLUMEPATH
 USER_VARS += MACHINEACCELERATORS
 USER_VARS += CPUFEATURES
 USER_VARS += DEFMACHINETYPE_CLH
@@ -429,12 +439,15 @@ USER_VARS += DEFNETWORKMODEL_ACRN
 USER_VARS += DEFNETWORKMODEL_CLH
 USER_VARS += DEFNETWORKMODEL_FC
 USER_VARS += DEFNETWORKMODEL_QEMU
+USER_VARS += DEFDISABLEGUESTEMPTYDIR
 USER_VARS += DEFDISABLEGUESTSECCOMP
+USER_VARS += DEFDISABLESELINUX
 USER_VARS += DEFAULTEXPFEATURES
 USER_VARS += DEFDISABLEBLOCK
 USER_VARS += DEFBLOCKSTORAGEDRIVER_ACRN
 USER_VARS += DEFBLOCKSTORAGEDRIVER_FC
 USER_VARS += DEFBLOCKSTORAGEDRIVER_QEMU
+USER_VARS += DEFSHAREDFS_CLH_VIRTIOFS
 USER_VARS += DEFSHAREDFS_QEMU_VIRTIOFS
 USER_VARS += DEFVIRTIOFSDAEMON
 USER_VARS += DEFVALIDVIRTIOFSDAEMONPATHS
@@ -456,7 +469,6 @@ USER_VARS += DEFSTATICRESOURCEMGMT
 USER_VARS += DEFSTATICRESOURCEMGMT_FC
 USER_VARS += DEFBINDMOUNTS
 USER_VARS += DEFVFIOMODE
-USER_VARS += FEATURE_SELINUX
 USER_VARS += BUILDFLAGS
 
 
@@ -471,21 +483,6 @@ QUIET_TEST     = $(Q:@=@echo    '     TEST     '$@;)
 
 BUILDTAGS :=
 
-ifneq ($(FEATURE_SELINUX),no)
-    SELINUXTAG := $(shell ./hack/selinux_tag.sh)
-
-    ifneq ($(SELINUXTAG),)
-        override FEATURE_SELINUX = yes
-        BUILDTAGS += --tags "$(SELINUXTAG)"
-    else
-        ifeq ($(FEATURE_SELINUX),yes)
-            $(error "ERROR: SELinux support requested, but libselinux is not available")
-        endif
-
-        override FEATURE_SELINUX = no
-    endif
-endif
-
 # go build common flags
 BUILDFLAGS := -buildmode=pie -mod=vendor ${BUILDTAGS}
 
@@ -564,8 +561,8 @@ $(SHIMV2_OUTPUT): $(SOURCES) $(GENERATED_FILES) $(MAKEFILE_LIST)
 	$(QUIET_BUILD)(cd $(SHIMV2_DIR)/ && go build -ldflags "$(KATA_LDFLAGS)" $(BUILDFLAGS) -o $@ .)
 
 $(MONITOR_OUTPUT): $(SOURCES) $(GENERATED_FILES) $(MAKEFILE_LIST) .git-commit
-	$(QUIET_BUILD)(cd $(MONITOR_DIR)/ && CGO_ENABLED=0 go build \
-		--ldflags "-X main.GitCommit=$(shell cat .git-commit)" $(BUILDFLAGS) -buildmode=exe -o $@ .)
+	$(QUIET_BUILD)(cd $(MONITOR_DIR)/ && go build \
+		--ldflags "-X main.GitCommit=$(shell cat .git-commit)" $(BUILDFLAGS) -o $@ .)
 
 .PHONY: \
 	check \
@@ -592,12 +589,10 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit
 
 generate-config: $(CONFIGS)
 
-test: install-hook go-test
+test: hook go-test
 
-install-hook:
+hook:
 	make -C virtcontainers hook
-	echo "installing mock hook"
-	sudo -E make -C virtcontainers install
 
 go-test: $(GENERATED_FILES)
 	go clean -testcache
@@ -745,9 +740,6 @@ endif
 	@printf "\tKnown: $(sort $(HYPERVISORS))\n"
 	@printf "\tAvailable for this architecture: $(sort $(KNOWN_HYPERVISORS))\n"
 	@printf "\n"
-	@printf "• Features:\n"
-	@printf "\tSELinux (FEATURE_SELINUX): $(FEATURE_SELINUX)\n"
-	@printf "\n"
 	@printf "• Summary:\n"
 	@printf "\n"
 	@printf "\tdestination install path (DESTDIR) : %s\n" $(abspath $(DESTDIR))

src/runtime/README.md

@@ -2,19 +2,26 @@
 
 # Runtime
 
-This repository contains the runtime for the
-[Kata Containers](https://github.com/kata-containers) project.
+## Binary names
+
+This repository contains the following components:
+
+| Binary name | Description |
+|-|-|
+| `containerd-shim-kata-v2` | The [shimv2 runtime](../../docs/design/architecture/README.md#runtime) |
+| `kata-runtime` | [utility program](../../docs/design/architecture/README.md#utility-program) |
+| `kata-monitor` | [metrics collector daemon](cmd/kata-monitor/README.md) |
 
 For details of the other Kata Containers repositories, see the
 [repository summary](https://github.com/kata-containers/kata-containers).
 
 ## Introduction
 
-`kata-runtime`, referred to as "the runtime", is the Command-Line Interface
-(CLI) part of the Kata Containers runtime component. It leverages the
+The `containerd-shim-kata-v2` [binary](#binary-names) is the Kata
+Containers [shimv2](../../docs/design/architecture/README.md#shim-v2-architecture) runtime. It leverages the
 [virtcontainers](virtcontainers)
 package to provide a high-performance standards-compliant runtime that creates
-hardware-virtualized [Linux](https://www.kernel.org/) containers running on Linux hosts.
+hardware-virtualized [Linux](https://www.kernel.org) containers running on Linux hosts.
 
 The runtime is
 [OCI](https://github.com/opencontainers/runtime-spec)-compatible,
@@ -23,51 +30,13 @@ The runtime is
  allowing it
 to work seamlessly with both Docker and Kubernetes respectively.
 
-## License
-
-The code is licensed under an Apache 2.0 license.
-See [the license file](https://github.com/kata-containers/kata-containers/blob/main/LICENSE) for further details.
-
-## Platform support
-
-Kata Containers currently works on systems supporting the following
-technologies:
-
-- [Intel](https://www.intel.com) VT-x technology.
-- [ARM](https://www.arm.com) Hyp mode (virtualization extension).
-- [IBM](https://www.ibm.com) Power Systems.
-- [IBM](https://www.ibm.com) Z mainframes.
-### Hardware requirements
-
-The runtime has a built-in command to determine if your host system is capable
-of running and creating a Kata Container:
-
-```bash
-$ kata-runtime check
-```
-
-> **Note:**
->
-> - By default, only a brief success / failure message is printed.
-> If more details are needed, the `--verbose` flag can be used to display the
-> list of all the checks performed.
->
-> - `root` permission is needed to check if the system is capable of running
-> Kata containers. In this case, additional checks are performed (e.g., if another
-> incompatible hypervisor is running).
-
 ## Download and install
 
 [![Get it from the Snap Store](https://snapcraft.io/static/images/badges/en/snap-store-black.svg)](https://snapcraft.io/kata-containers)
 
-See the [installation guides](https://github.com/kata-containers/kata-containers/blob/main/docs/install/README.md)
+See the [installation guides](../../docs/install/README.md)
 available for various operating systems.
 
-## Quick start for developers
-
-See the
-[developer guide](../../docs/Developer-Guide.md).
-
 ## Architecture overview
 
 See the [architecture overview](../../docs/design/architecture)
@@ -76,7 +45,11 @@ for details on the Kata Containers design.
 ## Configuration
 
 The runtime uses a TOML format configuration file called `configuration.toml`.
-The file contains comments explaining all options.
+The file is divided into sections for settings related to various
+parts of the system including the runtime itself, the [agent](../agent) and
+the [hypervisor](#hypervisor-specific-configuration).
+
+Each option has a comment explaining its use.
 
 > **Note:**
 >
@@ -84,6 +57,36 @@ The file contains comments explaining all options.
 > You may need to modify this file to optimise or tailor your system, or if you have
 > specific requirements.
 
+### Configuration file location
+
+#### Runtime configuration file location
+
+The shimv2 runtime looks for its configuration in the following places (in order):
+
+- The `io.data containers.config.config_path` annotation specified
+  in the OCI configuration file (`config.json` file) used to create the pod sandbox.
+
+- The containerd
+  [shimv2](/docs/design/architecture/README.md#shim-v2-architecture)
+  options passed to the runtime.
+
+- The value of the `KATA_CONF_FILE` environment variable.
+
+- The [default configuration paths](#stateless-systems).
+
+#### Utility program configuration file location
+
+The `kata-runtime` utility program looks for its configuration in the
+following locations (in order):
+
+- The path specified by the `--config` command-line option.
+
+- The value of the `KATA_CONF_FILE` environment variable.
+
+- The [default configuration paths](#stateless-systems).
+
+> **Note:** For both binaries, the first path that exists will be used.
+
 ### Hypervisor specific configuration
 
 Kata Containers supports multiple hypervisors so your `configuration.toml`
@@ -108,13 +111,6 @@ runtime attempts to load. The first path that exists will be used:
 $ kata-runtime --show-default-config-paths
 ```
 
-Aside from the built-in locations, it is possible to specify the path to a
-custom configuration file using the `--config` option:
-
-```bash
-$ kata-runtime --config=/some/where/configuration.toml ...
-```
-
 The runtime will log the full path to the configuration file it is using. See
 the [logging](#logging) section for further details.
 
@@ -132,27 +128,15 @@ components, see the documentation for the
 [`kata-log-parser`](https://github.com/kata-containers/tests/tree/main/cmd/log-parser)
 tool.
 
-For runtime logs, see the following sections for the CRI-O and containerd shimv2 based runtimes.
-
-### Kata OCI
-
-The Kata OCI runtime (including when used with CRI-O), provides `--log=` and `--log-format=` options.
-However, the runtime also always logs to the system log (`syslog` or `journald`).
-
-To view runtime log output:
-
-```bash
-$ sudo journalctl -t kata-runtime
-```
-
 ### Kata containerd shimv2
 
 The Kata containerd shimv2 runtime logs through `containerd`, and its logs will be sent
 to wherever the `containerd` logs are directed. However, the
-shimv2 runtime also always logs to the system log (`syslog` or `journald`) under the
-identifier name of `kata`.
+shimv2 runtime also always logs to the system log (`syslog` or `journald`) using the `kata` identifier.
 
-To view the `shimv2` runtime log output:
+> **Note:** Kata logging [requires containerd debug to be enabled](../../docs/Developer-Guide.md#enabling-full-containerd-debug).
+
+To view the `shimv2` runtime logs:
 
 ```bash
 $ sudo journalctl -t kata

src/runtime/cmd/kata-monitor/README.md

@@ -0,0 +1,68 @@
+# Kata monitor
+
+## Overview
+`kata-monitor` is a daemon able to collect and expose metrics related to all the Kata Containers workloads running on the same host.
+Once started, it detects all the running Kata Containers runtimes (`containerd-shim-kata-v2`) in the system and exposes few http endpoints to allow the retrieval of the available data.
+The main endpoint is the `/metrics` one which aggregates metrics from all the kata workloads.
+Available metrics include:
+  * Kata runtime metrics
+  * Kata agent metrics
+  * Kata guest OS metrics
+  * Hypervisor metrics
+  * Firecracker metrics
+  * Kata monitor metrics
+
+All the provided metrics are in Prometheus format. While `kata-monitor` can be used as a standalone daemon on any host running Kata Containers workloads and can be used for retrieving profiling data from the running Kata runtimes, its main expected usage is to be deployed as a DaemonSet on a Kubernetes cluster: there Prometheus should scrape the metrics from the kata-monitor endpoints.
+For more information on the Kata Containers metrics architecture and a detailed list of the available metrics provided by Kata monitor check the [Kata 2.0 Metrics Design](../../../../docs/design/kata-2-0-metrics.md) document.
+
+## Usage
+Each `kata-monitor` instance detects and monitors the Kata Container workloads running on the same node.
+
+### Kata monitor arguments
+The `kata-monitor` binary accepts the following arguments:
+
+* `--listen-address` _IP:PORT_
+* `--runtime-enpoint` _PATH_TO_THE_CONTAINER_MANAGER_CRI_INTERFACE_
+* `--log-level` _[ trace | debug | info | warn | error | fatal | panic ]_
+
+The **listen-address** specifies the IP and TCP port where the kata-monitor HTTP endpoints will be exposed. It defaults to `127.0.0.1:8090`.
+
+The **runtime-endpoint** is the CRI of a CRI compliant container manager: it will be used to retrieve the CRI `PodSandboxMetadata` (`uid`, `name` and `namespace`) which will be attached to the Kata metrics through the labels `cri_uid`, `cri_name` and `cri_namespace`. It defaults to the containerd socket: `/run/containerd/containerd.sock`.
+
+The **log-level** allows the chose how verbose the logs should be. The default is `info`.
+### Kata monitor HTTP endpoints
+`kata-monitor` exposes the following endpoints:
+  * `/metrics`             : get Kata sandboxes metrics.
+  * `/sandboxes`           : list all the Kata sandboxes running on the host.
+  * `/agent-url`           : Get the agent URL of a Kata sandbox.
+  * `/debug/vars`          : Internal data of the Kata runtime shim.
+  * `/debug/pprof/`        : Golang profiling data of the Kata runtime shim: index page.
+  * `/debug/pprof/cmdline` : Golang profiling data of the Kata runtime shim: `cmdline` endpoint.
+  * `/debug/pprof/profile` : Golang profiling data of the Kata runtime shim: `profile` endpoint (CPU profiling).
+  * `/debug/pprof/symbol`  : Golang profiling data of the Kata runtime shim: `symbol` endpoint.
+  * `/debug/pprof/trace`   : Golang profiling data of the Kata runtime shim: `trace` endpoint.
+
+**NOTE: The debug endpoints are available only if the [Kata Containers configuration file](https://github.com/kata-containers/kata-containers/blob/9d5b03a1b70bbd175237ec4b9f821d6ccee0a1f6/src/runtime/config/configuration-qemu.toml.in#L590-L592) includes** `enable_pprof = true` **in the** `[runtime]` **section**.
+
+The `/sandboxes` endpoint lists the _sandbox ID_ of all the detected Kata runtimes. If accessed via a web browser, it provides html links to the endpoints available for each sandbox.
+
+In order to retrieve data for a specific Kata workload, the _sandbox ID_ should be passed in the query string using the _sandbox_ key. The `/agent-url`, and all the `/debug/`* endpoints require `sandbox_id` to be specified in the query string.
+<br>
+#### Examples
+Retrieve the IDs of the available sandboxes:
+```bash
+$ curl 127.0.0.1:8090/sandboxes
+```
+output:
+```
+6fcf0a90b01e90d8747177aa466c3462d02e02a878bc393649df83d4c314af0c
+df96b24bd49ec437c872c1a758edc084121d607ce1242ff5d2263a0e1b693343
+```
+Retrieve the `agent-url` of the sandbox with ID _df96b24bd49ec437c872c1a758edc084121d607ce1242ff5d2263a0e1b693343_:
+```bash
+$ curl 127.0.0.1:8090/agent-url?sandbox=df96b24bd49ec437c872c1a758edc084121d607ce1242ff5d2263a0e1b693343
+```
+output:
+```
+vsock://830455376:1024
+```

src/runtime/cmd/kata-monitor/main.go

@@ -18,15 +18,17 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-var monitorListenAddr = flag.String("listen-address", ":8090", "The address to listen on for HTTP requests.")
-var runtimeEndpoint = flag.String("runtime-endpoint", "/run/containerd/containerd.sock", `Endpoint of CRI container runtime service. (default: "/run/containerd/containerd.sock")`)
+const defaultListenAddress = "127.0.0.1:8090"
+
+var monitorListenAddr = flag.String("listen-address", defaultListenAddress, "The address to listen on for HTTP requests.")
+var runtimeEndpoint = flag.String("runtime-endpoint", "/run/containerd/containerd.sock", "Endpoint of CRI container runtime service.")
 var logLevel = flag.String("log-level", "info", "Log level of logrus(trace/debug/info/warn/error/fatal/panic).")
 
 // These values are overridden via ldflags
 var (
 	appName = "kata-monitor"
 	// version is the kata monitor version.
-	version = "0.2.0"
+	version = "0.3.0"
 
 	GitCommit = "unknown-commit"
 )
@@ -173,6 +175,15 @@ func main() {
 }
 
 func indexPage(w http.ResponseWriter, r *http.Request) {
+	htmlResponse := kataMonitor.IfReturnHTMLResponse(w, r)
+	if htmlResponse {
+		indexPageHTML(w, r)
+	} else {
+		indexPageText(w, r)
+	}
+}
+
+func indexPageText(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte("Available HTTP endpoints:\n"))
 
 	spacing := 0
@@ -182,13 +193,35 @@ func indexPage(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 	spacing = spacing + 3
+	formatter := fmt.Sprintf("%%-%ds: %%s\n", spacing)
 
-	formattedString := fmt.Sprintf("%%-%ds: %%s\n", spacing)
 	for _, endpoint := range endpoints {
-		w.Write([]byte(fmt.Sprintf(formattedString, endpoint.path, endpoint.desc)))
+		w.Write([]byte(fmt.Sprintf(formatter, endpoint.path, endpoint.desc)))
 	}
 }
 
+func indexPageHTML(w http.ResponseWriter, r *http.Request) {
+
+	w.Write([]byte("<h1>Available HTTP endpoints:</h1>\n"))
+
+	var formattedString string
+	needLinkPaths := []string{"/metrics", "/sandboxes"}
+
+	w.Write([]byte("<ul>"))
+	for _, endpoint := range endpoints {
+		formattedString = fmt.Sprintf("<b>%s</b>: %s\n", endpoint.path, endpoint.desc)
+		for _, linkPath := range needLinkPaths {
+			if linkPath == endpoint.path {
+				formattedString = fmt.Sprintf("<b><a href='%s'>%s</a></b>: %s\n", endpoint.path, endpoint.path, endpoint.desc)
+				break
+			}
+		}
+		formattedString = fmt.Sprintf("<li>%s</li>", formattedString)
+		w.Write([]byte(formattedString))
+	}
+	w.Write([]byte("</ul>"))
+}
+
 // initLog setup logger
 func initLog() {
 	kataMonitorLog := logrus.WithFields(logrus.Fields{

src/runtime/cmd/kata-runtime/factory.go

@@ -24,6 +24,7 @@ import (
 	"github.com/urfave/cli"
 	"golang.org/x/sys/unix"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 )
 
 var factorySubCmds = []cli.Command{
@@ -235,7 +236,7 @@ var destroyFactoryCommand = cli.Command{
 		}
 
 		if runtimeConfig.FactoryConfig.VMCacheNumber > 0 {
-			conn, err := grpc.Dial(fmt.Sprintf("unix://%s", runtimeConfig.FactoryConfig.VMCacheEndpoint), grpc.WithInsecure())
+			conn, err := grpc.Dial(fmt.Sprintf("unix://%s", runtimeConfig.FactoryConfig.VMCacheEndpoint), grpc.WithTransportCredentials(insecure.NewCredentials()))
 			if err != nil {
 				return errors.Wrapf(err, "failed to connect %q", runtimeConfig.FactoryConfig.VMCacheEndpoint)
 			}
@@ -285,7 +286,7 @@ var statusFactoryCommand = cli.Command{
 		}
 
 		if runtimeConfig.FactoryConfig.VMCacheNumber > 0 {
-			conn, err := grpc.Dial(fmt.Sprintf("unix://%s", runtimeConfig.FactoryConfig.VMCacheEndpoint), grpc.WithInsecure())
+			conn, err := grpc.Dial(fmt.Sprintf("unix://%s", runtimeConfig.FactoryConfig.VMCacheEndpoint), grpc.WithTransportCredentials(insecure.NewCredentials()))
 			if err != nil {
 				fmt.Fprintln(defaultOutputFile, errors.Wrapf(err, "failed to connect %q", runtimeConfig.FactoryConfig.VMCacheEndpoint))
 			} else {

src/runtime/cmd/kata-runtime/kata-exec.go

@@ -19,8 +19,8 @@ import (
 	"time"
 
 	"github.com/containerd/console"
-	kataMonitor "github.com/kata-containers/kata-containers/src/runtime/pkg/kata-monitor"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils/shimclient"
 	clientUtils "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/client"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli"
@@ -154,7 +154,7 @@ func (s *iostream) Read(data []byte) (n int, err error) {
 }
 
 func getConn(sandboxID string, port uint64) (net.Conn, error) {
-	client, err := kataMonitor.BuildShimClient(sandboxID, defaultTimeout)
+	client, err := shimclient.BuildShimClient(sandboxID, defaultTimeout)
 	if err != nil {
 		return nil, err
 	}

src/runtime/cmd/kata-runtime/kata-volume.go

@@ -0,0 +1,167 @@
+// Copyright (c) 2022 Databricks Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/url"
+
+	containerdshim "github.com/kata-containers/kata-containers/src/runtime/pkg/containerd-shim-v2"
+	volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils/shimclient"
+
+	"github.com/urfave/cli"
+)
+
+var volumeSubCmds = []cli.Command{
+	addCommand,
+	removeCommand,
+	statsCommand,
+	resizeCommand,
+}
+
+var (
+	mountInfo  string
+	volumePath string
+	size       uint64
+)
+
+var kataVolumeCommand = cli.Command{
+	Name:        "direct-volume",
+	Usage:       "directly assign a volume to Kata Containers to manage",
+	Subcommands: volumeSubCmds,
+	Action: func(context *cli.Context) {
+		cli.ShowSubcommandHelp(context)
+	},
+}
+
+var addCommand = cli.Command{
+	Name:  "add",
+	Usage: "add a direct assigned block volume device to the Kata Containers runtime",
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:        "volume-path",
+			Usage:       "the target volume path the volume is published to",
+			Destination: &volumePath,
+		},
+		cli.StringFlag{
+			Name:        "mount-info",
+			Usage:       "the mount info for the Kata Containers runtime to manage the volume",
+			Destination: &mountInfo,
+		},
+	},
+	Action: func(c *cli.Context) error {
+		if err := volume.Add(volumePath, mountInfo); err != nil {
+			return cli.NewExitError(err.Error(), 1)
+		}
+		return nil
+	},
+}
+
+var removeCommand = cli.Command{
+	Name:  "remove",
+	Usage: "remove a direct assigned block volume device from the Kata Containers runtime",
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:        "volume-path",
+			Usage:       "the target volume path the volume is published to",
+			Destination: &volumePath,
+		},
+	},
+	Action: func(c *cli.Context) error {
+		if err := volume.Remove(volumePath); err != nil {
+			return cli.NewExitError(err.Error(), 1)
+		}
+		return nil
+	},
+}
+
+var statsCommand = cli.Command{
+	Name:  "stats",
+	Usage: "get the filesystem stat of a direct assigned volume",
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:        "volume-path",
+			Usage:       "the target volume path the volume is published to",
+			Destination: &volumePath,
+		},
+	},
+	Action: func(c *cli.Context) error {
+		stats, err := Stats(volumePath)
+		if err != nil {
+			return cli.NewExitError(err.Error(), 1)
+		}
+
+		fmt.Println(string(stats))
+		return nil
+	},
+}
+
+var resizeCommand = cli.Command{
+	Name:  "resize",
+	Usage: "resize a direct assigned block volume",
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:        "volume-path",
+			Usage:       "the target volume path the volume is published to",
+			Destination: &volumePath,
+		},
+		cli.Uint64Flag{
+			Name:        "size",
+			Usage:       "the new size of the volume",
+			Destination: &size,
+		},
+	},
+	Action: func(c *cli.Context) error {
+		if err := Resize(volumePath, size); err != nil {
+			return cli.NewExitError(err.Error(), 1)
+		}
+		return nil
+	},
+}
+
+// Stats retrieves the filesystem stats of the direct volume inside the guest.
+func Stats(volumePath string) ([]byte, error) {
+	sandboxId, err := volume.GetSandboxIdForVolume(volumePath)
+	if err != nil {
+		return nil, err
+	}
+	volumeMountInfo, err := volume.VolumeMountInfo(volumePath)
+	if err != nil {
+		return nil, err
+	}
+
+	urlSafeDevicePath := url.PathEscape(volumeMountInfo.Device)
+	body, err := shimclient.DoGet(sandboxId, defaultTimeout,
+		fmt.Sprintf("%s?%s=%s", containerdshim.DirectVolumeStatUrl, containerdshim.DirectVolumePathKey, urlSafeDevicePath))
+	if err != nil {
+		return nil, err
+	}
+	return body, nil
+}
+
+// Resize resizes a direct volume inside the guest.
+func Resize(volumePath string, size uint64) error {
+	sandboxId, err := volume.GetSandboxIdForVolume(volumePath)
+	if err != nil {
+		return err
+	}
+	volumeMountInfo, err := volume.VolumeMountInfo(volumePath)
+	if err != nil {
+		return err
+	}
+
+	resizeReq := containerdshim.ResizeRequest{
+		VolumePath: volumeMountInfo.Device,
+		Size:       size,
+	}
+	encoded, err := json.Marshal(resizeReq)
+	if err != nil {
+		return err
+	}
+	return shimclient.DoPost(sandboxId, defaultTimeout, containerdshim.DirectVolumeResizeUrl, encoded)
+}

src/runtime/cmd/kata-runtime/main.go

@@ -124,6 +124,7 @@ var runtimeCommands = []cli.Command{
 	kataExecCLICommand,
 	kataMetricsCLICommand,
 	factoryCLICommand,
+	kataVolumeCommand,
 }
 
 // runtimeBeforeSubcommands is the function to run before command-line

src/runtime/config/configuration-acrn.toml.in

@@ -1,4 +1,5 @@
 # Copyright (c) 2017-2019 Intel Corporation
+# Copyright (c) 2021 Adobe Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -185,6 +186,9 @@ internetworking_model="@DEFNETWORKMODEL_ACRN@"
 # (default: true)
 disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
 
+# disable applying SELinux on the VMM process (default false)
+disable_selinux=@DEFDISABLESELINUX@
+
 # If enabled, the runtime will create opentracing.io traces and spans.
 # (See https://www.jaegertracing.io/docs/getting-started).
 # (default: disabled)
@@ -216,6 +220,10 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
 # See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType
 sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
 
+# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
+# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
+disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@
+
 # Enabled experimental feature list, format: ["a", "b"].
 # Experimental features are features not stable enough for production,
 # they may break compatibility, and are prepared for a big version bump.

src/runtime/config/configuration-clh.toml.in

@@ -1,4 +1,5 @@
 # Copyright (c) 2019 Ericsson Eurolab Deutschland GmbH
+# Copyright (c) 2021 Adobe Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -15,6 +16,40 @@ path = "@CLHPATH@"
 kernel = "@KERNELPATH_CLH@"
 image = "@IMAGEPATH@"
 
+# Enable confidential guest support.
+# Toggling that setting may trigger different hardware features, ranging
+# from memory encryption to both memory and CPU-state encryption and integrity.
+# The Kata Containers runtime dynamically detects the available feature set and
+# aims at enabling the largest possible one, returning an error if none is
+# available, or none is supported by the hypervisor.
+#
+# Known limitations:
+# * Does not work by design:
+#   - CPU Hotplug 
+#   - Memory Hotplug
+#   - NVDIMM devices
+#   - SharedFS, such as virtio-fs and virtio-fs-nydus
+#
+# Requirements:
+# * virtio-block used as rootfs, thus the usage of devmapper snapshotter.
+#
+# Supported TEEs:
+# * Intel TDX
+#
+# Default false
+# confidential_guest = true
+
+# Path to the firmware.
+# If you want Cloud Hypervisor to use a specific firmware, set its path below.
+# This is option is only used when confidential_guest is enabled.
+#
+# For more information about firmwared that can be used with specific TEEs,
+# please, refer to:
+# * Intel TDX:
+#   - td-shim: https://github.com/confidential-containers/td-shim
+#
+# firmware = "@FIRMWAREPATH@"
+
 # List of valid annotation names for the hypervisor
 # Each member of the list is a regular expression, which is the base name
 # of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
@@ -70,6 +105,11 @@ default_memory = @DEFMEMSZ@
 # This is will determine the times that memory will be hotadded to sandbox/VM.
 #memory_slots = @DEFMEMSLOTS@
 
+# Shared file system type:
+#   - virtio-fs (default)
+#   - virtio-fs-nydus
+shared_fs = "@DEFSHAREDFS_CLH_VIRTIOFS@"
+
 # Path to vhost-user-fs daemon.
 virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@"
 
@@ -200,6 +240,9 @@ internetworking_model="@DEFNETWORKMODEL_CLH@"
 # (default: true)
 disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
 
+# disable applying SELinux on the VMM process (default false)
+disable_selinux=@DEFDISABLESELINUX@
+
 # If enabled, the runtime will create opentracing.io traces and spans.
 # (See https://www.jaegertracing.io/docs/getting-started).
 # (default: disabled)
@@ -267,6 +310,10 @@ sandbox_bind_mounts=@DEFBINDMOUNTS@
 #
 vfio_mode="@DEFVFIOMODE@"
 
+# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
+# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
+disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@
+
 # Enabled experimental feature list, format: ["a", "b"].
 # Experimental features are features not stable enough for production,
 # they may break compatibility, and are prepared for a big version bump.

src/runtime/config/configuration-fc.toml.in

@@ -1,4 +1,5 @@
 # Copyright (c) 2017-2019 Intel Corporation
+# Copyright (c) Adobe Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -103,14 +104,6 @@ default_memory = @DEFMEMSZ@
 # Default 0
 #memory_offset = 0
 
-# Disable block device from being used for a container's rootfs.
-# In case of a storage driver like devicemapper where a container's
-# root file system is backed by a block device, the block device is passed
-# directly to the hypervisor for performance reasons.
-# This flag prevents the block device from being passed to the hypervisor,
-# 9pfs is used instead to pass the rootfs.
-disable_block_device_use = @DEFDISABLEBLOCK@
-
 # Block storage driver to be used for the hypervisor in case the container
 # rootfs is backed by a block device. This is virtio-scsi, virtio-blk
 # or nvdimm.
@@ -309,6 +302,9 @@ internetworking_model="@DEFNETWORKMODEL_FC@"
 # (default: true)
 disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
 
+# disable applying SELinux on the VMM process (default false)
+disable_selinux=@DEFDISABLESELINUX@
+
 # If enabled, the runtime will create opentracing.io traces and spans.
 # (See https://www.jaegertracing.io/docs/getting-started).
 # (default: disabled)
@@ -349,6 +345,10 @@ sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
 # - When running single containers using a tool like ctr, container sizing information will be available.
 static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT_FC@
 
+# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
+# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
+disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@
+
 # Enabled experimental feature list, format: ["a", "b"].
 # Experimental features are features not stable enough for production,
 # they may break compatibility, and are prepared for a big version bump.

src/runtime/config/configuration-qemu.toml.in

@@ -1,4 +1,5 @@
 # Copyright (c) 2017-2019 Intel Corporation
+# Copyright (c) 2021 Adobe Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -20,7 +21,15 @@ machine_type = "@MACHINETYPE@"
 # Toggling that setting may trigger different hardware features, ranging
 # from memory encryption to both memory and CPU-state encryption and integrity.
 # The Kata Containers runtime dynamically detects the available feature set and
-# aims at enabling the largest possible one.
+# aims at enabling the largest possible one, returning an error if none is
+# available, or none is supported by the hypervisor.
+#
+# Known limitations:
+# * Does not work by design:
+#   - CPU Hotplug 
+#   - Memory Hotplug
+#   - NVDIMM devices
+#
 # Default false
 # confidential_guest = true
 
@@ -56,6 +65,12 @@ kernel_params = "@KERNELPARAMS@"
 # If you want that qemu uses the default firmware leave this option empty
 firmware = "@FIRMWAREPATH@"
 
+# Path to the firmware volume.
+# firmware TDVF or OVMF can be split into FIRMWARE_VARS.fd (UEFI variables
+# as configuration) and FIRMWARE_CODE.fd (UEFI program image). UEFI variables
+# can be customized per each user while UEFI code is kept same.
+firmware_volume = "@FIRMWAREVOLUMEPATH@"
+
 # Machine accelerators
 # comma-separated list of machine accelerators to pass to the hypervisor.
 # For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"`
@@ -129,12 +144,13 @@ default_memory = @DEFMEMSZ@
 # root file system is backed by a block device, the block device is passed
 # directly to the hypervisor for performance reasons.
 # This flag prevents the block device from being passed to the hypervisor,
-# 9pfs is used instead to pass the rootfs.
+# virtio-fs is used instead to pass the rootfs.
 disable_block_device_use = @DEFDISABLEBLOCK@
 
 # Shared file system type:
 #   - virtio-fs (default)
 #   - virtio-9p
+#   - virtio-fs-nydus
 shared_fs = "@DEFSHAREDFS_QEMU_VIRTIOFS@"
 
 # Path to vhost-user-fs daemon.
@@ -272,6 +288,9 @@ pflashes = []
 
 # If false and nvdimm is supported, use nvdimm device to plug guest image.
 # Otherwise virtio-block device is used.
+#
+# nvdimm is not supported when `confidential_guest = true`.
+#
 # Default is false
 #disable_image_nvdimm = true
 
@@ -485,6 +504,9 @@ internetworking_model="@DEFNETWORKMODEL_QEMU@"
 # (default: true)
 disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
 
+# disable applying SELinux on the VMM process (default false)
+disable_selinux=@DEFDISABLESELINUX@
+
 # If enabled, the runtime will create opentracing.io traces and spans.
 # (See https://www.jaegertracing.io/docs/getting-started).
 # (default: disabled)
@@ -552,6 +574,10 @@ sandbox_bind_mounts=@DEFBINDMOUNTS@
 #
 vfio_mode="@DEFVFIOMODE@"
 
+# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
+# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
+disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@
+
 # Enabled experimental feature list, format: ["a", "b"].
 # Experimental features are features not stable enough for production,
 # they may break compatibility, and are prepared for a big version bump.

src/runtime/go.mod

@@ -3,13 +3,13 @@ module github.com/kata-containers/kata-containers/src/runtime
 go 1.14
 
 require (
+	code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5
 	github.com/BurntSushi/toml v0.3.1
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/blang/semver/v4 v4.0.0
-	github.com/containerd/cgroups v1.0.2
+	github.com/containerd/cgroups v1.0.3
 	github.com/containerd/console v1.0.3
-	github.com/containerd/containerd v1.6.0-beta.4
-	github.com/containerd/containerd/api v1.6.0-beta.3
+	github.com/containerd/containerd v1.6.1
 	github.com/containerd/cri-containerd v1.11.1-0.20190125013620-4dd6735020f5
 	github.com/containerd/fifo v1.0.0
 	github.com/containerd/ttrpc v1.1.0
@@ -17,6 +17,7 @@ require (
 	github.com/containernetworking/plugins v1.0.1
 	github.com/coreos/go-systemd/v22 v22.3.2
 	github.com/cri-o/cri-o v1.0.0-rc2.0.20170928185954-3394b3b2d6af
+	github.com/docker/go-units v0.4.0
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/go-ini/ini v1.28.2
 	github.com/go-openapi/errors v0.18.0
@@ -26,36 +27,34 @@ require (
 	github.com/go-openapi/validate v0.18.0
 	github.com/godbus/dbus/v5 v5.0.4
 	github.com/gogo/protobuf v1.3.2
-	github.com/hashicorp/go-multierror v1.0.0
+	github.com/hashicorp/go-multierror v1.1.1
 	github.com/intel-go/cpuid v0.0.0-20210602155658-5747e5cec0d9
-	github.com/mdlayher/vsock v0.0.0-20191108225356-d9c65923cb8f
+	github.com/mdlayher/vsock v1.1.0
 	github.com/opencontainers/image-spec v1.0.2 // indirect
-	github.com/opencontainers/runc v1.0.3
+	github.com/opencontainers/runc v1.1.0
 	github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417
-	github.com/opencontainers/selinux v1.8.2
+	github.com/opencontainers/selinux v1.10.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.11.0
 	github.com/prometheus/client_model v0.2.0
-	github.com/prometheus/common v0.26.0
-	github.com/prometheus/procfs v0.6.0
+	github.com/prometheus/common v0.30.0
+	github.com/prometheus/procfs v0.7.3
 	github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
 	github.com/urfave/cli v1.22.2
 	github.com/vishvananda/netlink v1.1.1-0.20210924202909-187053b97868
 	github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f
-	go.opencensus.io v0.23.0 // indirect
-	go.opentelemetry.io/otel v1.0.1
+	go.opentelemetry.io/otel v1.3.0
 	go.opentelemetry.io/otel/exporters/jaeger v1.0.0
-	go.opentelemetry.io/otel/sdk v1.0.1
-	go.opentelemetry.io/otel/trace v1.0.1
-	golang.org/x/net v0.0.0-20210825183410-e898025ed96a
-	golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93
-	golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/grpc v1.41.0
-	k8s.io/apimachinery v0.22.0
-	k8s.io/cri-api v0.23.0-alpha.4
+	go.opentelemetry.io/otel/sdk v1.3.0
+	go.opentelemetry.io/otel/trace v1.3.0
+	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd
+	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f
+	golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a
+	google.golang.org/grpc v1.43.0
+	k8s.io/apimachinery v0.22.5
+	k8s.io/cri-api v0.23.1
 )
 
 replace (

src/runtime/go.sum

@@ -15,6 +15,11 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV
 cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
 cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
 cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
+cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
+cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
+cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
+cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
@@ -33,6 +38,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5 h1:tM5+dn2C9xZw1RzgI6WTQW1rGqdUimKB3RFbyu4h6Hc=
+code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5/go.mod h1:v4VVB6oBMz/c9fRY6vZrwr5xKRWOH5NPDjQZlPk0Gbs=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg=
 github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
@@ -74,8 +81,9 @@ github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn
 github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600=
 github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4=
 github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4=
-github.com/Microsoft/hcsshim v0.9.1 h1:VfDCj+QnY19ktX5TsH22JHcjaZ05RWQiwDbOyEg5ziM=
-github.com/Microsoft/hcsshim v0.9.1/go.mod h1:Y/0uV2jUab5kBI7SQgl62at0AVX7uaruzADAVmxm3eM=
+github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg=
+github.com/Microsoft/hcsshim v0.9.2 h1:wB06W5aYFfUB3IvootYAY2WnOmIdgPGfqSI6tufQNnY=
+github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc=
 github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU=
 github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
@@ -129,13 +137,15 @@ github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8n
 github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50=
 github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
 github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
+github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
 github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
-github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
+github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@@ -148,9 +158,13 @@ github.com/cilium/ebpf v0.6.2 h1:iHsfF/t4aW4heW2YKfeHrVPGdtYTL4C4KocpM8KTSnI=
 github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
 github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
 github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
@@ -169,8 +183,8 @@ github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4S
 github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo=
 github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE=
 github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU=
-github.com/containerd/cgroups v1.0.2 h1:mZBclaSgNDfPWtfhj2xJY28LZ9nYIgzB0pwSURPl6JM=
-github.com/containerd/cgroups v1.0.2/go.mod h1:qpbpJ1jmlqsR9f2IyaLPsdkCdnt0rbDVqIDlhuu5tRY=
+github.com/containerd/cgroups v1.0.3 h1:ADZftAkglvCiD44c77s5YmMqaP2pzVCFZvBmAlBdAP4=
+github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8=
 github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
 github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
 github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE=
@@ -186,16 +200,16 @@ github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMX
 github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
+github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ=
 github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU=
 github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI=
 github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s=
 github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g=
 github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c=
-github.com/containerd/containerd v1.6.0-beta.4 h1:khjl5fy4TaUm8Nq9P8yPGzfgyDc5HE7/qsk5utCMsCc=
-github.com/containerd/containerd v1.6.0-beta.4/go.mod h1:K/hqBtTs+ifkK1zGMfm6YSKBSdxuyu/9jC+ThEbUMos=
-github.com/containerd/containerd/api v1.6.0-beta.3 h1:+w8zh0hbn4cNIkAtt4v95dBylcwp1hEsFJ5lxbr8wgY=
-github.com/containerd/containerd/api v1.6.0-beta.3/go.mod h1:fkctx1jj7m92mQDI6mIEXF+SH3tt2Rv/azUHqrOxYPc=
+github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s=
+github.com/containerd/containerd v1.6.1 h1:oa2uY0/0G+JX4X7hpGCYvkp9FjUancz56kSNnb1sG3o=
+github.com/containerd/containerd v1.6.1/go.mod h1:1nJz5xCZPusx6jJU8Frfct988y0NpumIq9ODB0kLtoE=
 github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
@@ -203,8 +217,8 @@ github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cE
 github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y=
 github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ=
 github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM=
-github.com/containerd/continuity v0.2.2-0.20211201162329-8e53e7cac79d h1:X5aD4AgIfNi00260miYN8nCzxPQCusmnNapaphze0FM=
-github.com/containerd/continuity v0.2.2-0.20211201162329-8e53e7cac79d/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk=
+github.com/containerd/continuity v0.2.2 h1:QSqfxcn8c+12slxwu00AtzXrsami0MJb/MQs9lOLHLA=
+github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk=
 github.com/containerd/cri-containerd v1.11.1-0.20190125013620-4dd6735020f5 h1:/srF029I+oDfm/qeltxCGJyJ8urmlqWGOQmQ7HvwrRc=
 github.com/containerd/cri-containerd v1.11.1-0.20190125013620-4dd6735020f5/go.mod h1:wxbGdReWGCalzGOEpifoHeYCK4xAgnj4o/4bVB+9voU=
 github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
@@ -216,7 +230,8 @@ github.com/containerd/fifo v1.0.0 h1:6PirWBr9/L7GDamKr+XM0IeUFXu5mf3M/BPpH9gaLBU
 github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4=
 github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU=
 github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk=
-github.com/containerd/go-cni v1.1.1-0.20211026134925-aa8bf14323a5/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA=
+github.com/containerd/go-cni v1.1.0/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA=
+github.com/containerd/go-cni v1.1.3/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA=
 github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
 github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
 github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g=
@@ -227,6 +242,7 @@ github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak
 github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA=
 github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow=
 github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms=
+github.com/containerd/imgcrypt v1.1.3/go.mod h1:/TPA1GIDXMzbj01yd8pIbQiLdQxed5ue1wb8bP7PQu4=
 github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c=
 github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
 github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
@@ -260,6 +276,7 @@ github.com/containernetworking/plugins v1.0.1/go.mod h1:QHCfGpaTwYTbbH+nZXKVTxNB
 github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc=
 github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4=
 github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY=
+github.com/containers/ocicrypt v1.1.2/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -327,6 +344,7 @@ github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
@@ -366,6 +384,13 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.2 h1:ahHml/yUpnlb96Rp8HCvtYVPY8ZYpxq3g7UYchIYwbs=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI=
 github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
 github.com/go-openapi/analysis v0.17.2 h1:eYp14J1o8TTSCzndHBtsNuckikV1PfZOSnx4BcBeu0c=
@@ -446,6 +471,7 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
 github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -479,14 +505,17 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
+github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
@@ -494,6 +523,10 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -525,14 +558,16 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFb
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
+github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
-github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
@@ -548,6 +583,7 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
@@ -556,6 +592,7 @@ github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/intel-go/cpuid v0.0.0-20210602155658-5747e5cec0d9 h1:x9HFDMDCsaxTvC4X3o0ZN6mw99dT/wYnTItGwhBRmg0=
 github.com/intel-go/cpuid v0.0.0-20210602155658-5747e5cec0d9/go.mod h1:RmeVYf9XrPRbRc3XIx0gLYA8qOFvNoPOfaEZduRlEp4=
+github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ=
 github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
 github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -570,6 +607,7 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
@@ -617,8 +655,10 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY=
-github.com/mdlayher/vsock v0.0.0-20191108225356-d9c65923cb8f h1:t9bhAC/9+wqdIb49Jamux+Sxqa7MhkyuTtsHkmVg6tk=
-github.com/mdlayher/vsock v0.0.0-20191108225356-d9c65923cb8f/go.mod h1:4GtNxrXX+cNil8xnCdz0zGYemDZDDHSsXbopCRZrRRw=
+github.com/mdlayher/socket v0.2.0 h1:EY4YQd6hTAg2tcXF84p5DTHazShE50u5HeBzBaNgjkA=
+github.com/mdlayher/socket v0.2.0/go.mod h1:QLlNPkFR88mRUNQIzRBMfXxwKal8H7u1h3bL1CV+f0E=
+github.com/mdlayher/vsock v1.1.0 h1:2k9udP/hUkLUOboGxXMHOk4f0GWWZwS3IuE3Ee/YYfk=
+github.com/mdlayher/vsock v1.1.0/go.mod h1:nsVhPsVuBBwAKh6i6PzdNoke6/TNYTjkxoRKAp/+pXs=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
@@ -647,6 +687,7 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
@@ -655,7 +696,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
@@ -683,8 +723,9 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J
 github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
-github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU=
 github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
+github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c=
+github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
@@ -705,8 +746,9 @@ github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.m
 github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
 github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE=
 github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo=
-github.com/opencontainers/selinux v1.8.2 h1:c4ca10UMgRcvZ6h0K4HtS15UaVSBEaE+iln2LVpAuGc=
 github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=
+github.com/opencontainers/selinux v1.10.0 h1:rAiKF8hTcgLI3w0DHm6i0ylVVcOrlgR1kK99DRLDhyU=
+github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g=
@@ -744,8 +786,9 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
-github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
+github.com/prometheus/common v0.30.0 h1:JEkYlQnpzrzQFxi6gnukFPdQ+ac82oRhzMcIduJu/Ug=
+github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
 github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@@ -756,8 +799,9 @@ github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
+github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
@@ -882,39 +926,43 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.24.0/go.mod h1:O0cG0vP6TP3c323kh70JmeG1jN69Sn9Z5HxgmeASFWY=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0/go.mod h1:vEhqr0m4eTc+DWxfsXoXue2GBgV2uUwVznkGIHW/e5w=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4=
 go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo=
 go.opentelemetry.io/otel v1.0.0/go.mod h1:AjRVh9A5/5DE7S+mZtTR6t8vpKKryam+0lREnfmS4cg=
-go.opentelemetry.io/otel v1.0.1 h1:4XKyXmfqJLOQ7feyV5DB6gsBFZ0ltB8vLtp6pj4JIcc=
-go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU=
+go.opentelemetry.io/otel v1.3.0 h1:APxLf0eiBwLl+SOXiJJCVYzA1OOJNyAoV8C5RNRyy7Y=
+go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs=
 go.opentelemetry.io/otel/exporters/jaeger v1.0.0 h1:cLhx8llHw02h5JTqGqaRbYn+QVKHmrzD9vEbKnSPk5U=
 go.opentelemetry.io/otel/exporters/jaeger v1.0.0/go.mod h1:q10N1AolE1JjqKrFJK2tYw0iZpmX+HBaXBtuCzRnBGQ=
 go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.0.1/go.mod h1:Kv8liBeVNFkkkbilbgWRpV+wWuu+H5xdOT6HAgd30iw=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.0.1/go.mod h1:xOvWoTOrQjxjW61xtOmD/WKGRYb/P4NzRo3bs65U6Rk=
+go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0/go.mod h1:hO1KLR7jcKaDDKDkvI9dP/FIhpmna5lkqPUQdEjFAM8=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0/go.mod h1:keUU7UfnwWTWpJ+FWnyqmogPa82nuU5VUANFq49hlMY=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0/go.mod h1:QNX1aly8ehqqX1LEa6YniTU7VY9I6R3X/oPxhGdTceE=
 go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU=
 go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw=
 go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc=
 go.opentelemetry.io/otel/sdk v1.0.0/go.mod h1:PCrDHlSy5x1kjezSdL37PhbFUMjrsLRshJ2zCzeXwbM=
-go.opentelemetry.io/otel/sdk v1.0.1 h1:wXxFEWGo7XfXupPwVJvTBOaPBC9FEg0wB8hMNrKk+cA=
-go.opentelemetry.io/otel/sdk v1.0.1/go.mod h1:HrdXne+BiwsOHYYkBE5ysIcv2bvdZstxzmCQhxTcZkI=
+go.opentelemetry.io/otel/sdk v1.3.0 h1:3278edCoH89MEJ0Ky8WQXVmDQv3FX4ZJ3Pp+9fJreAI=
+go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs=
 go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE=
 go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE=
 go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw=
 go.opentelemetry.io/otel/trace v1.0.0/go.mod h1:PXTWqayeFUlJV1YDNhsJYB184+IvAH814St6o6ajzIs=
-go.opentelemetry.io/otel/trace v1.0.1 h1:StTeIH6Q3G4r0Fiw34LTokUFESZgIDUr0qIJ7mKmAfw=
-go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk=
+go.opentelemetry.io/otel/trace v1.3.0 h1:doy8Hzb1RJ+I3yFhtDmwNc7tIyw1tNMOIsyPzp1NOGY=
+go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
-go.opentelemetry.io/proto/otlp v0.9.0/go.mod h1:1vKfU9rv61e9EVGthD1zNvUbiwPcimSsOPU9brfSHJg=
+go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg3NAKnUGl9YpQ=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
+go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
@@ -934,6 +982,7 @@ golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -955,6 +1004,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
 golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
@@ -963,6 +1013,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -988,7 +1040,6 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191108221443-4ba9e2ef068c/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -1006,23 +1057,37 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210825183410-e898025ed96a h1:bRuuGXV8wwSdGTB+CtJf+FjgO1APK1CoO39T4BN/XBw=
+golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93 h1:alLDrZkL34Y2bnGHfvC1CYBRBXCXgx8AC2vY4MRtYX4=
+golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f h1:Qmd2pbz05z7z6lm0DrgQVVPuBm92jqujBKMHMOlOQEw=
+golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1066,7 +1131,6 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191105231009-c1f44814a5cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1096,6 +1160,7 @@ golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1106,8 +1171,14 @@ golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1116,22 +1187,31 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210820121016-41cdb8703e55/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 h1:2B5p2L5IfGiD7+b9BOoRMC6DgObAVZV+Fsp050NqXik=
+golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a h1:ppl5mZgokTT8uPkmYOyEUmPTr3ypaKkg5eFOGrAmxxE=
+golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1191,10 +1271,17 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
 golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
+golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1217,6 +1304,11 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
 google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
 google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
+google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
+google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
+google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
+google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1244,15 +1336,19 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa
 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
 google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
+google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
 google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
 google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
-google.golang.org/grpc v1.41.0 h1:f+PlOh7QV4iIJkPrx5NQ7qaNGFQ3OTse67yaDHfju4E=
-google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k=
+google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
+google.golang.org/grpc v1.43.0 h1:Eeu7bZtDZ2DpRCsLhUlcrLnvYaMK1Gz86a+hMVvELmM=
+google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -1271,8 +1367,9 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
@@ -1316,31 +1413,32 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
 k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo=
 k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ=
 k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8=
-k8s.io/api v0.22.0/go.mod h1:0AoXXqst47OI/L0oGKq9DG61dvGRPXs7X4/B7KyjBCU=
+k8s.io/api v0.22.5/go.mod h1:mEhXyLaSD1qTOf40rRiKXkc+2iCem09rWLlFwhCEiAs=
 k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
 k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
 k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc=
-k8s.io/apimachinery v0.22.0 h1:CqH/BdNAzZl+sr3tc0D3VsK3u6ARVSo3GWyLmfIjbP0=
-k8s.io/apimachinery v0.22.0/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0=
+k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0=
+k8s.io/apimachinery v0.22.5 h1:cIPwldOYm1Slq9VLBRPtEYpyhjIm1C6aAMAoENuvN9s=
+k8s.io/apimachinery v0.22.5/go.mod h1:xziclGKwuuJ2RM5/rSFQSYAj0zdbci3DH8kj+WvyN0U=
 k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU=
 k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM=
 k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q=
-k8s.io/apiserver v0.22.0/go.mod h1:04kaIEzIQrTGJ5syLppQWvpkLJXQtJECHmae+ZGc/nc=
+k8s.io/apiserver v0.22.5/go.mod h1:s2WbtgZAkTKt679sYtSudEQrTGWUSQAPe6MupLnlmaQ=
 k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y=
 k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k=
 k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0=
-k8s.io/client-go v0.22.0/go.mod h1:GUjIuXR5PiEv/RVK5OODUsm6eZk7wtSWZSaSJbpFdGg=
+k8s.io/client-go v0.22.5/go.mod h1:cs6yf/61q2T1SdQL5Rdcjg9J1ElXSwbjSrW2vFImM4Y=
 k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0=
 k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk=
 k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI=
 k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM=
-k8s.io/component-base v0.22.0/go.mod h1:SXj6Z+V6P6GsBhHZVbWCw9hFjUdUYnJerlhhPnYCBCg=
+k8s.io/component-base v0.22.5/go.mod h1:VK3I+TjuF9eaa+Ln67dKxhGar5ynVbwnGrUiNF4MqCI=
 k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM=
 k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=
 k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=
 k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc=
-k8s.io/cri-api v0.23.0-alpha.4 h1:VY9Bxk+254iz5rop9IWorEfVQIzcv8IkUVEwWcvODgM=
-k8s.io/cri-api v0.23.0-alpha.4/go.mod h1:qVxNSzR1gwLmZWK61jKRA5NhbyYrNoXUaZpQ7yOUYOQ=
+k8s.io/cri-api v0.23.1 h1:0DHL/hpTf4Fp+QkUXFefWcp1fhjXr9OlNdY9X99c+O8=
+k8s.io/cri-api v0.23.1/go.mod h1:REJE3PSU0h/LOV1APBrupxrEJqnoxZC8KWzkBUHwrK4=
 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
@@ -1348,12 +1446,14 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
+k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o=
 k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=
 k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
+k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
 k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20210707171843-4b05e18ac7d9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=

src/runtime/pkg/containerd-shim-v2/create.go

@@ -1,6 +1,7 @@
 // Copyright (c) 2014,2015,2016 Docker, Inc.
 // Copyright (c) 2017 Intel Corporation
 // Copyright (c) 2018 HyperHQ Inc.
+// Copyright (c) 2021 Adobe Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -57,10 +58,10 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con
 
 	detach := !r.Terminal
 	ociSpec, bundlePath, err := loadSpec(r)
+
 	if err != nil {
 		return nil, err
 	}
-
 	containerType, err := oci.ContainerType(*ociSpec)
 	if err != nil {
 		return nil, err
@@ -69,16 +70,18 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con
 	disableOutput := noNeedForOutput(detach, ociSpec.Process.Terminal)
 	rootfs := filepath.Join(r.Bundle, "rootfs")
 
+	runtimeConfig, err := loadRuntimeConfig(s, r, ociSpec.Annotations)
+	if err != nil {
+		return nil, err
+	}
+
 	switch containerType {
 	case vc.PodSandbox, vc.SingleContainer:
 		if s.sandbox != nil {
 			return nil, fmt.Errorf("cannot create another sandbox in sandbox: %s", s.sandbox.ID())
 		}
 
-		s.config, err = loadRuntimeConfig(s, r, ociSpec.Annotations)
-		if err != nil {
-			return nil, err
-		}
+		s.config = runtimeConfig
 
 		// create tracer
 		// This is the earliest location we can create the tracer because we must wait
@@ -176,7 +179,7 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con
 			}
 		}()
 
-		_, err = katautils.CreateContainer(ctx, s.sandbox, *ociSpec, rootFs, r.ID, bundlePath, "", disableOutput)
+		_, err = katautils.CreateContainer(ctx, s.sandbox, *ociSpec, rootFs, r.ID, bundlePath, "", disableOutput, runtimeConfig.DisableGuestEmptyDir)
 		if err != nil {
 			return nil, err
 		}
@@ -263,6 +266,10 @@ func checkAndMount(s *service, r *taskAPI.CreateTaskRequest) (bool, error) {
 		if katautils.IsBlockDevice(m.Source) && !s.config.HypervisorConfig.DisableBlockDeviceUse {
 			return false, nil
 		}
+		if m.Type == vc.NydusRootFSType {
+			// if kata + nydus, do not mount
+			return false, nil
+		}
 	}
 	rootfs := filepath.Join(r.Bundle, "rootfs")
 	if err := doMount(r.Rootfs, rootfs); err != nil {

src/runtime/pkg/containerd-shim-v2/create_test.go

@@ -50,7 +50,6 @@ func TestCreateSandboxSuccess(t *testing.T) {
 	}()
 
 	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
-	// defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
@@ -99,7 +98,6 @@ func TestCreateSandboxFail(t *testing.T) {
 	assert := assert.New(t)
 
 	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
-	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
@@ -137,7 +135,6 @@ func TestCreateSandboxConfigFail(t *testing.T) {
 	assert := assert.New(t)
 
 	tmpdir, bundlePath, _ := ktu.SetupOCIConfigFile(t)
-	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
@@ -187,7 +184,6 @@ func TestCreateContainerSuccess(t *testing.T) {
 	}
 
 	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
-	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
@@ -227,7 +223,6 @@ func TestCreateContainerFail(t *testing.T) {
 	assert := assert.New(t)
 
 	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
-	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)
@@ -278,7 +273,6 @@ func TestCreateContainerConfigFail(t *testing.T) {
 	}()
 
 	tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t)
-	defer os.RemoveAll(tmpdir)
 
 	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
 	assert.NoError(err)

src/runtime/pkg/containerd-shim-v2/delete_test.go

@@ -7,7 +7,6 @@
 package containerdshim
 
 import (
-	"os"
 	"testing"
 
 	taskAPI "github.com/containerd/containerd/runtime/v2/task"
@@ -25,8 +24,8 @@ func TestDeleteContainerSuccessAndFail(t *testing.T) {
 		MockID: testSandboxID,
 	}
 
-	rootPath, bundlePath, _ := ktu.SetupOCIConfigFile(t)
-	defer os.RemoveAll(rootPath)
+	_, bundlePath, _ := ktu.SetupOCIConfigFile(t)
+
 	_, err := compatoci.ParseConfigJSON(bundlePath)
 	assert.NoError(err)
 

src/runtime/pkg/containerd-shim-v2/event_forwarder.go

@@ -0,0 +1,88 @@
+// Copyright (c) 2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"context"
+	"os"
+	"time"
+
+	"github.com/containerd/containerd/events"
+)
+
+type forwarderType string
+
+const (
+	forwarderTypeLog        forwarderType = "log"
+	forwarderTypeContainerd forwarderType = "containerd"
+
+	// A time span used to wait for publish a containerd event,
+	// once it costs a longer time than timeOut, it will be canceld.
+	timeOut = 5 * time.Second
+
+	// ttrpc address passed from container runtime.
+	// For now containerd will pass the address, and CRI-O will not
+	ttrpcAddressEnv = "TTRPC_ADDRESS"
+)
+
+type eventsForwarder interface {
+	forward()
+	forwarderType() forwarderType
+}
+
+type logForwarder struct {
+	s *service
+}
+
+func (lf *logForwarder) forward() {
+	for e := range lf.s.events {
+		shimLog.WithField("topic", getTopic(e)).Infof("post event: %+v", e)
+	}
+}
+
+func (lf *logForwarder) forwarderType() forwarderType {
+	return forwarderTypeLog
+}
+
+type containerdForwarder struct {
+	s         *service
+	ctx       context.Context
+	publisher events.Publisher
+}
+
+func (cf *containerdForwarder) forward() {
+	for e := range cf.s.events {
+		ctx, cancel := context.WithTimeout(cf.ctx, timeOut)
+		err := cf.publisher.Publish(ctx, getTopic(e), e)
+		cancel()
+		if err != nil {
+			shimLog.WithError(err).Error("post event")
+		}
+	}
+}
+
+func (cf *containerdForwarder) forwarderType() forwarderType {
+	return forwarderTypeContainerd
+}
+
+func (s *service) newEventsForwarder(ctx context.Context, publisher events.Publisher) eventsForwarder {
+	var forwarder eventsForwarder
+	ttrpcAddress := os.Getenv(ttrpcAddressEnv)
+	if ttrpcAddress == "" {
+		// non containerd will use log forwarder to write events to log
+		forwarder = &logForwarder{
+			s: s,
+		}
+	} else {
+		forwarder = &containerdForwarder{
+			s:         s,
+			ctx:       ctx,
+			publisher: publisher,
+		}
+	}
+
+	return forwarder
+}

src/runtime/pkg/containerd-shim-v2/event_forwarder_test.go

@@ -0,0 +1,45 @@
+// Copyright (c) 2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/containerd/containerd/events"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/vcmock"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewEventsForwarder(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	s := &service{
+		id:         testSandboxID,
+		sandbox:    sandbox,
+		containers: make(map[string]*container),
+	}
+
+	// newEventsForwarder will not call publisher to publish events
+	// so here we can use a nil pointer to test newEventsForwarder
+	var publisher events.Publisher
+
+	// check log forwarder
+	forwarder := s.newEventsForwarder(context.Background(), publisher)
+	assert.Equal(forwarderTypeLog, forwarder.forwarderType())
+
+	// check containerd forwarder
+	os.Setenv(ttrpcAddressEnv, "/foo/bar.sock")
+	defer os.Setenv(ttrpcAddressEnv, "")
+	forwarder = s.newEventsForwarder(context.Background(), publisher)
+	assert.Equal(forwarderTypeContainerd, forwarder.forwarderType())
+}

src/runtime/pkg/containerd-shim-v2/service.go

@@ -17,7 +17,6 @@ import (
 	eventstypes "github.com/containerd/containerd/api/events"
 	"github.com/containerd/containerd/api/types/task"
 	"github.com/containerd/containerd/errdefs"
-	"github.com/containerd/containerd/events"
 	"github.com/containerd/containerd/namespaces"
 	cdruntime "github.com/containerd/containerd/runtime"
 	cdshim "github.com/containerd/containerd/runtime/v2/shim"
@@ -51,10 +50,6 @@ const (
 
 	chSize      = 128
 	exitCode255 = 255
-
-	// A time span used to wait for publish a containerd event,
-	// once it costs a longer time than timeOut, it will be canceld.
-	timeOut = 5 * time.Second
 )
 
 var (
@@ -100,7 +95,8 @@ func New(ctx context.Context, id string, publisher cdshim.Publisher, shutdown fu
 
 	go s.processExits()
 
-	go s.forward(ctx, publisher)
+	forwarder := s.newEventsForwarder(ctx, publisher)
+	go forwarder.forward()
 
 	return s, nil
 }
@@ -256,17 +252,6 @@ func (s *service) StartShim(ctx context.Context, opts cdshim.StartOpts) (_ strin
 	return address, nil
 }
 
-func (s *service) forward(ctx context.Context, publisher events.Publisher) {
-	for e := range s.events {
-		ctx, cancel := context.WithTimeout(ctx, timeOut)
-		err := publisher.Publish(ctx, getTopic(e), e)
-		cancel()
-		if err != nil {
-			shimLog.WithError(err).Error("post event")
-		}
-	}
-}
-
 func (s *service) send(evt interface{}) {
 	// for unit test, it will not initialize s.events
 	if s.events != nil {
@@ -791,6 +776,8 @@ func (s *service) Kill(ctx context.Context, r *taskAPI.KillRequest) (_ *ptypes.E
 			return empty, errors.New("The exec process does not exist")
 		}
 		processStatus = execs.status
+	} else {
+		r.All = true
 	}
 
 	// According to CRI specs, kubelet will call StopPodSandbox()

src/runtime/pkg/containerd-shim-v2/service_test.go

@@ -41,8 +41,7 @@ func TestServiceCreate(t *testing.T) {
 
 	assert := assert.New(t)
 
-	tmpdir, bundleDir, _ := ktu.SetupOCIConfigFile(t)
-	defer os.RemoveAll(tmpdir)
+	_, bundleDir, _ := ktu.SetupOCIConfigFile(t)
 
 	ctx := context.Background()
 

src/runtime/pkg/containerd-shim-v2/shim_management.go

@@ -7,26 +7,35 @@ package containerdshim
 
 import (
 	"context"
+	"encoding/json"
 	"expvar"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"net/http/pprof"
+	"net/url"
 	"path/filepath"
 	"strconv"
 	"strings"
 
+	"google.golang.org/grpc/codes"
+
 	cdshim "github.com/containerd/containerd/runtime/v2/shim"
+	mutils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/prometheus/client_golang/prometheus"
 	dto "github.com/prometheus/client_model/go"
 	"github.com/prometheus/common/expfmt"
+)
 
-	"google.golang.org/grpc/codes"
+const (
+	DirectVolumePathKey = "path"
 
-	mutils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
+	DirectVolumeStatUrl   = "/direct-volume/stats"
+	DirectVolumeResizeUrl = "/direct-volume/resize"
 )
 
 var (
@@ -34,6 +43,11 @@ var (
 	shimMgtLog               = shimLog.WithField("subsystem", "shim-management")
 )
 
+type ResizeRequest struct {
+	VolumePath string
+	Size       uint64
+}
+
 // agentURL returns URL for agent
 func (s *service) agentURL(w http.ResponseWriter, r *http.Request) {
 	url, err := s.sandbox.GetAgentURL()
@@ -126,6 +140,61 @@ func decodeAgentMetrics(body string) []*dto.MetricFamily {
 	return list
 }
 
+func (s *service) serveVolumeStats(w http.ResponseWriter, r *http.Request) {
+	val := r.URL.Query().Get(DirectVolumePathKey)
+	if val == "" {
+		msg := fmt.Sprintf("Required parameter %s not found", DirectVolumePathKey)
+		shimMgtLog.Info(msg)
+		w.WriteHeader(http.StatusBadRequest)
+		w.Write([]byte(msg))
+		return
+	}
+
+	volumePath, err := url.PathUnescape(val)
+	if err != nil {
+		shimMgtLog.WithError(err).Error("failed to unescape the volume stat url path")
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error()))
+		return
+	}
+
+	buf, err := s.sandbox.GuestVolumeStats(context.Background(), volumePath)
+	if err != nil {
+		shimMgtLog.WithError(err).WithField("volume-path", volumePath).Error("failed to get volume stats")
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error()))
+		return
+	}
+	w.Write(buf)
+}
+
+func (s *service) serveVolumeResize(w http.ResponseWriter, r *http.Request) {
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		shimMgtLog.WithError(err).Error("failed to read request body")
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error()))
+		return
+	}
+	var resizeReq ResizeRequest
+	err = json.Unmarshal(body, &resizeReq)
+	if err != nil {
+		shimMgtLog.WithError(err).Error("failed to unmarshal the http request body")
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error()))
+		return
+	}
+
+	err = s.sandbox.ResizeGuestVolume(context.Background(), resizeReq.VolumePath, resizeReq.Size)
+	if err != nil {
+		shimMgtLog.WithError(err).WithField("volume-path", resizeReq.VolumePath).Error("failed to resize the volume")
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error()))
+		return
+	}
+	w.Write([]byte(""))
+}
+
 func (s *service) startManagementServer(ctx context.Context, ociSpec *specs.Spec) {
 	// metrics socket will under sandbox's bundle path
 	metricsAddress := SocketAddress(s.id)
@@ -148,6 +217,8 @@ func (s *service) startManagementServer(ctx context.Context, ociSpec *specs.Spec
 	m := http.NewServeMux()
 	m.Handle("/metrics", http.HandlerFunc(s.serveMetrics))
 	m.Handle("/agent-url", http.HandlerFunc(s.agentURL))
+	m.Handle(DirectVolumeStatUrl, http.HandlerFunc(s.serveVolumeStats))
+	m.Handle(DirectVolumeResizeUrl, http.HandlerFunc(s.serveVolumeResize))
 	s.mountPprofHandle(m, ociSpec)
 
 	// register shim metrics

src/runtime/pkg/containerd-shim-v2/start.go

@@ -8,12 +8,14 @@ package containerdshim
 import (
 	"context"
 	"fmt"
+	"github.com/sirupsen/logrus"
 
 	"github.com/containerd/containerd/api/types/task"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
 )
 
 func startContainer(ctx context.Context, s *service, c *container) (retErr error) {
+	shimLog.WithField("container", c.id).Debug("start container")
 	defer func() {
 		if retErr != nil {
 			// notify the wait goroutine to continue
@@ -78,7 +80,8 @@ func startContainer(ctx context.Context, s *service, c *container) (retErr error
 			return err
 		}
 		c.ttyio = tty
-		go ioCopy(c.exitIOch, c.stdinCloser, tty, stdin, stdout, stderr)
+
+		go ioCopy(shimLog.WithField("container", c.id), c.exitIOch, c.stdinCloser, tty, stdin, stdout, stderr)
 	} else {
 		// close the io exit channel, since there is no io for this container,
 		// otherwise the following wait goroutine will hang on this channel.
@@ -94,6 +97,10 @@ func startContainer(ctx context.Context, s *service, c *container) (retErr error
 }
 
 func startExec(ctx context.Context, s *service, containerID, execID string) (e *exec, retErr error) {
+	shimLog.WithFields(logrus.Fields{
+		"container": containerID,
+		"exec":      execID,
+	}).Debug("start container execution")
 	// start an exec
 	c, err := s.getContainer(containerID)
 	if err != nil {
@@ -140,7 +147,10 @@ func startExec(ctx context.Context, s *service, containerID, execID string) (e *
 	}
 	execs.ttyio = tty
 
-	go ioCopy(execs.exitIOch, execs.stdinCloser, tty, stdin, stdout, stderr)
+	go ioCopy(shimLog.WithFields(logrus.Fields{
+		"container": c.id,
+		"exec":      execID,
+	}), execs.exitIOch, execs.stdinCloser, tty, stdin, stdout, stderr)
 
 	go wait(ctx, s, c, execID)
 

src/runtime/pkg/containerd-shim-v2/stream.go

@@ -12,6 +12,7 @@ import (
 	"syscall"
 
 	"github.com/containerd/fifo"
+	"github.com/sirupsen/logrus"
 )
 
 // The buffer size used to specify the buffer for IO streams copy
@@ -86,18 +87,20 @@ func newTtyIO(ctx context.Context, stdin, stdout, stderr string, console bool) (
 	return ttyIO, nil
 }
 
-func ioCopy(exitch, stdinCloser chan struct{}, tty *ttyIO, stdinPipe io.WriteCloser, stdoutPipe, stderrPipe io.Reader) {
+func ioCopy(shimLog *logrus.Entry, exitch, stdinCloser chan struct{}, tty *ttyIO, stdinPipe io.WriteCloser, stdoutPipe, stderrPipe io.Reader) {
 	var wg sync.WaitGroup
 
 	if tty.Stdin != nil {
 		wg.Add(1)
 		go func() {
+			shimLog.Debug("stdin io stream copy started")
 			p := bufPool.Get().(*[]byte)
 			defer bufPool.Put(p)
 			io.CopyBuffer(stdinPipe, tty.Stdin, *p)
 			// notify that we can close process's io safely.
 			close(stdinCloser)
 			wg.Done()
+			shimLog.Debug("stdin io stream copy exited")
 		}()
 	}
 
@@ -105,6 +108,7 @@ func ioCopy(exitch, stdinCloser chan struct{}, tty *ttyIO, stdinPipe io.WriteClo
 		wg.Add(1)
 
 		go func() {
+			shimLog.Debug("stdout io stream copy started")
 			p := bufPool.Get().(*[]byte)
 			defer bufPool.Put(p)
 			io.CopyBuffer(tty.Stdout, stdoutPipe, *p)
@@ -113,20 +117,24 @@ func ioCopy(exitch, stdinCloser chan struct{}, tty *ttyIO, stdinPipe io.WriteClo
 				// close stdin to make the other routine stop
 				tty.Stdin.Close()
 			}
+			shimLog.Debug("stdout io stream copy exited")
 		}()
 	}
 
 	if tty.Stderr != nil && stderrPipe != nil {
 		wg.Add(1)
 		go func() {
+			shimLog.Debug("stderr io stream copy started")
 			p := bufPool.Get().(*[]byte)
 			defer bufPool.Put(p)
 			io.CopyBuffer(tty.Stderr, stderrPipe, *p)
 			wg.Done()
+			shimLog.Debug("stderr io stream copy exited")
 		}()
 	}
 
 	wg.Wait()
 	tty.close()
 	close(exitch)
+	shimLog.Debug("all io stream copy goroutines exited")
 }

src/runtime/pkg/containerd-shim-v2/stream_test.go

@@ -7,6 +7,7 @@ package containerdshim
 
 import (
 	"context"
+	"github.com/sirupsen/logrus"
 	"io"
 	"os"
 	"path/filepath"
@@ -179,7 +180,7 @@ func TestIoCopy(t *testing.T) {
 		defer tty.close()
 
 		// start the ioCopy threads : copy from src to dst
-		go ioCopy(exitioch, stdinCloser, tty, dstInW, srcOutR, srcErrR)
+		go ioCopy(logrus.WithContext(context.Background()), exitioch, stdinCloser, tty, dstInW, srcOutR, srcErrR)
 
 		var firstW, secondW, thirdW io.WriteCloser
 		var firstR, secondR, thirdR io.Reader

src/runtime/pkg/containerd-shim-v2/wait.go

@@ -15,7 +15,6 @@ import (
 	"github.com/containerd/containerd/api/types/task"
 	"github.com/containerd/containerd/mount"
 	"github.com/sirupsen/logrus"
-	"google.golang.org/grpc/codes"
 
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
 )
@@ -31,12 +30,17 @@ func wait(ctx context.Context, s *service, c *container, execID string) (int32,
 	if execID == "" {
 		//wait until the io closed, then wait the container
 		<-c.exitIOch
+		shimLog.WithField("container", c.id).Debug("The container io streams closed")
 	} else {
 		execs, err = c.getExec(execID)
 		if err != nil {
 			return exitCode255, err
 		}
 		<-execs.exitIOch
+		shimLog.WithFields(logrus.Fields{
+			"container": c.id,
+			"exec":      execID,
+		}).Debug("The container process io streams closed")
 		//This wait could be triggered before exec start which
 		//will get the exec's id, thus this assignment must after
 		//the exec exit, to make sure it get the exec's id.
@@ -63,6 +67,7 @@ func wait(ctx context.Context, s *service, c *container, execID string) (int32,
 		if c.cType.IsSandbox() {
 			// cancel watcher
 			if s.monitor != nil {
+				shimLog.WithField("sandbox", s.sandbox.ID()).Info("cancel watcher")
 				s.monitor <- nil
 			}
 			if err = s.sandbox.Stop(ctx, true); err != nil {
@@ -73,7 +78,7 @@ func wait(ctx context.Context, s *service, c *container, execID string) (int32,
 				shimLog.WithField("sandbox", s.sandbox.ID()).Error("failed to delete sandbox")
 			}
 		} else {
-			if _, err = s.sandbox.StopContainer(ctx, c.id, false); err != nil {
+			if _, err = s.sandbox.StopContainer(ctx, c.id, true); err != nil {
 				shimLog.WithError(err).WithField("container", c.id).Warn("stop container failed")
 			}
 		}
@@ -82,13 +87,17 @@ func wait(ctx context.Context, s *service, c *container, execID string) (int32,
 		c.exitTime = timeStamp
 
 		c.exitCh <- uint32(ret)
-
+		shimLog.WithField("container", c.id).Debug("The container status is StatusStopped")
 	} else {
 		execs.status = task.StatusStopped
 		execs.exitCode = ret
 		execs.exitTime = timeStamp
 
 		execs.exitCh <- uint32(ret)
+		shimLog.WithFields(logrus.Fields{
+			"container": c.id,
+			"exec":      execID,
+		}).Debug("The container exec status is StatusStopped")
 	}
 	s.mu.Unlock()
 
@@ -102,6 +111,7 @@ func watchSandbox(ctx context.Context, s *service) {
 		return
 	}
 	err := <-s.monitor
+	shimLog.WithError(err).WithField("sandbox", s.sandbox.ID()).Info("watchSandbox gets an error or stop signal")
 	if err == nil {
 		return
 	}
@@ -147,13 +157,11 @@ func watchOOMEvents(ctx context.Context, s *service) {
 		default:
 			containerID, err := s.sandbox.GetOOMEvent(ctx)
 			if err != nil {
-				shimLog.WithError(err).Warn("failed to get OOM event from sandbox")
-				// If the GetOOMEvent call is not implemented, then the agent is most likely an older version,
-				// stop attempting to get OOM events.
-				// for rust agent, the response code is not found
-				if isGRPCErrorCode(codes.NotFound, err) || err.Error() == "Dead agent" {
+				if err.Error() == "ttrpc: closed" || err.Error() == "Dead agent" {
+					shimLog.WithError(err).Warn("agent has shutdown, return from watching of OOM events")
 					return
 				}
+				shimLog.WithError(err).Warn("failed to get OOM event from sandbox")
 				time.Sleep(defaultCheckInterval)
 				continue
 			}

src/runtime/pkg/direct-volume/utils.go

@@ -0,0 +1,127 @@
+// Copyright (c) 2022 Databricks Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package volume
+
+import (
+	b64 "encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+const (
+	mountInfoFileName = "mountInfo.json"
+
+	FSGroupMetadataKey             = "fsGroup"
+	FSGroupChangePolicyMetadataKey = "fsGroupChangePolicy"
+)
+
+// FSGroupChangePolicy holds policies that will be used for applying fsGroup to a volume.
+// This type and the allowed values are tracking the PodFSGroupChangePolicy defined in
+// https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/api/core/v1/types.go
+// It is up to the client using the direct-assigned volume feature (e.g. CSI drivers) to determine
+// the optimal setting for this change policy (i.e. from Pod spec or assuming volume ownership
+// based on the storage offering).
+type FSGroupChangePolicy string
+
+const (
+	// FSGroupChangeAlways indicates that volume's ownership should always be changed.
+	FSGroupChangeAlways FSGroupChangePolicy = "Always"
+	// FSGroupChangeOnRootMismatch indicates that volume's ownership will be changed
+	// only when ownership of root directory does not match with the desired group id.
+	FSGroupChangeOnRootMismatch FSGroupChangePolicy = "OnRootMismatch"
+)
+
+var kataDirectVolumeRootPath = "/run/kata-containers/shared/direct-volumes"
+
+// MountInfo contains the information needed by Kata to consume a host block device and mount it as a filesystem inside the guest VM.
+type MountInfo struct {
+	// The type of the volume (ie. block)
+	VolumeType string `json:"volume-type"`
+	// The device backing the volume.
+	Device string `json:"device"`
+	// The filesystem type to be mounted on the volume.
+	FsType string `json:"fstype"`
+	// Additional metadata to pass to the agent regarding this volume.
+	Metadata map[string]string `json:"metadata,omitempty"`
+	// Additional mount options.
+	Options []string `json:"options,omitempty"`
+}
+
+// Add writes the mount info of a direct volume into a filesystem path known to Kata Container.
+func Add(volumePath string, mountInfo string) error {
+	volumeDir := filepath.Join(kataDirectVolumeRootPath, b64.URLEncoding.EncodeToString([]byte(volumePath)))
+	stat, err := os.Stat(volumeDir)
+	if err != nil {
+		if !errors.Is(err, os.ErrNotExist) {
+			return err
+		}
+		if err := os.MkdirAll(volumeDir, 0700); err != nil {
+			return err
+		}
+	}
+	if stat != nil && !stat.IsDir() {
+		return fmt.Errorf("%s should be a directory", volumeDir)
+	}
+
+	var deserialized MountInfo
+	if err := json.Unmarshal([]byte(mountInfo), &deserialized); err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(filepath.Join(volumeDir, mountInfoFileName), []byte(mountInfo), 0600)
+}
+
+// Remove deletes the direct volume path including all the files inside it.
+func Remove(volumePath string) error {
+	return os.RemoveAll(filepath.Join(kataDirectVolumeRootPath, b64.URLEncoding.EncodeToString([]byte(volumePath))))
+}
+
+// VolumeMountInfo retrieves the mount info of a direct volume.
+func VolumeMountInfo(volumePath string) (*MountInfo, error) {
+	mountInfoFilePath := filepath.Join(kataDirectVolumeRootPath, b64.URLEncoding.EncodeToString([]byte(volumePath)), mountInfoFileName)
+	if _, err := os.Stat(mountInfoFilePath); err != nil {
+		return nil, err
+	}
+	buf, err := ioutil.ReadFile(mountInfoFilePath)
+	if err != nil {
+		return nil, err
+	}
+	var mountInfo MountInfo
+	if err := json.Unmarshal(buf, &mountInfo); err != nil {
+		return nil, err
+	}
+	return &mountInfo, nil
+}
+
+// RecordSandboxId associates a sandbox id with a direct volume.
+func RecordSandboxId(sandboxId string, volumePath string) error {
+	encodedPath := b64.URLEncoding.EncodeToString([]byte(volumePath))
+	mountInfoFilePath := filepath.Join(kataDirectVolumeRootPath, encodedPath, mountInfoFileName)
+	if _, err := os.Stat(mountInfoFilePath); err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(filepath.Join(kataDirectVolumeRootPath, encodedPath, sandboxId), []byte(""), 0600)
+}
+
+func GetSandboxIdForVolume(volumePath string) (string, error) {
+	files, err := ioutil.ReadDir(filepath.Join(kataDirectVolumeRootPath, b64.URLEncoding.EncodeToString([]byte(volumePath))))
+	if err != nil {
+		return "", err
+	}
+	// Find the id of the first sandbox.
+	// We expect a direct-assigned volume is associated with only a sandbox at a time.
+	for _, file := range files {
+		if file.Name() != mountInfoFileName {
+			return file.Name(), nil
+		}
+	}
+	return "", fmt.Errorf("no sandbox found for %s", volumePath)
+}