Merge pull request #3402 from snir911/2.3.1-branch-bump

# Kata Containers 2.3.1

.github/workflows/gather-artifacts.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-# Copyright (c) 2019 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-set -o errexit
-set -o pipefail
-
-pushd kata-artifacts >>/dev/null
-for c in ./*.tar.gz
-do
-    echo "untarring tarball $c"
-    tar -xvf $c
-done
-
-tar cvfJ ../kata-static.tar.xz ./opt
-popd >>/dev/null

.github/workflows/generate-artifact-tarball.sh

@@ -1,36 +0,0 @@
-#!/bin/bash
-# Copyright (c) 2019 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-set -o errexit
-set -o pipefail
-
-
-main() {
-    artifact_stage=${1:-}
-    artifact=$(echo  ${artifact_stage} | sed -n -e 's/^install_//p' | sed -r 's/_/-/g')
-    if [ -z "${artifact}" ]; then
-        "Scripts needs artifact name to build"
-        exit 1
-    fi
-
-    tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-    export GOPATH=$HOME/go
-
-    go get github.com/kata-containers/packaging || true
-    pushd $GOPATH/src/github.com/kata-containers/packaging/release >>/dev/null
-    git checkout $tag
-    pushd ../obs-packaging
-    ./gen_versions_txt.sh $tag
-    popd
-
-    source ./kata-deploy-binaries.sh
-    ${artifact_stage} $tag
-    popd
-
-    mv $HOME/go/src/github.com/kata-containers/packaging/release/kata-static-${artifact}.tar.gz .
-}
-
-main $@

.github/workflows/generate-local-artifact-tarball.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-# Copyright (c) 2019 Intel Corporation
-# Copyright (c) 2020 Ant Group
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-set -o errexit
-set -o pipefail
-
-
-main() {
-    artifact_stage=${1:-}
-    artifact=$(echo  ${artifact_stage} | sed -n -e 's/^install_//p' | sed -r 's/_/-/g')
-    if [ -z "${artifact}" ]; then
-        "Scripts needs artifact name to build"
-        exit 1
-    fi
-
-    tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-    pushd $GITHUB_WORKSPACE/tools/packaging
-    git checkout $tag
-    ./scripts/gen_versions_txt.sh $tag
-    popd
-
-    pushd $GITHUB_WORKSPACE/tools/packaging/release
-    source ./kata-deploy-binaries.sh
-    ${artifact_stage} $tag
-    popd
-
-    mv $GITHUB_WORKSPACE/tools/packaging/release/kata-static-${artifact}.tar.gz .
-}
-
-main $@

.github/workflows/kata-deploy-push.yaml

@@ -0,0 +1,68 @@
+name: kata deploy build
+
+on: [push, pull_request]
+
+jobs:
+  build-asset:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        asset:
+          - kernel
+          - kernel-experimental
+          - shim-v2
+          - qemu
+          - cloud-hypervisor
+          - firecracker
+          - rootfs-image
+          - rootfs-initrd
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install docker
+        run: |
+          curl -fsSL https://test.docker.com -o test-docker.sh
+          sh test-docker.sh
+
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r --preserve=all "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: build-asset
+    steps:
+      - uses: actions/checkout@v2
+      - name: get-artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: kata-artifacts
+          path: build
+      - name: merge-artifacts
+        run: |
+          make merge-builds
+      - name: store-artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: kata-static-tarball
+          path: kata-static.tar.xz
+
+  make-kata-tarball:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: make kata-tarball
+        run: |
+          make kata-tarball
+          sudo make install-tarball

.github/workflows/kata-deploy-test.yaml

@@ -5,58 +5,121 @@ on:
 name: test-kata-deploy
 
 jobs:
-  check_comments:
-    if: ${{ github.event.issue.pull_request }}
+  check-comment-and-membership:
     runs-on: ubuntu-latest
+    if: |
+      github.event.issue.pull_request
+      && github.event_name == 'issue_comment'
+      && github.event.action == 'created'
+      && startsWith(github.event.comment.body, '/test_kata_deploy')
     steps:
-      - name: Check for Command
-        id: command
-        uses: kata-containers/slash-command-action@v1
+      - name: Check membership
+        uses: kata-containers/is-organization-member@1.0.1
+        id: is_organization_member
         with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          command: "test_kata_deploy"
-          reaction: "true"
-          reaction-type: "eyes"
-          allow-edits: "false"
-          permission-level: admin
-      - name: verify command arg is kata-deploy
+          organization: kata-containers
+          username: ${{ github.event.comment.user.login }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Fail if not member
         run: |
-           echo "The command was '${{ steps.command.outputs.command-name }}' with arguments '${{ steps.command.outputs.command-arguments }}'"
+          result=${{ steps.is_organization_member.outputs.result }}
+          if [ $result == false ]; then
+              user=${{ github.event.comment.user.login }}
+              echo Either ${user} is not part of the kata-containers organization
+              echo or ${user} has its Organization Visibility set to Private at
+              echo https://github.com/orgs/kata-containers/people?query=${user}
+              echo 
+              echo Ensure you change your Organization Visibility to Public and
+              echo trigger the test again.
+              exit 1
+          fi
 
-  create-and-test-container:
-    needs: check_comments
+  build-asset:
     runs-on: ubuntu-latest
+    needs: check-comment-and-membership
+    strategy:
+      matrix:
+        asset:
+          - cloud-hypervisor
+          - firecracker
+          - kernel
+          - qemu
+          - rootfs-image
+          - rootfs-initrd
+          - shim-v2
     steps:
-      - name: get-PR-ref
-        id: get-PR-ref
+      - uses: actions/checkout@v2
+      - name: Install docker
         run: |
-            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
-            echo "reference for PR: " ${ref}
-            echo "##[set-output name=pr-ref;]${ref}"
+          curl -fsSL https://test.docker.com -o test-docker.sh
+          sh test-docker.sh
 
-      - name: check out
-        uses: actions/checkout@v2
-        with:
-           ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
-
-      - name: build-container-image
-        id: build-container-image
+      - name: Build ${{ matrix.asset }}
         run: |
-            PR_SHA=$(git log --format=format:%H -n1)
-            VERSION="2.0.0"
-            ARTIFACT_URL="https://github.com/kata-containers/kata-containers/releases/download/${VERSION}/kata-static-${VERSION}-x86_64.tar.xz"
-            wget "${ARTIFACT_URL}" -O tools/packaging/kata-deploy/kata-static.tar.xz
-            docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:${PR_SHA} ./tools/packaging/kata-deploy
-            docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
-            docker push katadocker/kata-deploy-ci:$PR_SHA
-            echo "##[set-output name=pr-sha;]${PR_SHA}"
-
-      - name: test-kata-deploy-ci-in-aks
-        uses: ./tools/packaging/kata-deploy/action
-        with:
-          packaging-sha: ${{ steps.build-container-image.outputs.pr-sha }}
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
         env:
-          PKG_SHA: ${{ steps.build-container-image.outputs.pr-sha }}
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: build-asset
+    steps:
+      - uses: actions/checkout@v2
+      - name: get-artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: kata-artifacts
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: kata-static-tarball
+          path: kata-static.tar.xz
+
+  kata-deploy:
+    needs: create-kata-tarball
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v2
+        with:
+          name: kata-static-tarball
+      - name: build-and-push-kata-deploy-ci
+        id: build-and-push-kata-deploy-ci
+        run: |
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          pushd $GITHUB_WORKSPACE
+          git checkout $tag
+          pkg_sha=$(git rev-parse HEAD)
+          popd
+          mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
+          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t quay.io/kata-containers/kata-deploy-ci:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
+          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
+          docker push quay.io/kata-containers/kata-deploy-ci:$pkg_sha
+          mkdir -p packaging/kata-deploy
+          ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
+          echo "::set-output name=PKG_SHA::${pkg_sha}"
+      - name: test-kata-deploy-ci-in-aks
+        uses: ./packaging/kata-deploy/action
+        with:
+          packaging-sha: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
+        env:
+          PKG_SHA: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
           AZ_APPID: ${{ secrets.AZ_APPID }}
           AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
           AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}

.github/workflows/main.yaml

@@ -1,293 +0,0 @@
-name: Publish release tarball
-on: 
-  push: 
-    tags:
-     - '1.*'
-
-jobs:
-  get-artifact-list:
-    runs-on: ubuntu-latest
-    steps:
-      - name: get the list
-        run: |
-         pushd $GITHUB_WORKSPACE
-         tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-         git checkout $tag
-         popd
-         $GITHUB_WORKSPACE/tools/packaging/artifact-list.sh > artifact-list.txt
-      - name: save-artifact-list
-        uses: actions/upload-artifact@master
-        with:
-          name: artifact-list
-          path: artifact-list.txt
-
-  build-kernel:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_kernel"
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifact-list
-        uses: actions/download-artifact@master
-        with:
-          name: artifact-list
-      - run: |
-         sudo apt-get update && sudo apt install -y flex bison libelf-dev bc iptables
-      - name: build-kernel
-        run: |
-         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@master
-        with:
-          name: kata-artifacts
-          path: kata-static-kernel.tar.gz
-
-  build-experimental-kernel:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_experimental_kernel"
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifact-list
-        uses: actions/download-artifact@master
-        with:
-          name: artifact-list
-      - run: |
-         sudo apt-get update && sudo apt install -y flex bison libelf-dev bc iptables
-      - name: build-experimental-kernel
-        run: |
-         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@master
-        with:
-          name: kata-artifacts
-          path: kata-static-experimental-kernel.tar.gz
-
-  build-qemu:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_qemu"
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifact-list
-        uses: actions/download-artifact@master
-        with:
-          name: artifact-list
-      - name: build-qemu
-        run: |
-         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@master
-        with:
-          name: kata-artifacts
-          path: kata-static-qemu.tar.gz
-
-  # Job for building the image
-  build-image:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_image"
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifact-list
-        uses: actions/download-artifact@master
-        with:
-          name: artifact-list
-      - name: build-image
-        run: |
-         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@master
-        with:
-          name: kata-artifacts
-          path: kata-static-image.tar.gz
-
-  # Job for building firecracker hypervisor
-  build-firecracker:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_firecracker"
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifact-list
-        uses: actions/download-artifact@master
-        with:
-          name: artifact-list
-      - name: build-firecracker
-        run: |
-         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@master
-        with:
-          name: kata-artifacts
-          path: kata-static-firecracker.tar.gz
-
-  # Job for building cloud-hypervisor
-  build-clh:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_clh"
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifact-list
-        uses: actions/download-artifact@master
-        with:
-          name: artifact-list
-      - name: build-clh
-        run: |
-         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@master
-        with:
-          name: kata-artifacts
-          path: kata-static-clh.tar.gz
-
-  # Job for building kata components
-  build-kata-components:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_kata_components"
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifact-list
-        uses: actions/download-artifact@master
-        with:
-          name: artifact-list
-      - name: build-kata-components
-        run: |
-         if grep -q $buildstr ./artifact-list/artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@master
-        with:
-          name: kata-artifacts
-          path: kata-static-kata-components.tar.gz
-
-  gather-artifacts:
-    runs-on: ubuntu-16.04
-    needs: [build-experimental-kernel, build-kernel, build-qemu, build-image, build-firecracker, build-kata-components, build-clh]
-    steps:
-      - uses: actions/checkout@v1
-      - name: get-artifacts
-        uses: actions/download-artifact@master
-        with:
-          name: kata-artifacts
-      - name: colate-artifacts
-        run: |
-          $GITHUB_WORKSPACE/.github/workflows/gather-artifacts.sh
-      - name: store-artifacts
-        uses: actions/upload-artifact@master
-        with:
-          name: release-candidate
-          path: kata-static.tar.xz
-
-  kata-deploy:
-    needs: gather-artifacts
-    runs-on: ubuntu-latest
-    steps:
-      - name: get-artifacts
-        uses: actions/download-artifact@master
-        with:
-          name: release-candidate
-      - name: build-and-push-kata-deploy-ci
-        id: build-and-push-kata-deploy-ci
-        run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          git clone https://github.com/kata-containers/packaging
-          pushd packaging
-          git checkout $tag
-          pkg_sha=$(git rev-parse HEAD)
-          popd
-          mv release-candidate/kata-static.tar.xz ./packaging/kata-deploy/kata-static.tar.xz
-          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:$pkg_sha ./packaging/kata-deploy
-          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
-          docker push katadocker/kata-deploy-ci:$pkg_sha
-          echo "::set-output name=PKG_SHA::${pkg_sha}"
-      - name: test-kata-deploy-ci-in-aks
-        uses: ./packaging/kata-deploy/action
-        with:
-          packaging-sha: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
-        env:
-          PKG_SHA: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
-          AZ_APPID: ${{ secrets.AZ_APPID }}
-          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
-          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
-          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
-      - name: push-tarball
-        run: |
-          # tag the container image we created and push to DockerHub
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          docker tag katadocker/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} katadocker/kata-deploy:${tag}
-          docker push katadocker/kata-deploy:${tag}
-
-  upload-static-tarball:
-    needs: kata-deploy
-    runs-on: ubuntu-latest
-    steps:
-      - name: download-artifacts
-        uses: actions/download-artifact@master
-        with:
-          name: release-candidate
-      - name: install hub
-        run: |
-          HUB_VER=$(curl -s "https://api.github.com/repos/github/hub/releases/latest" | jq -r .tag_name | sed 's/^v//')
-          wget -q -O- https://github.com/github/hub/releases/download/v$HUB_VER/hub-linux-amd64-$HUB_VER.tgz | \
-          tar xz --strip-components=2 --wildcards '*/bin/hub' && sudo mv hub /usr/local/bin/hub
-      - name: push static tarball to github
-        run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          tarball="kata-static-$tag-x86_64.tar.xz"
-          repo="https://github.com/kata-containers/runtime.git"
-          mv release-candidate/kata-static.tar.xz "release-candidate/${tarball}"
-          git clone "${repo}"
-          cd runtime
-          echo "uploading asset '${tarball}' to '${repo}' tag: ${tag}"
-          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "../release-candidate/${tarball}" "${tag}"

.github/workflows/release.yaml

@@ -5,213 +5,45 @@ on:
      - '2.*'
 
 jobs:
-  get-artifact-list:
+  build-asset:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        asset:
+          - cloud-hypervisor
+          - firecracker
+          - kernel
+          - qemu
+          - rootfs-image
+          - rootfs-initrd
+          - shim-v2
     steps:
       - uses: actions/checkout@v2
-      - name: get the list
+      - name: Install docker
         run: |
-         pushd $GITHUB_WORKSPACE
-         tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-         git checkout $tag
-         popd
-         $GITHUB_WORKSPACE/tools/packaging/artifact-list.sh > artifact-list.txt
-      - name: save-artifact-list
-        uses: actions/upload-artifact@v2
-        with:
-          name: artifact-list
-          path: artifact-list.txt
+          curl -fsSL https://test.docker.com -o test-docker.sh
+          sh test-docker.sh
 
-  build-kernel:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_kernel"
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifact-list
-        uses: actions/download-artifact@v2
-        with:
-          name: artifact-list
-      - run: |
-         sudo apt-get update && sudo apt install -y flex bison libelf-dev bc iptables
-      - name: build-kernel
+      - name: Build ${{ matrix.asset }}
         run: |
-         if grep -q $buildstr artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-local-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh --build="${KATA_ASSET}"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+
+      - name: store-artifact ${{ matrix.asset }}
         uses: actions/upload-artifact@v2
         with:
           name: kata-artifacts
-          path: kata-static-kernel.tar.gz
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          if-no-files-found: error
 
-  build-experimental-kernel:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_experimental_kernel"
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifact-list
-        uses: actions/download-artifact@v2
-        with:
-          name: artifact-list
-      - run: |
-         sudo apt-get update && sudo apt install -y flex bison libelf-dev bc iptables
-      - name: build-experimental-kernel
-        run: |
-         if grep -q $buildstr artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-local-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-static-experimental-kernel.tar.gz
-
-  build-qemu:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_qemu"
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifact-list
-        uses: actions/download-artifact@v2
-        with:
-          name: artifact-list
-      - name: build-qemu
-        run: |
-         if grep -q $buildstr artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-local-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-static-qemu.tar.gz
-
-  build-image:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_image"
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifact-list
-        uses: actions/download-artifact@v2
-        with:
-          name: artifact-list
-      - name: build-image
-        run: |
-         if grep -q $buildstr artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-local-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-static-image.tar.gz
-
-  build-firecracker:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_firecracker"
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifact-list
-        uses: actions/download-artifact@v2
-        with:
-          name: artifact-list
-      - name: build-firecracker
-        run: |
-         if grep -q $buildstr artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-local-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-static-firecracker.tar.gz
-
-
-  build-clh:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_clh"
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifact-list
-        uses: actions/download-artifact@v2
-        with:
-          name: artifact-list
-      - name: build-clh
-        run: |
-         if grep -q $buildstr artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-local-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-static-clh.tar.gz
-
-  build-kata-components:
-    runs-on: ubuntu-16.04
-    needs: get-artifact-list
-    env:
-      buildstr: "install_kata_components"
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifact-list
-        uses: actions/download-artifact@v2
-        with:
-          name: artifact-list
-      - name: build-kata-components
-        run: |
-         if grep -q $buildstr artifact-list.txt; then
-           $GITHUB_WORKSPACE/.github/workflows/generate-local-artifact-tarball.sh $buildstr
-           echo "artifact-built=true" >> $GITHUB_ENV
-         else
-           echo "artifact-built=false" >> $GITHUB_ENV
-         fi
-      - name: store-artifacts
-        if: ${{ env.artifact-built }} == 'true'
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-static-kata-components.tar.gz
-
-  gather-artifacts:
-    runs-on: ubuntu-16.04
-    needs: [build-experimental-kernel, build-kernel, build-qemu, build-image, build-firecracker, build-kata-components, build-clh]
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: build-asset
     steps:
       - uses: actions/checkout@v2
       - name: get-artifacts
@@ -219,24 +51,24 @@ jobs:
         with:
           name: kata-artifacts
           path: kata-artifacts
-      - name: colate-artifacts
+      - name: merge-artifacts
         run: |
-          $GITHUB_WORKSPACE/.github/workflows/gather-artifacts.sh
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
       - name: store-artifacts
         uses: actions/upload-artifact@v2
         with:
-          name: release-candidate
+          name: kata-static-tarball
           path: kata-static.tar.xz
 
   kata-deploy:
-    needs: gather-artifacts
+    needs: create-kata-tarball
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: get-artifacts
+      - name: get-kata-tarball
         uses: actions/download-artifact@v2
         with:
-          name: release-candidate
+          name: kata-static-tarball
       - name: build-and-push-kata-deploy-ci
         id: build-and-push-kata-deploy-ci
         run: |
@@ -246,9 +78,11 @@ jobs:
           pkg_sha=$(git rev-parse HEAD)
           popd
           mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
-          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
+          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:$pkg_sha -t quay.io/kata-containers/kata-deploy-ci:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
           docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
           docker push katadocker/kata-deploy-ci:$pkg_sha
+          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
+          docker push quay.io/kata-containers/kata-deploy-ci:$pkg_sha
           mkdir -p packaging/kata-deploy
           ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
           echo "::set-output name=PKG_SHA::${pkg_sha}"
@@ -266,8 +100,14 @@ jobs:
         run: |
           # tag the container image we created and push to DockerHub
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          docker tag katadocker/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} katadocker/kata-deploy:${tag}
-          docker push katadocker/kata-deploy:${tag}
+          tags=($tag)
+          tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
+          for tag in ${tags[@]}; do \
+            docker tag katadocker/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} katadocker/kata-deploy:${tag} && \
+            docker tag quay.io/kata-containers/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} quay.io/kata-containers/kata-deploy:${tag} && \
+            docker push katadocker/kata-deploy:${tag} && \
+            docker push quay.io/kata-containers/kata-deploy:${tag}; \
+          done
 
   upload-static-tarball:
     needs: kata-deploy
@@ -277,7 +117,7 @@ jobs:
       - name: download-artifacts
         uses: actions/download-artifact@v2
         with:
-          name: release-candidate
+          name: kata-static-tarball
       - name: install hub
         run: |
           HUB_VER=$(curl -s "https://api.github.com/repos/github/hub/releases/latest" | jq -r .tag_name | sed 's/^v//')
@@ -291,3 +131,49 @@ jobs:
           pushd $GITHUB_WORKSPACE
           echo "uploading asset '${tarball}' for tag: ${tag}"
           GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}"
+          popd
+
+  upload-cargo-vendored-tarball:
+    needs: upload-static-tarball
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: generate-and-upload-tarball
+        run: |
+          pushd $GITHUB_WORKSPACE/src/agent
+          cargo vendor >> .cargo/config
+          popd
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tarball="kata-containers-$tag-vendor.tar.gz"
+          pushd $GITHUB_WORKSPACE
+          tar -cvzf "${tarball}" src/agent/.cargo/config src/agent/vendor
+          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}" 
+          popd
+
+  upload-libseccomp-tarball:
+    needs: upload-cargo-vendored-tarball
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: download-and-upload-tarball
+        env:
+          GITHUB_TOKEN: ${{ secrets.GIT_UPLOAD_TOKEN }}
+          GOPATH: ${HOME}/go
+        run: |
+          pushd $GITHUB_WORKSPACE
+          ./ci/install_yq.sh
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          versions_yaml="versions.yaml"
+          version=$(${GOPATH}/bin/yq read ${versions_yaml} "externals.libseccomp.version")
+          repo_url=$(${GOPATH}/bin/yq read ${versions_yaml} "externals.libseccomp.url")
+          download_url="${repo_url}/releases/download/v${version}"
+          tarball="libseccomp-${version}.tar.gz"
+          asc="${tarball}.asc"
+          curl -sSLO "${download_url}/${tarball}"
+          curl -sSLO "${download_url}/${asc}"
+          # "-m" option should be empty to re-use the existing release title
+          # without opening a text editor.
+          # For the details, check https://hub.github.com/hub-release.1.html.
+          hub release edit -m "" -a "${tarball}" "${tag}"
+          hub release edit -m "" -a "${asc}" "${tag}"
+          popd

.github/workflows/require-pr-porting-labels.yaml

@@ -12,8 +12,7 @@ on:
       - reopened
       - labeled
       - unlabeled
-   pull_request:
-     branches:
+    branches:
       - main
 
 jobs:
@@ -32,8 +31,6 @@ jobs:
 
       - name: Checkout code to allow hub to communicate with the project
         uses: actions/checkout@v2
-        with:
-          token: ${{ secrets.KATA_GITHUB_ACTIONS_TOKEN }}
 
       - name: Install porting checker script
         run: |

.github/workflows/static-checks.yaml

@@ -2,17 +2,18 @@ on:
   pull_request:
     types:
       - opened
+      - edited
       - reopened
+      - synchronize
       - labeled
       - unlabeled
 
 name: Static checks
 jobs:
   test:
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
     strategy:
       matrix:
-        go-version: [1.15.x, 1.16.x]
+        go-version: [1.16.x, 1.17.x]
         os: [ubuntu-20.04]
     runs-on: ${{ matrix.os }}
     env:
@@ -24,55 +25,74 @@ jobs:
       target_branch: ${{ github.base_ref }}
     steps:
     - name: Install Go
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       uses: actions/setup-go@v2
       with:
         go-version: ${{ matrix.go-version }}
       env:
         GOPATH: ${{ runner.workspace }}/kata-containers
     - name: Setup GOPATH
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
         echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}"
         echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}"
         echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}"
         echo "TRAVIS: ${TRAVIS}"
     - name: Set env
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
         echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
         echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
     - name: Checkout code
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       uses: actions/checkout@v2
       with:
         fetch-depth: 0
         path: ./src/github.com/${{ github.repository }}
     - name: Setup travis references
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
         echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}"
         target_branch=${TRAVIS_BRANCH}
     - name: Setup
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
         cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
       env:
         GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Building rust
+    - name: Installing rust
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
         cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh
         PATH=$PATH:"$HOME/.cargo/bin"
         rustup target add x86_64-unknown-linux-musl
         rustup component add rustfmt clippy
+    - name: Setup seccomp
+      run: |
+        libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
+        gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
+        echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
+        echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
+        echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
     # Check whether the vendored code is up-to-date & working as the first thing
     - name: Check vendored code
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
         cd ${GOPATH}/src/github.com/${{ github.repository }} && make vendor
-    # Must build before static checks as we depend on some generated code in runtime and agent
-    - name: Build
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make
     - name: Static Checks
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/static-checks.sh
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && make static-checks
     - name: Run Compiler Checks
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
         cd ${GOPATH}/src/github.com/${{ github.repository }} && make check
     - name: Run Unit Tests
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
         cd ${GOPATH}/src/github.com/${{ github.repository }} && make test
+    - name: Run Unit Tests As Root User
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && sudo -E PATH="$PATH" make test

Glossary.md

@@ -0,0 +1,94 @@
+# Glossary
+
+[A](#a), [B](#b), [C](#c), [D](#d), [E](#e), [F](#f), [G](#g), [H](#h), [I](#i), [J](#j), [K](#k), [L](#l), [M](#m), [N](#n), [O](#o), [P](#p), [Q](#q), [R](#r), [S](#s), [T](#t), [U](#u), [V](#v), [W](#w), [X](#x), [Y](#y), [Z](#z)
+
+## A
+
+### Auto Scaling
+a method used in cloud computing, whereby the amount of computational resources in a server farm, typically measured in terms of the number of active servers, which vary automatically based on the load on the farm.
+
+## B
+
+## C
+
+### Container Security Solutions
+The process of implementing security tools and policies that will give you the assurance that everything in your container is running as intended, and only as intended.
+
+### Container Software
+A standard unit of software that packages up code and all its dependencies so the application runs quickly and reliably from one computing environment to another.
+
+### Container Runtime Interface
+A plugin interface which enables Kubelet to use a wide variety of container runtimes, without the need to recompile.
+
+### Container Virtualization
+A container is a virtual runtime environment that runs on top of a single operating system (OS) kernel and emulates an operating system rather than the underlying hardware.
+
+## D
+
+## E
+
+## F
+
+## G
+
+## H
+
+## I
+
+### Infrastructure Architecture
+A structured and modern approach for supporting an organization and facilitating innovation within an enterprise.
+
+## J
+
+## K
+
+### Kata Containers
+Kata containers is an open source project delivering increased container security and Workload isolation through an implementation of lightweight virtual machines.
+
+## L
+
+## M
+
+## N
+
+## O
+
+## P
+
+### Pod Containers
+A Group of one or more containers , with shared storage/network, and a specification for how to run the containers.
+
+### Private Cloud
+A computing model that offers a proprietary environment dedicated to a single business entity.
+
+### Public Cloud
+Computing services offered by third-party providers over the public Internet, making them available to anyone who wants to use or purchase them.
+
+## Q
+
+## R
+
+## S
+
+### Serverless Containers
+An architecture in which code is executed on-demand. Serverless workloads are typically in the cloud, but on-premises serverless platforms exist, too.
+
+## T
+
+## U
+
+## V
+
+### Virtual Machine Monitor
+Computer software, firmware or hardware that creates and runs virtual machines.
+
+### Virtual Machine Software
+A software program or operating system that not only exhibits the behavior of a separate computer, but is also capable of performing tasks such as running applications and programs like a separate computer.
+
+## W
+
+## X
+
+## Y
+
+## Z

Makefile

@@ -18,6 +18,7 @@ TOOLS += agent-ctl
 STANDARD_TARGETS = build check clean install test vendor
 
 include utils.mk
+include ./tools/packaging/kata-deploy/local-build/Makefile
 
 all: build
 
@@ -29,4 +30,8 @@ $(eval $(call create_all_rules,$(COMPONENTS),$(TOOLS),$(STANDARD_TARGETS)))
 generate-protocols:
 	make -C src/agent generate-protocols
 
-.PHONY: all default
+# Some static checks rely on generated source files of components.
+static-checks: build
+	bash ci/static-checks.sh
+
+.PHONY: all default static-checks binary-tarball install-binary-tarball

README.md

@@ -2,22 +2,6 @@
 
 # Kata Containers
 
-* [Kata Containers](#kata-containers)
-    * [Introduction](#introduction)
-    * [Getting started](#getting-started)
-    * [Documentation](#documentation)
-    * [Community](#community)
-    * [Getting help](#getting-help)
-        * [Raising issues](#raising-issues)
-            * [Kata Containers 1.x versions](#kata-containers-1x-versions)
-    * [Developers](#developers)
-        * [Components](#components)
-            * [Kata Containers 1.x components](#kata-containers-1x-components)
-        * [Common repositories](#common-repositories)
-        * [Packaging and releases](#packaging-and-releases)
-
----
-
 Welcome to Kata Containers!
 
 This repository is the home of the Kata Containers code for the 2.0 and newer
@@ -26,11 +10,6 @@ releases.
 If you want to learn about Kata Containers, visit the main
 [Kata Containers website](https://katacontainers.io).
 
-For further details on the older (first generation) Kata Containers 1.x
-versions, see the
-[Kata Containers 1.x components](#kata-containers-1x-components)
-section.
-
 ## Introduction
 
 Kata Containers is an open source project and community working to build a
@@ -67,69 +46,34 @@ Please raise an issue
 > **Note:**
 > If you are reporting a security issue, please follow the [vulnerability reporting process](https://github.com/kata-containers/community#vulnerability-handling)
 
-#### Kata Containers 1.x versions
-
-For older Kata Containers 1.x releases, please raise an issue in the
-[Kata Containers 1.x component repository](#kata-containers-1x-components)
-that seems most appropriate.
-
-If in doubt, raise an issue
-[in the Kata Containers 1.x runtime repository](https://github.com/kata-containers/runtime/issues).
-
 ## Developers
 
 ### Components
 
+### Main components
+
+The table below lists the core parts of the project:
+
 | Component | Type | Description |
 |-|-|-|
-| [agent-ctl](tools/agent-ctl) | utility | Tool that provides low-level access for testing the agent. |
+| [runtime](src/runtime) | core | Main component run by a container manager and providing a containerd shimv2 runtime implementation. |
 | [agent](src/agent) | core | Management process running inside the virtual machine / POD that sets up the container environment. |
 | [documentation](docs) | documentation | Documentation common to all components (such as design and install documentation). |
-| [osbuilder](tools/osbuilder) | infrastructure | Tool to create "mini O/S" rootfs and initrd images for the hypervisor. |
-| [packaging](tools/packaging) | infrastructure | Scripts and metadata for producing packaged binaries<br/>(components, hypervisors, kernel and rootfs). |
-| [runtime](src/runtime) | core | Main component run by a container manager and providing a containerd shimv2 runtime implementation. |
-| [trace-forwarder](src/trace-forwarder) | utility | Agent tracing helper. |
+| [tests](https://github.com/kata-containers/tests) | tests | Excludes unit tests which live with the main code. |
 
-#### Kata Containers 1.x components
+### Additional components
 
-For the first generation of Kata Containers (1.x versions), each component was
-kept in a separate repository.
-
-For information on the Kata Containers 1.x releases, see the
-[Kata Containers 1.x releases page](https://github.com/kata-containers/runtime/releases).
-
-For further information on particular Kata Containers 1.x components, see the
-individual component repositories:
+The table below lists the remaining parts of the project:
 
 | Component | Type | Description |
 |-|-|-|
-| [agent](https://github.com/kata-containers/agent) | core | See [components](#components). |
-| [documentation](https://github.com/kata-containers/documentation) | documentation | |
-| [KSM throttler](https://github.com/kata-containers/ksm-throttler) | optional core | Daemon that monitors containers and deduplicates memory to maximize container density on the host. |
-| [osbuilder](https://github.com/kata-containers/osbuilder) | infrastructure | See [components](#components). |
-| [packaging](https://github.com/kata-containers/packaging) | infrastructure | See [components](#components). |
-| [proxy](https://github.com/kata-containers/proxy) | core | Multiplexes communications between the shims, agent and runtime. |
-| [runtime](https://github.com/kata-containers/runtime) | core | See [components](#components). |
-| [shim](https://github.com/kata-containers/shim) | core | Handles standard I/O and signals on behalf of the container process. |
-
-> **Note:**
->
-> - There are more components for the original Kata Containers 1.x implementation.
-> - The current implementation simplifies the design significantly:
->   compare the [current](docs/design/architecture.md) and
->   [previous generation](https://github.com/kata-containers/documentation/blob/master/design/architecture.md)
->   designs.
-
-### Common repositories
-
-The following repositories are used by both the current and first generation Kata Containers implementations:
-
-| Component | Description | Current | First generation | Notes |
-|-|-|-|-|-|
-| CI | Continuous Integration configuration files and scripts. | [Kata 2.x](https://github.com/kata-containers/ci/tree/main) | [Kata 1.x](https://github.com/kata-containers/ci/tree/master) | |
-| kernel | The Linux kernel used by the hypervisor to boot the guest image. | [Kata 2.x][kernel] | [Kata 1.x][kernel] | Patches are stored in the packaging component. |
-| tests | Test code. | [Kata 2.x](https://github.com/kata-containers/tests/tree/main) | [Kata 1.x](https://github.com/kata-containers/tests/tree/master) | Excludes unit tests which live with the main code. |
-| www.katacontainers.io | Contains the source for the [main web site](https://www.katacontainers.io). | [Kata 2.x][github-katacontainers.io] | [Kata 1.x][github-katacontainers.io] | | |
+| [packaging](tools/packaging) | infrastructure | Scripts and metadata for producing packaged binaries<br/>(components, hypervisors, kernel and rootfs). |
+| [kernel](https://www.kernel.org) | kernel | Linux kernel used by the hypervisor to boot the guest image. Patches are stored [here](tools/packaging/kernel). |
+| [osbuilder](tools/osbuilder) | infrastructure | Tool to create "mini O/S" rootfs and initrd images and kernel for the hypervisor. |
+| [`agent-ctl`](tools/agent-ctl) | utility | Tool that provides low-level access for testing the agent. |
+| [`trace-forwarder`](src/trace-forwarder) | utility | Agent tracing helper. |
+| [`ci`](https://github.com/kata-containers/ci) | CI | Continuous Integration configuration files and scripts. |
+| [`katacontainers.io`](https://github.com/kata-containers/www.katacontainers.io) | Source for the [`katacontainers.io`](https://www.katacontainers.io) site. |
 
 ### Packaging and releases
 
@@ -138,6 +82,9 @@ Kata Containers is now
 However, packaging scripts and metadata are still used to generate snap and GitHub releases. See
 the [components](#components) section for further details.
 
+## Glossary of Terms
+
+See the [glossary of terms](Glossary.md) related to Kata Containers.
 ---
 
 [kernel]: https://www.kernel.org

VERSION

@@ -1 +1 @@
-2.2.0-alpha1
+2.3.1

ci/go-no-os-exit.sh

@@ -1,30 +0,0 @@
-#!/bin/bash
-# Copyright (c) 2018 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-# Check there are no os.Exit() calls creeping into the code
-# We don't use that exit path in the Kata codebase.
-
-# Allow the path to check to be over-ridden.
-# Default to the current directory.
-go_packages=${1:-.}
-
-echo "Checking for no os.Exit() calls for package [${go_packages}]"
-
-candidates=`go list -f '{{.Dir}}/*.go' $go_packages`
-for f in $candidates; do
-	filename=`basename $f`
-	# skip all go test files
-	[[ $filename == *_test.go ]] && continue
-	# skip exit.go where, the only file we should call os.Exit() from.
-	[[ $filename == "exit.go" ]] && continue
-	files="$f $files"
-done
-
-[ -z "$files" ] && echo "No files to check, skipping" && exit 0
-
-if egrep -n '\<os\.Exit\>' $files; then
-	echo "Direct calls to os.Exit() are forbidden, please use exit() so atexit() works"
-	exit 1
-fi

ci/install_libseccomp.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+#
+# Copyright 2021 Sony Group Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -o errexit
+
+cidir=$(dirname "$0")
+source "${cidir}/lib.sh"
+
+clone_tests_repo
+
+source "${tests_repo_dir}/.ci/lib.sh"
+
+# The following variables if set on the environment will change the behavior
+# of gperf and libseccomp configure scripts, that may lead this script to
+# fail. So let's ensure they are unset here.
+unset PREFIX DESTDIR
+
+arch=$(uname -m)
+workdir="$(mktemp -d --tmpdir build-libseccomp.XXXXX)"
+
+# Variables for libseccomp
+# Currently, specify the libseccomp version directly without using `versions.yaml`
+# because the current Snap workflow is incomplete.
+# After solving the issue, replace this code by using the `versions.yaml`.
+# libseccomp_version=$(get_version "externals.libseccomp.version")
+# libseccomp_url=$(get_version "externals.libseccomp.url")
+libseccomp_version="2.5.1"
+libseccomp_url="https://github.com/seccomp/libseccomp"
+libseccomp_tarball="libseccomp-${libseccomp_version}.tar.gz"
+libseccomp_tarball_url="${libseccomp_url}/releases/download/v${libseccomp_version}/${libseccomp_tarball}"
+cflags="-O2"
+
+# Variables for gperf
+# Currently, specify the gperf version directly without using `versions.yaml`
+# because the current Snap workflow is incomplete.
+# After solving the issue, replace this code by using the `versions.yaml`.
+# gperf_version=$(get_version "externals.gperf.version")
+# gperf_url=$(get_version "externals.gperf.url")
+gperf_version="3.1"
+gperf_url="https://ftp.gnu.org/gnu/gperf"
+gperf_tarball="gperf-${gperf_version}.tar.gz"
+gperf_tarball_url="${gperf_url}/${gperf_tarball}"
+
+# We need to build the libseccomp library from sources to create a static library for the musl libc.
+# However, ppc64le and s390x have no musl targets in Rust. Hence, we do not set cflags for the musl libc.
+if ([ "${arch}" != "ppc64le" ] && [ "${arch}" != "s390x" ]); then
+    # Set FORTIFY_SOURCE=1 because the musl-libc does not have some functions about FORTIFY_SOURCE=2
+    cflags="-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=1 -O2"
+fi
+
+die() {
+    msg="$*"
+    echo "[Error] ${msg}" >&2
+    exit 1
+}
+
+finish() {
+    rm -rf "${workdir}"
+}
+
+trap finish EXIT
+
+build_and_install_gperf() {
+    echo "Build and install gperf version ${gperf_version}"
+    mkdir -p "${gperf_install_dir}"
+    curl -sLO "${gperf_tarball_url}"
+    tar -xf "${gperf_tarball}"
+    pushd "gperf-${gperf_version}"
+    ./configure --prefix="${gperf_install_dir}"
+    make
+    make install
+    export PATH=$PATH:"${gperf_install_dir}"/bin
+    popd
+    echo "Gperf installed successfully"
+}
+
+build_and_install_libseccomp() {
+    echo "Build and install libseccomp version ${libseccomp_version}"
+    mkdir -p "${libseccomp_install_dir}"
+    curl -sLO "${libseccomp_tarball_url}"
+    tar -xf "${libseccomp_tarball}"
+    pushd "libseccomp-${libseccomp_version}"
+    ./configure --prefix="${libseccomp_install_dir}" CFLAGS="${cflags}" --enable-static
+    make
+    make install
+    popd
+    echo "Libseccomp installed successfully"
+}
+
+main() {
+    local libseccomp_install_dir="${1:-}"
+    local gperf_install_dir="${2:-}"
+
+    if [ -z "${libseccomp_install_dir}" ] || [ -z "${gperf_install_dir}" ]; then
+        die "Usage: ${0} <libseccomp-install-dir> <gperf-install-dir>"
+    fi
+
+    pushd "$workdir"
+    # gperf is required for building the libseccomp.
+    build_and_install_gperf
+    build_and_install_libseccomp
+    popd
+}
+
+main "$@"

ci/install_rust.sh

@@ -12,5 +12,5 @@ source "${cidir}/lib.sh"
 clone_tests_repo
 
 pushd ${tests_repo_dir}
-.ci/install_rust.sh
+.ci/install_rust.sh ${1:-}
 popd

ci/install_yq.sh

@@ -15,12 +15,18 @@ die() {
 # Install the yq yaml query package from the mikefarah github repo
 # Install via binary download, as we may not have golang installed at this point
 function install_yq() {
-	GOPATH=${GOPATH:-${HOME}/go}
-	local yq_path="${GOPATH}/bin/yq"
 	local yq_pkg="github.com/mikefarah/yq"
 	local yq_version=3.4.1
+	INSTALL_IN_GOPATH=${INSTALL_IN_GOPATH:-true}
 
-	[ -x  "${GOPATH}/bin/yq" ] && [ "`${GOPATH}/bin/yq --version`"X == "yq version ${yq_version}"X ] && return
+	if [ "${INSTALL_IN_GOPATH}"  == "true" ];then
+		GOPATH=${GOPATH:-${HOME}/go}
+		mkdir -p "${GOPATH}/bin"
+		local yq_path="${GOPATH}/bin/yq"
+	else
+		yq_path="/usr/local/bin/yq"
+	fi
+	[ -x  "${yq_path}" ] && [ "`${yq_path} --version`"X == "yq version ${yq_version}"X ] && return
 
 	read -r -a sysInfo <<< "$(uname -sm)"
 
@@ -51,7 +57,6 @@ function install_yq() {
 		;;
 	esac
 
-	mkdir -p "${GOPATH}/bin"
 
 	# Check curl
 	if ! command -v "curl" >/dev/null; then

ci/lib.sh

@@ -3,6 +3,8 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+set -o nounset
+
 export tests_repo="${tests_repo:-github.com/kata-containers/tests}"
 export tests_repo_dir="$GOPATH/src/$tests_repo"
 export branch="${target_branch:-main}"
@@ -15,7 +17,7 @@ export branch="${target_branch:-main}"
 clone_tests_repo()
 {
 	if [ -d "$tests_repo_dir" ]; then
-		[ -n "$CI" ] && return
+		[ -n "${CI:-}" ] && return
 		pushd "${tests_repo_dir}"
 		git checkout "${branch}"
 		git pull

ci/openshift-ci/images/Dockerfile.buildroot

@@ -4,6 +4,11 @@
 #
 # This is the build root image for Kata Containers on OpenShift CI.
 #
-FROM centos:8
+FROM registry.centos.org/centos:8
 
-RUN yum -y update && yum -y install git sudo wget
+RUN yum -y update && \
+    yum -y install \
+    git \
+    sudo \
+    wget && \
+    yum clean all

ci/run.sh

@@ -8,9 +8,14 @@
 set -e
 cidir=$(dirname "$0")
 source "${cidir}/lib.sh"
+export CI_JOB="${CI_JOB:-}"
 
 clone_tests_repo
 
 pushd ${tests_repo_dir}
 .ci/run.sh
+# temporary fix, see https://github.com/kata-containers/tests/issues/3878
+if [ "$(uname -m)" != "s390x" ] && [ "$CI_JOB" == "CRI_CONTAINERD_K8S_MINIMAL" ]; then
+	tracing/test-agent-shutdown.sh
+fi
 popd

docs/Developer-Guide.md

@@ -1,56 +1,3 @@
-- [Warning](#warning)
-- [Assumptions](#assumptions)
-- [Initial setup](#initial-setup)
-- [Requirements to build individual components](#requirements-to-build-individual-components)
-- [Build and install the Kata Containers runtime](#build-and-install-the-kata-containers-runtime)
-- [Check hardware requirements](#check-hardware-requirements)
-  - [Configure to use initrd or rootfs image](#configure-to-use-initrd-or-rootfs-image)
-  - [Enable full debug](#enable-full-debug)
-    - [debug logs and shimv2](#debug-logs-and-shimv2)
-      - [Enabling full `containerd` debug](#enabling-full-containerd-debug)
-      - [Enabling just `containerd shim` debug](#enabling-just-containerd-shim-debug)
-      - [Enabling `CRI-O` and `shimv2` debug](#enabling-cri-o-and-shimv2-debug)
-    - [journald rate limiting](#journald-rate-limiting)
-      - [`systemd-journald` suppressing messages](#systemd-journald-suppressing-messages)
-      - [Disabling `systemd-journald` rate limiting](#disabling-systemd-journald-rate-limiting)
-- [Create and install rootfs and initrd image](#create-and-install-rootfs-and-initrd-image)
-  - [Build a custom Kata agent - OPTIONAL](#build-a-custom-kata-agent---optional)
-  - [Get the osbuilder](#get-the-osbuilder)
-  - [Create a rootfs image](#create-a-rootfs-image)
-    - [Create a local rootfs](#create-a-local-rootfs)
-    - [Add a custom agent to the image - OPTIONAL](#add-a-custom-agent-to-the-image---optional)
-    - [Build a rootfs image](#build-a-rootfs-image)
-    - [Install the rootfs image](#install-the-rootfs-image)
-  - [Create an initrd image - OPTIONAL](#create-an-initrd-image---optional)
-    - [Create a local rootfs for initrd image](#create-a-local-rootfs-for-initrd-image)
-    - [Build an initrd image](#build-an-initrd-image)
-    - [Install the initrd image](#install-the-initrd-image)
-- [Install guest kernel images](#install-guest-kernel-images)
-- [Install a hypervisor](#install-a-hypervisor)
-  - [Build a custom QEMU](#build-a-custom-qemu)
-    - [Build a custom QEMU for aarch64/arm64 - REQUIRED](#build-a-custom-qemu-for-aarch64arm64---required)
-- [Run Kata Containers with Containerd](#run-kata-containers-with-containerd)
-- [Run Kata Containers with Kubernetes](#run-kata-containers-with-kubernetes)
-- [Troubleshoot Kata Containers](#troubleshoot-kata-containers)
-- [Appendices](#appendices)
-  - [Checking Docker default runtime](#checking-docker-default-runtime)
-  - [Set up a debug console](#set-up-a-debug-console)
-    - [Simple debug console setup](#simple-debug-console-setup)
-      - [Enable agent debug console](#enable-agent-debug-console)
-      - [Start `kata-monitor` - ONLY NEEDED FOR 2.0.x](#start-kata-monitor---only-needed-for-20x)
-      - [Connect to debug console](#connect-to-debug-console)
-    - [Traditional debug console setup](#traditional-debug-console-setup)
-      - [Create a custom image containing a shell](#create-a-custom-image-containing-a-shell)
-      - [Build the debug image](#build-the-debug-image)
-      - [Configure runtime for custom debug image](#configure-runtime-for-custom-debug-image)
-      - [Create a container](#create-a-container)
-      - [Connect to the virtual machine using the debug console](#connect-to-the-virtual-machine-using-the-debug-console)
-        - [Enabling debug console for QEMU](#enabling-debug-console-for-qemu)
-        - [Enabling debug console for cloud-hypervisor / firecracker](#enabling-debug-console-for-cloud-hypervisor--firecracker)
-        - [Connecting to the debug console](#connecting-to-the-debug-console)
-  - [Obtain details of the image](#obtain-details-of-the-image)
-  - [Capturing kernel boot logs](#capturing-kernel-boot-logs)
-
 # Warning
 
 This document is written **specifically for developers**: it is not intended for end users.
@@ -139,6 +86,16 @@ One of the `initrd` and `image` options in Kata runtime config file **MUST** be
 The main difference between the options is that the size of `initrd`(10MB+) is significantly smaller than
 rootfs `image`(100MB+).
 
+## Enable seccomp
+
+Enable seccomp as follows:
+
+```
+$ sudo sed -i '/^disable_guest_seccomp/ s/true/false/' /etc/kata-containers/configuration.toml
+```
+
+This will pass container seccomp profiles to the kata agent.
+
 ## Enable full debug
 
 Enable full debug as follows:
@@ -269,6 +226,18 @@ $ go get -d -u github.com/kata-containers/kata-containers
 $ cd $GOPATH/src/github.com/kata-containers/kata-containers/src/agent && make
 ```
 
+The agent is built with seccomp capability by default.
+If you want to build the agent without the seccomp capability, you need to run `make` with `SECCOMP=no` as follows.
+
+```
+$ make -C $GOPATH/src/github.com/kata-containers/kata-containers/src/agent SECCOMP=no
+```
+
+> **Note:**
+>
+> - If you enable seccomp in the main configuration file but build the agent without seccomp capability,
+>   the runtime exits conservatively with an error message.
+
 ## Get the osbuilder
 
 ```
@@ -287,9 +256,21 @@ the following example.
 $ export ROOTFS_DIR=${GOPATH}/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder/rootfs
 $ sudo rm -rf ${ROOTFS_DIR}
 $ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder
-$ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
+$ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true ./rootfs.sh ${distro}'
+```
+
+You MUST choose a distribution (e.g., `ubuntu`) for `${distro}`.
+You can get a supported distributions list in the Kata Containers by running the following.
+
+```
+$ ./rootfs.sh -l
+```
+
+If you want to build the agent without seccomp capability, you need to run the `rootfs.sh` script with `SECCOMP=no` as follows.
+
+```
+$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
 ```
-You MUST choose one of `alpine`, `centos`, `clearlinux`, `debian`, `euleros`, `fedora`, `suse`, and `ubuntu` for `${distro}`. By default `seccomp` packages are not included in the rootfs image. Set `SECCOMP` to `yes` to include them.
 
 > **Note:**
 >
@@ -325,6 +306,7 @@ $ script -fec 'sudo -E USE_DOCKER=true ./image_builder.sh ${ROOTFS_DIR}'
 > - If you do *not* wish to build under Docker, remove the `USE_DOCKER`
 >   variable in the previous command and ensure the `qemu-img` command is
 >   available on your system.
+>   - If `qemu-img` is not installed, you will likely see errors such as `ERROR: File /dev/loop19p1 is not a block device` and `losetup: /tmp/tmp.bHz11oY851: Warning: file is smaller than 512 bytes; the loop device may be useless or invisible for system tools`. These can be mitigated by installing the `qemu-img` command (available in the `qemu-img` package on Fedora or the `qemu-utils` package on Debian).
 
 
 ### Install the rootfs image
@@ -343,12 +325,23 @@ $ (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
 $ export ROOTFS_DIR="${GOPATH}/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder/rootfs"
 $ sudo rm -rf ${ROOTFS_DIR}
 $ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder
-$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
+$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true ./rootfs.sh ${distro}'
 ```
 `AGENT_INIT` controls if the guest image uses the Kata agent as the guest `init` process. When you create an initrd image,
-always set `AGENT_INIT` to `yes`. By default `seccomp` packages are not included in the initrd image. Set `SECCOMP` to `yes` to include them.
+always set `AGENT_INIT` to `yes`.
 
-You MUST choose one of `alpine`, `centos`, `clearlinux`, `euleros`, and `fedora` for `${distro}`.
+You MUST choose a distribution (e.g., `ubuntu`) for `${distro}`.
+You can get a supported distributions list in the Kata Containers by running the following.
+
+```
+$ ./rootfs.sh -l
+```
+
+If you want to build the agent without seccomp capability, you need to run the `rootfs.sh` script with `SECCOMP=no` as follows.
+
+```
+$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
+```
 
 > **Note:**
 >
@@ -470,7 +463,7 @@ script and paste its output directly into a
 > [runtime](../src/runtime) repository.
 
 To perform analysis on Kata logs, use the
-[`kata-log-parser`](https://github.com/kata-containers/tests/tree/master/cmd/log-parser)
+[`kata-log-parser`](https://github.com/kata-containers/tests/tree/main/cmd/log-parser)
 tool, which can convert the logs into formats (e.g. JSON, TOML, XML, and YAML).
 
 See [Set up a debug console](#set-up-a-debug-console).

docs/Documentation-Requirements.md

@@ -1,16 +1,3 @@
-* [Introduction](#introduction)
-* [General requirements](#general-requirements)
-* [Linking advice](#linking-advice)
-* [Notes](#notes)
-* [Warnings and other admonitions](#warnings-and-other-admonitions)
-* [Files and command names](#files-and-command-names)
-* [Code blocks](#code-blocks)
-* [Images](#images)
-* [Spelling](#spelling)
-* [Names](#names)
-* [Version numbers](#version-numbers)
-* [The apostrophe](#the-apostrophe)
-
 # Introduction
 
 This document outlines the requirements for all documentation in the [Kata
@@ -23,10 +10,6 @@ All documents must:
 - Be written in simple English.
 - Be written in [GitHub Flavored Markdown](https://github.github.com/gfm) format.
 - Have a `.md` file extension.
-- Include a TOC (table of contents) at the top of the document with links to
-  all heading sections. We recommend using the
-  [`kata-check-markdown`](https://github.com/kata-containers/tests/tree/main/cmd/check-markdown)
-  tool to generate the TOC.
 - Be linked to from another document in the same repository.
 
   Although GitHub allows navigation of the entire repository, it should be
@@ -43,6 +26,10 @@ All documents must:
   which can then execute the commands specified to ensure the instructions are
   correct. This avoids documents becoming out of date over time.
 
+> **Note:**
+>
+> Do not add a table of contents (TOC) since GitHub will auto-generate one.
+
 # Linking advice
 
 Linking between documents is strongly encouraged to help users and developers

docs/Licensing-strategy.md

@@ -1,9 +1,5 @@
 # Licensing strategy
 
-* [Project License](#project-license)
-* [License file](#license-file)
-* [License for individual files](#license-for-individual-files)      
-
 ## Project License
 
 The license for the [Kata Containers](https://github.com/kata-containers)

docs/Limitations.md

@@ -1,35 +1,3 @@
-* [Overview](#overview)
-* [Definition of a limitation](#definition-of-a-limitation)
-* [Scope](#scope)
-* [Contributing](#contributing)
-* [Pending items](#pending-items)
-    * [Runtime commands](#runtime-commands)
-        * [checkpoint and restore](#checkpoint-and-restore)
-        * [events command](#events-command)
-        * [update command](#update-command)
-    * [Networking](#networking)
-        * [Docker swarm and compose support](#docker-swarm-and-compose-support)
-    * [Resource management](#resource-management)
-        * [docker run and shared memory](#docker-run-and-shared-memory)
-        * [docker run and sysctl](#docker-run-and-sysctl)
-    * [Docker daemon features](#docker-daemon-features)
-        * [SELinux support](#selinux-support)
-* [Architectural limitations](#architectural-limitations)
-    * [Networking limitations](#networking-limitations)
-        * [Support for joining an existing VM network](#support-for-joining-an-existing-vm-network)
-        * [docker --net=host](#docker---nethost)
-        * [docker run --link](#docker-run---link)
-    * [Storage limitations](#storage-limitations)
-        * [Kubernetes `volumeMounts.subPaths`](#kubernetes-volumemountssubpaths)
-    * [Host resource sharing](#host-resource-sharing)
-        * [docker run --privileged](#docker-run---privileged)
-* [Miscellaneous](#miscellaneous)
-    * [Docker --security-opt option partially supported](#docker---security-opt-option-partially-supported)
-* [Appendices](#appendices)
-    * [The constraints challenge](#the-constraints-challenge)
-
-***
-
 # Overview
 
 A [Kata Container](https://github.com/kata-containers) utilizes a Virtual Machine (VM) to enhance security and

docs/README.md

@@ -1,16 +1,5 @@
 # Documentation
 
-* [Getting Started](#getting-started)
-* [More User Guides](#more-user-guides)
-* [Kata Use-Cases](#kata-use-cases)
-* [Developer Guide](#developer-guide)
-    * [Design and Implementations](#design-and-implementations)
-    * [How to Contribute](#how-to-contribute)
-    * [Code Licensing](#code-licensing)
-    * [The Release Process](#the-release-process)
-* [Help Improving the Documents](#help-improving-the-documents)
-* [Website Changes](#website-changes)
-
 The [Kata Containers](https://github.com/kata-containers)
 documentation repository hosts overall system documentation, with information
 common to multiple components.
@@ -22,6 +11,10 @@ For details of the other Kata Containers repositories, see the
 
 * [Installation guides](./install/README.md): Install and run Kata Containers with Docker or Kubernetes
 
+## Tracing
+
+See the [tracing documentation](tracing.md).
+
 ## More User Guides
 
 * [Upgrading](Upgrading.md): how to upgrade from [Clear Containers](https://github.com/clearcontainers) and [runV](https://github.com/hyperhq/runv) to [Kata Containers](https://github.com/kata-containers) and how to upgrade an existing Kata Containers system to the latest version.
@@ -51,6 +44,7 @@ Documents that help to understand and contribute to Kata Containers.
 * [Kata Containers Architecture](design/architecture.md): Architectural overview of Kata Containers
 * [Kata Containers E2E Flow](design/end-to-end-flow.md): The entire end-to-end flow of Kata Containers
 * [Kata Containers design](./design/README.md): More Kata Containers design documents
+* [Kata Containers threat model](./threat-model/threat-model.md): Kata Containers threat model
 
 ### How to Contribute
 

docs/Release-Process.md

@@ -1,20 +1,6 @@
-
 # How to do a Kata Containers Release
   This document lists the tasks required to create a Kata Release.
 
-<!-- TOC START min:1 max:3 link:true asterisk:false update:true -->
-- [How to do a Kata Containers Release](#how-to-do-a-kata-containers-release)
-  - [Requirements](#requirements)
-  - [Release Process](#release-process)
-    - [Bump all Kata repositories](#bump-all-kata-repositories)
-    - [Merge all bump version Pull requests](#merge-all-bump-version-pull-requests)
-    - [Tag all Kata repositories](#tag-all-kata-repositories)
-    - [Check Git-hub Actions](#check-git-hub-actions)
-    - [Create release notes](#create-release-notes)
-    - [Announce the release](#announce-the-release)
-<!-- TOC END -->
-
-
 ## Requirements
 
 - [hub](https://github.com/github/hub)
@@ -78,7 +64,7 @@
 
 ### Check Git-hub Actions
 
-  We make use of [GitHub actions](https://github.com/features/actions) in this [file](https://github.com/kata-containers/kata-containers/blob/main/.github/workflows/main.yaml) in the `kata-containers/kata-containers` repository to build and upload release artifacts. This action is auto triggered with the above step when a new tag is pushed to the `kata-containers/kata-containers` repository.
+  We make use of [GitHub actions](https://github.com/features/actions) in this [file](https://github.com/kata-containers/kata-containers/blob/main/.github/workflows/release.yaml) in the `kata-containers/kata-containers` repository to build and upload release artifacts. This action is auto triggered with the above step when a new tag is pushed to the `kata-containers/kata-containers` repository.
 
   Check the [actions status page](https://github.com/kata-containers/kata-containers/actions) to verify all steps in the actions workflow have completed successfully. On success, a static tarball containing Kata release artifacts will be uploaded to the [Release page](https://github.com/kata-containers/kata-containers/releases).
 

docs/Upgrading.md

@@ -1,16 +1,3 @@
-* [Introduction](#introduction)
-* [Maintenance warning](#maintenance-warning)
-* [Determine current version](#determine-current-version)
-* [Determine latest version](#determine-latest-version)
-* [Configuration changes](#configuration-changes)
-* [Upgrade Kata Containers](#upgrade-kata-containers)
-    * [Upgrade native distribution packaged version](#upgrade-native-distribution-packaged-version)
-    * [Static installation](#static-installation)
-        * [Determine if you are using a static installation](#determine-if-you-are-using-a-static-installation)
-        * [Remove a static installation](#remove-a-static-installation)
-        * [Upgrade a static installation](#upgrade-a-static-installation)
-* [Custom assets](#custom-assets)
-
 # Introduction
 
 This document outlines the options for upgrading from a

docs/design/README.md

@@ -10,3 +10,7 @@ Kata Containers design documents:
 - [Host cgroups](host-cgroups.md)
 - [`Inotify` support](inotify.md)
 - [Metrics(Kata 2.0)](kata-2-0-metrics.md)
+
+---
+
+- [Design proposals](proposals)

docs/design/VSocks.md

@@ -1,12 +1,5 @@
 # Kata Containers and VSOCKs
 
-- [Introduction](#introduction)
-    - [VSOCK communication diagram](#vsock-communication-diagram)
-- [System requirements](#system-requirements)
-- [Advantages of using VSOCKs](#advantages-of-using-vsocks)
-    - [High density](#high-density)
-    - [Reliability](#reliability)
-
 ## Introduction
 
 There are two different ways processes in the virtual machine can communicate

docs/design/architecture.md

@@ -1,26 +1,5 @@
 # Kata Containers Architecture
 
-
-- [Kata Containers Architecture](#kata-containers-architecture)
-  - [Overview](#overview)
-  - [Virtualization](#virtualization)
-  - [Guest assets](#guest-assets)
-    - [Guest kernel](#guest-kernel)
-    - [Guest image](#guest-image)
-      - [Root filesystem image](#root-filesystem-image)
-      - [Initrd image](#initrd-image)
-  - [Agent](#agent)
-  - [Runtime](#runtime)
-    - [Configuration](#configuration)
-  - [Networking](#networking)
-    - [Network Hotplug](#network-hotplug)
-  - [Storage](#storage)
-  - [Kubernetes support](#kubernetes-support)
-      - [OCI annotations](#oci-annotations)
-      - [Mixing VM based and namespace based runtimes](#mixing-vm-based-and-namespace-based-runtimes)
-- [Appendices](#appendices)
-  - [DAX](#dax)
-
 ## Overview
 
 This is an architectural overview of Kata Containers, based on the 2.0 release.
@@ -35,7 +14,7 @@ through the [CRI-O\*](https://github.com/kubernetes-incubator/cri-o) and
 
 Kata Containers creates a QEMU\*/KVM virtual machine for pod that `kubelet` (Kubernetes) creates respectively.
 
-The [`containerd-shim-kata-v2` (shown as `shimv2` from this point onwards)](../../src/runtime/containerd-shim-v2) 
+The [`containerd-shim-kata-v2` (shown as `shimv2` from this point onwards)](../../src/runtime/cmd/containerd-shim-kata-v2/)
 is the Kata Containers entrypoint, which
 implements the [Containerd Runtime V2 (Shim API)](https://github.com/containerd/containerd/tree/master/runtime/v2) for Kata.
 
@@ -280,7 +259,7 @@ With `RuntimeClass`, users can define Kata Containers as a `RuntimeClass` and th
 
 ## DAX
 
-Kata Containers utilizes the Linux kernel DAX [(Direct Access filesystem)](https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/Documentation/filesystems/dax.txt)
+Kata Containers utilizes the Linux kernel DAX [(Direct Access filesystem)](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/filesystems/dax.rst?h=v5.14)
 feature to efficiently map some host-side files into the guest VM space.
 In particular, Kata Containers uses the QEMU NVDIMM feature to provide a
 memory-mapped virtual device that can be used to DAX map the virtual machine's

docs/design/end-to-end-flow.md

@@ -1,4 +1,3 @@
 # Kata Containers E2E Flow
 
-
 ![Kata containers e2e flow](arch-images/katacontainers-e2e-with-bg.jpg)

docs/design/host-cgroups.md

@@ -1,18 +1,3 @@
-- [Host cgroup management](#host-cgroup-management)
-  - [Introduction](#introduction)
-  - [`SandboxCgroupOnly` enabled](#sandboxcgrouponly-enabled)
-    - [What does Kata do in this configuration?](#what-does-kata-do-in-this-configuration)
-    - [Why create a Kata-cgroup under the parent cgroup?](#why-create-a-kata-cgroup-under-the-parent-cgroup)
-    - [Improvements](#improvements)
-  - [`SandboxCgroupOnly` disabled (default, legacy)](#sandboxcgrouponly-disabled-default-legacy)
-    - [What does this method do?](#what-does-this-method-do)
-      - [Impact](#impact)
-- [Supported cgroups](#supported-cgroups)
-  - [Cgroups V1](#cgroups-v1)
-  - [Cgroups V2](#cgroups-v2)
-    - [Distro Support](#distro-support)
-- [Summary](#summary)
-
 # Host cgroup management
 
 ## Introduction
@@ -27,187 +12,244 @@ The OCI [runtime specification][linux-config] provides guidance on where the con
   > [`cgroupsPath`][cgroupspath]: (string, OPTIONAL) path to the cgroups. It can be used to either control the cgroups
   > hierarchy for containers or to run a new process in an existing container
 
-cgroups are hierarchical, and this can be seen with the following pod example:
+Cgroups are hierarchical, and this can be seen with the following pod example:
 
 - Pod 1: `cgroupsPath=/kubepods/pod1`
-  - Container 1:
-`cgroupsPath=/kubepods/pod1/container1`
-  - Container 2:
-`cgroupsPath=/kubepods/pod1/container2`
+  - Container 1: `cgroupsPath=/kubepods/pod1/container1`
+  - Container 2: `cgroupsPath=/kubepods/pod1/container2`
 
 - Pod 2: `cgroupsPath=/kubepods/pod2`
-  - Container 1:
-`cgroupsPath=/kubepods/pod2/container2`
-  - Container 2:
-`cgroupsPath=/kubepods/pod2/container2`
+  - Container 1: `cgroupsPath=/kubepods/pod2/container2`
+  - Container 2: `cgroupsPath=/kubepods/pod2/container2`
 
-Depending on the upper-level orchestrator, the cgroup under which the pod is placed is
-managed by the orchestrator. In the case of Kubernetes, the pod-cgroup is created by Kubelet,
-while the container cgroups are to be handled by the runtime. Kubelet will size the pod-cgroup
-based on the container resource requirements.
+Depending on the upper-level orchestration layers, the cgroup under which the pod is placed is
+managed by the orchestrator or not. In the case of Kubernetes, the pod cgroup is created by Kubelet,
+while the container cgroups are to be handled by the runtime.
+Kubelet will size the pod cgroup based on the container resource requirements, to which it may add
+a configured set of [pod resource overheads](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-overhead/).
 
-Kata Containers introduces a non-negligible overhead for running a sandbox (pod). Based on this, two scenarios are possible:
- 1) The upper-layer orchestrator takes the overhead of running a sandbox into account when sizing the pod-cgroup, or
- 2) Kata Containers do not fully constrain the VMM and associated processes, instead placing a subset of them outside of the pod-cgroup.
+Kata Containers introduces a non-negligible resource overhead for running a sandbox (pod). Typically, the Kata shim,
+through its underlying VMM invocation, will create many additional threads compared to process based container runtimes:
+the para-virtualized I/O back-ends, the VMM instance or even the Kata shim process, all of those host processes consume
+memory and CPU time not directly tied to the container workload, and introduces a sandbox resource overhead.
+In order for a Kata workload to run without significant performance degradation, its sandbox overhead must be
+provisioned accordingly. Two scenarios are possible:
 
-Kata Containers provides two options for how cgroups are handled on the host. Selection of these options is done through
-the `SandboxCgroupOnly` flag within the Kata Containers [configuration](../../src/runtime/README.md#configuration)
-file.
+ 1) The upper-layer orchestrator takes the overhead of running a sandbox into account when sizing the pod cgroup.
+    For example, Kubernetes [`PodOverhead`](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-overhead/)
+	feature lets the orchestrator add a configured sandbox overhead to the sum of all its containers resources. In
+	that case, the pod sandbox is properly sized and all Kata created processes will run under the pod cgroup
+	defined constraints and limits.
+ 2) The upper-layer orchestrator does **not** take the sandbox overhead into account and the pod cgroup is not
+	sized to properly run all Kata created processes. With that scenario, attaching all the Kata processes to the sandbox
+	cgroup may lead to non-negligible workload performance degradations. As a consequence, Kata Containers will move
+	all processes but the vCPU threads into a dedicated overhead cgroup under `/kata_overhead`. The Kata runtime will
+	not apply any constraints or limits to that cgroup, it is up to the infrastructure owner to optionally set it up.
 
-## `SandboxCgroupOnly` enabled
+Those 2 scenarios are not dynamically detected by the Kata Containers runtime implementation, and thus the
+infrastructure owner must configure the runtime according to how the upper-layer orchestrator creates and sizes the
+pod cgroup. That configuration selection is done through the `sandbox_cgroup_only` flag within the Kata Containers
+[configuration](../../src/runtime/README.md#configuration) file.
 
-With `SandboxCgroupOnly` enabled, it is expected that the parent cgroup is sized to take the overhead of running
-a sandbox into account. This is ideal, as all the applicable Kata Containers components can be placed within the
-given cgroup-path.
+## `sandbox_cgroup_only = true`
 
-In the context of Kubernetes, Kubelet will size the pod-cgroup to take the overhead of running a Kata-based sandbox
-into account. This will be feasible in the 1.16 Kubernetes release through the `PodOverhead` feature.
+Setting `sandbox_cgroup_only` to `true` from the Kata Containers configuration file means that the pod cgroup is
+properly sized and takes the pod overhead into account. This is ideal, as all the applicable Kata Containers processes
+can simply be placed within the given cgroup path.
+
+In the context of Kubernetes, Kubelet can size the pod cgroup to take the overhead of running a Kata-based sandbox
+into account. This has been supported since the 1.16 Kubernetes release, through the
+[`PodOverhead`](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-overhead/) feature.
 
 ```
-+----------------------------------------------------------+
-|    +---------------------------------------------------+ |
-|    |   +---------------------------------------------+ | |
-|    |   |   +--------------------------------------+  | | |
-|    |   |   | kata-shimv2, VMM and threads:        |  | | |
-|    |   |   |  (VMM, IO-threads, vCPU threads, etc)|  | | |
-|    |   |   |                                      |  | | |
-|    |   |   | kata_<sandbox-id>                    |  | | |
-|    |   |   +--------------------------------------+  | | |
-|    |   |                                             | | |
-|    |   |Pod 1                                        | | |
-|    |   +---------------------------------------------+ | |
-|    |                                                   | |
-|    |   +---------------------------------------------+ | |
-|    |   |   +--------------------------------------+  | | |
-|    |   |   | kata-shimv2, VMM and threads:        |  | | |
-|    |   |   |  (VMM, IO-threads, vCPU threads, etc)|  | | |
-|    |   |   |                                      |  | | |
-|    |   |   | kata_<sandbox-id>                    |  | | |
-|    |   |   +--------------------------------------+  | | |
-|    |   |Pod 2                                        | | |
-|    |   +---------------------------------------------+ | |
-|    |kubepods                                           | |
-|    +---------------------------------------------------+ |
-|                                                          |
-|Node                                                      |
-+----------------------------------------------------------+
+┌─────────────────────────────────────────┐
+│                                         │
+│  ┌──────────────────────────────────┐   │
+│  │                                  │   │
+│  │ ┌─────────────────────────────┐  │   │
+│  │ │                             │  │   │
+│  │ │ ┌─────────────────────┐     │  │   │
+│  │ │ │ vCPU threads        │     │  │   │
+│  │ │ │ I/O threads         │     │  │   │
+│  │ │ │ VMM                 │     │  │   │
+│  │ │ │ Kata Shim           │     │  │   │
+│  │ │ │                     │     │  │   │
+│  │ │ │ /kata_<sandbox_id>  │     │  │   │
+│  │ │ └─────────────────────┘     │  │   │
+│  │ │Pod 1                        │  │   │
+│  │ └─────────────────────────────┘  │   │
+│  │                                  │   │
+│  │ ┌─────────────────────────────┐  │   │
+│  │ │                             │  │   │
+│  │ │ ┌─────────────────────┐     │  │   │
+│  │ │ │ vCPU threads        │     │  │   │
+│  │ │ │ I/O threads         │     │  │   │
+│  │ │ │ VMM                 │     │  │   │
+│  │ │ │ Kata Shim           │     │  │   │
+│  │ │ │                     │     │  │   │
+│  │ │ │ /kata_<sandbox_id>  │     │  │   │
+│  │ │ └─────────────────────┘     │  │   │
+│  │ │Pod 2                        │  │   │
+│  │ └─────────────────────────────┘  │   │
+│  │                                  │   │
+│  │/kubepods                         │   │
+│  └──────────────────────────────────┘   │
+│                                         │
+│ Node                                    │
+└─────────────────────────────────────────┘
 ```
 
-### What does Kata do in this configuration?
-1. Given a `PodSandbox` container creation, let:
+### Implementation details
 
-   ```
-   podCgroup=Parent(container.CgroupsPath)
-   KataSandboxCgroup=<podCgroup>/kata_<PodSandboxID>
-   ```
+When `sandbox_cgroup_only` is enabled, the Kata shim will create a per pod
+sub-cgroup under the pod's dedicated cgroup. For example, in the Kubernetes context,
+it will create a `/kata_<PodSandboxID>` under the `/kubepods` cgroup hierarchy.
+On a typical cgroup v1 hierarchy mounted under `/sys/fs/cgroup/`, the memory cgroup
+subsystem for a pod with sandbox ID `12345678` would live under
+`/sys/fs/cgroup/memory/kubepods/kata_12345678`.
 
-2. Create the cgroup, `KataSandboxCgroup`
+In most cases, the `/kata_<PodSandboxID>` created cgroup is unrestricted and inherits and shares all
+constraints and limits from the parent cgroup (`/kubepods` in the Kubernetes case). The exception is
+for the `cpuset` and `devices` cgroup subsystems, which are managed by the Kata shim.
 
-3. Join the `KataSandboxCgroup`
+After creating the `/kata_<PodSandboxID>` cgroup, the Kata Containers shim will move itself to it, **before** starting
+the virtual machine. As a consequence all processes subsequently created by the Kata Containers shim (the VMM itself, and
+all vCPU and I/O related threads) will be created in the `/kata_<PodSandboxID>` cgroup.
 
-Any process created by the runtime will be created in `KataSandboxCgroup`.
-The runtime will limit the cgroup in the host only if the sandbox doesn't have a
-container type annotation, but the caller is free to set the proper limits for the `podCgroup`.
+### Why create a kata-cgroup under the parent cgroup?
 
-In the example above the pod cgroups are `/kubepods/pod1` and `/kubepods/pod2`.
-Kata creates the unrestricted sandbox cgroup under the pod cgroup.
+And why not directly adding the per sandbox shim directly to the pod cgroup (e.g. 
+`/kubepods` in the Kubernetes context)?
 
-### Why create a Kata-cgroup under the parent cgroup?
+The Kata Containers shim implementation creates a per-sandbox cgroup
+(`/kata_<PodSandboxID>`) to support the `Docker` use case. Although `Docker` does not
+have a notion of pods, Kata Containers still creates a sandbox to support the pod-less,
+single container use case that `Docker` implements. Since `Docker` does create any
+cgroup hierarchy to place a container into, it would be very complex for Kata to map
+a particular container to its sandbox without placing it under a `/kata_<containerID>>`
+sub-cgroup first.
 
-`Docker` does not have a notion of pods, and will not create a cgroup directory
-to place a particular container in (i.e., all containers would be in a path like
-`/docker/container-id`. To simplify the implementation and continue to support `Docker`,
-Kata Containers creates the sandbox-cgroup, in the case of Kubernetes, or a container cgroup, in the case
-of docker.
+### Advantages
 
-### Improvements
+Keeping all Kata Containers processes under a properly sized pod cgroup is ideal
+and makes for a simpler Kata Containers implementation. It also helps with gathering
+accurate statistics and preventing Kata workloads from being noisy neighbors.
 
-- Get statistics about pod resources
+#### Pod resources statistics
 
 If the Kata caller wants to know the resource usage on the host it can get
 statistics from the pod cgroup. All cgroups stats in the hierarchy will include
 the Kata overhead. This gives the possibility of gathering usage-statics at the
 pod level and the container level.
 
-- Better host resource isolation
+#### Better host resource isolation
 
 Because the Kata runtime will place all the Kata processes in the pod cgroup,
 the resource limits that the caller applies to the pod cgroup will affect all
 processes that belong to the Kata sandbox in the host. This will improve the
 isolation in the host preventing Kata to become a noisy neighbor.
 
-## `SandboxCgroupOnly` disabled (default, legacy)
+## `sandbox_cgroup_only = false` (Default setting)
+
+If the cgroup provided to Kata is not sized appropriately, Kata components will
+consume resources that the actual container workloads expect to see and use.
+This can cause instability and performance degradations.
+
+To avoid that situation, Kata Containers creates an unconstrained overhead
+cgroup and moves all non workload related processes (Anything but the virtual CPU
+threads) to it. The name of this overhead cgroup is `/kata_overhead` and a per
+sandbox sub cgroup will be created under it for each sandbox Kata Containers creates.
+
+Kata Containers does not add any constraints or limitations on the overhead cgroup. It is up to the infrastructure
+owner to either:
+
+- Provision nodes with a pre-sized `/kata_overhead` cgroup. Kata Containers will
+  load that existing cgroup and move all non workload related processes to it.
+- Let Kata Containers create the `/kata_overhead` cgroup, leave it
+  unconstrained or resize it a-posteriori.
 
-If the cgroup provided to Kata is not sized appropriately, instability will be
-introduced when fully constraining Kata components, and the user-workload will
-see a subset of resources that were requested. Based on this, the default
-handling for Kata Containers is to not fully constrain the VMM and Kata
-components on the host.
 
 ```
-+----------------------------------------------------------+
-|    +---------------------------------------------------+ |
-|    |   +---------------------------------------------+ | |
-|    |   |   +--------------------------------------+  | | |
-|    |   |   |Container 1       |-|Container 2      |  | | |
-|    |   |   |                  |-|                 |  | | |
-|    |   |   | Shim+container1  |-| Shim+container2 |  | | |
-|    |   |   +--------------------------------------+  | | |
-|    |   |                                             | | |
-|    |   |Pod 1                                        | | |
-|    |   +---------------------------------------------+ | |
-|    |                                                   | |
-|    |   +---------------------------------------------+ | |
-|    |   |   +--------------------------------------+  | | |
-|    |   |   |Container 1       |-|Container 2      |  | | |
-|    |   |   |                  |-|                 |  | | |
-|    |   |   | Shim+container1  |-| Shim+container2 |  | | |
-|    |   |   +--------------------------------------+  | | |
-|    |   |                                             | | |
-|    |   |Pod 2                                        | | |
-|    |   +---------------------------------------------+ | |
-|    |kubepods                                           | |
-|    +---------------------------------------------------+ |
-|    +---------------------------------------------------+ |
-|    |  Hypervisor                                       | |
-|    |Kata                                               | |
-|    +---------------------------------------------------+ |
-|                                                          |
-|Node                                                      |
-+----------------------------------------------------------+
+┌────────────────────────────────────────────────────────────────────┐
+│                                                                    │
+│  ┌─────────────────────────────┐    ┌───────────────────────────┐  │
+│  │                             │    │                           │  │
+│  │   ┌─────────────────────────┼────┼─────────────────────────┐ │  │
+│  │   │                         │    │                         │ │  │
+│  │   │ ┌─────────────────────┐ │    │ ┌─────────────────────┐ │ │  │
+│  │   │ │  vCPU threads       │ │    │ │  VMM                │ │ │  │
+│  │   │ │                     │ │    │ │  I/O threads        │ │ │  │
+│  │   │ │                     │ │    │ │  Kata Shim          │ │ │  │
+│  │   │ │                     │ │    │ │                     │ │ │  │
+│  │   │ │ /kata_<sandbox_id>  │ │    │ │ /<sandbox_id>       │ │ │  │
+│  │   │ └─────────────────────┘ │    │ └─────────────────────┘ │ │  │
+│  │   │                         │    │                         │ │  │
+│  │   │  Pod 1                  │    │                         │ │  │
+│  │   └─────────────────────────┼────┼─────────────────────────┘ │  │
+│  │                             │    │                           │  │
+│  │                             │    │                           │  │
+│  │   ┌─────────────────────────┼────┼─────────────────────────┐ │  │
+│  │   │                         │    │                         │ │  │
+│  │   │ ┌─────────────────────┐ │    │ ┌─────────────────────┐ │ │  │
+│  │   │ │  vCPU threads       │ │    │ │  VMM                │ │ │  │
+│  │   │ │                     │ │    │ │  I/O threads        │ │ │  │
+│  │   │ │                     │ │    │ │  Kata Shim          │ │ │  │
+│  │   │ │                     │ │    │ │                     │ │ │  │
+│  │   │ │ /kata_<sandbox_id>  │ │    │ │ /<sandbox_id>       │ │ │  │
+│  │   │ └─────────────────────┘ │    │ └─────────────────────┘ │ │  │
+│  │   │                         │    │                         │ │  │
+│  │   │  Pod 2                  │    │                         │ │  │
+│  │   └─────────────────────────┼────┼─────────────────────────┘ │  │
+│  │                             │    │                           │  │
+│  │ /kubepods                   │    │ /kata_overhead            │  │
+│  └─────────────────────────────┘    └───────────────────────────┘  │
+│                                                                    │
+│                                                                    │
+│ Node                                                               │
+└────────────────────────────────────────────────────────────────────┘
 
 ```
 
-### What does this method do?
+### Implementation Details
 
-1. Given a container creation let `containerCgroupHost=container.CgroupsPath`
-1. Rename `containerCgroupHost` path to add `kata_`
-1. Let `PodCgroupPath=PodSanboxContainerCgroup` where `PodSanboxContainerCgroup` is the cgroup of a container of type `PodSandbox`
-1. Limit the `PodCgroupPath` with the sum of all the container limits in the Sandbox
-1. Move only vCPU threads of hypervisor to `PodCgroupPath`
-1. Per each container, move its `kata-shim` to its own `containerCgroupHost`
-1. Move hypervisor and applicable threads to memory cgroup `/kata`
+When `sandbox_cgroup_only` is disabled, the Kata Containers shim will create a per pod
+sub-cgroup under the pods dedicated cgroup, and another one under the overhead cgroup.
+For example, in the Kubernetes context, it will create a `/kata_<PodSandboxID>` under
+the `/kubepods` cgroup hierarchy, and a `/<PodSandboxID>` under the `/kata_overhead` one.
 
-_Note_: the Kata Containers runtime will not add all the hypervisor threads to
-the cgroup path requested, only vCPUs. These threads are run unconstrained.
+On a typical cgroup v1 hierarchy mounted under `/sys/fs/cgroup/`, for a pod which sandbox
+ID is `12345678`, create with `sandbox_cgroup_only` disabled, the 2 memory subsystems
+for the sandbox cgroup and the overhead cgroup would respectively live under 
+`/sys/fs/cgroup/memory/kubepods/kata_12345678` and `/sys/fs/cgroup/memory/kata_overhead/12345678`.
 
-This mitigates the risk of the VMM and other threads receiving an out of memory scenario (`OOM`).
+Unlike when `sandbox_cgroup_only` is enabled, the Kata Containers shim will move itself
+to the overhead cgroup first, and then move the vCPU threads to the sandbox cgroup as
+they're created. All Kata processes and threads will run under the overhead cgroup except for
+the vCPU threads. 
 
+With `sandbox_cgroup_only` disabled, Kata Containers assumes the pod cgroup is only sized
+to accommodate for the actual container workloads processes. For Kata, this maps
+to the VMM created virtual CPU threads and so they are the only ones running under the pod
+cgroup. This mitigates the risk of the VMM, the Kata shim and the I/O threads going through
+a catastrophic out of memory scenario (`OOM`).
 
-#### Impact
+#### Pros and Cons
 
-If resources are reserved at a system level to account for the overheads of
-running sandbox containers, this configuration can be utilized with adequate
-stability. In this scenario, non-negligible amounts of CPU and memory will be
-utilized unaccounted for on the host.
+Running all non vCPU threads under an unconstrained overhead cgroup could lead to workloads
+potentially consuming a large amount of host resources.
 
-[linux-config]: https://github.com/opencontainers/runtime-spec/blob/master/config-linux.md
-[cgroupspath]: https://github.com/opencontainers/runtime-spec/blob/master/config-linux.md#cgroups-path
+On the other hand, running all non vCPU threads under a dedicated overhead cgroup can provide
+accurate metrics on the actual Kata Container pod overhead, allowing for tuning the overhead
+cgroup size and constraints accordingly.
+
+[linux-config]: https://github.com/opencontainers/runtime-spec/blob/main/config-linux.md
+[cgroupspath]: https://github.com/opencontainers/runtime-spec/blob/main/config-linux.md#cgroups-path
 
 # Supported cgroups
 
-Kata Containers supports cgroups `v1` and `v2`. In the following sections each cgroup is
-described briefly and what changes are needed in Kata Containers to support it.
+Kata Containers currently only supports cgroups `v1`. 
+
+In the following sections each cgroup is described briefly.
 
 ## Cgroups V1
 
@@ -259,7 +301,7 @@ diagram:
 A process can join a cgroup by writing its process id (`pid`) to `cgroup.procs` file,
 or join a cgroup partially by writing the task (thread) id (`tid`) to the `tasks` file.
 
-Kata Containers supports `v1` by default and no change in the configuration file is needed.
+Kata Containers only supports `v1`.
 To know more about `cgroups v1`, see [cgroupsv1(7)][2].
 
 ## Cgroups V2
@@ -312,22 +354,13 @@ Same as `cgroups v1`, a process can join the cgroup by writing its process id (`
 `cgroup.procs` file, or join a cgroup partially by writing the task (thread) id (`tid`) to
 `cgroup.threads` file.
 
-For backwards compatibility Kata Containers defaults to supporting cgroups v1 by default.
-To change this to `v2`, set `sandbox_cgroup_only=true` in the `configuration.toml` file.
-To know more about `cgroups v2`, see [cgroupsv2(7)][3].
+Kata Containers does not support cgroups `v2` on the host.
 
 ### Distro Support
 
 Many Linux distributions do not yet support `cgroups v2`, as it is quite a recent addition.
 For more information about the status of this feature see [issue #2494][4].
 
-# Summary
-
-| cgroup option | default? | status | pros | cons | cgroups
-|-|-|-|-|-|-|
-| `SandboxCgroupOnly=false` | yes | legacy | Easiest to make Kata work | Unaccounted for memory and resource utilization | v1
-| `SandboxCgroupOnly=true` | no | recommended | Complete tracking of Kata memory and CPU utilization. In Kubernetes, the Kubelet can fully constrain Kata via the pod cgroup | Requires upper layer orchestrator which sizes sandbox cgroup appropriately | v1, v2
-
 
 [1]: http://man7.org/linux/man-pages/man5/tmpfs.5.html
 [2]: http://man7.org/linux/man-pages/man7/cgroups.7.html#CGROUPS_VERSION_1

docs/design/kata-2-0-metrics.md

@@ -1,20 +1,5 @@
 # Kata 2.0 Metrics Design
 
-* [Limitations of Kata 1.x and the target of Kata 2.0](#limitations-of-kata-1x-and-the-target-of-kata-20)
-* [Metrics architecture](#metrics-architecture)
-  * [Kata monitor](#kata-monitor)
-  * [Kata runtime](#kata-runtime)
-  * [Kata agent](#kata-agent)
-  * [Performance and overhead](#performance-and-overhead)
-* [Metrics list](#metrics-list)
-  * [Metric types](#metric-types)
-  * [Kata agent metrics](#kata-agent-metrics)
-  * [Firecracker metrics](#firecracker-metrics)
-  * [Kata guest OS metrics](#kata-guest-os-metrics)
-  * [Hypervisor metrics](#hypervisor-metrics)
-  * [Kata monitor metrics](#kata-monitor-metrics)
-  * [Kata containerd shim v2 metrics](#kata-containerd-shim-v2-metrics)
-
 Kata implement CRI's API and support [`ContainerStats`](https://github.com/kubernetes/kubernetes/blob/release-1.18/staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2/api.proto#L101) and [`ListContainerStats`](https://github.com/kubernetes/kubernetes/blob/release-1.18/staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2/api.proto#L103) interfaces to expose containers metrics. User can use these interface to get basic metrics about container.
 
 But unlike `runc`, Kata is a VM-based runtime and has a different architecture.
@@ -222,7 +207,7 @@ Metrics for Firecracker vmm.
 | `kata_firecracker_uart`: <br> Metrics specific to the UART device. | `GAUGE` |  | <ul><li>`item`<ul><li>`error_count`</li><li>`flush_count`</li><li>`missed_read_count`</li><li>`missed_write_count`</li><li>`read_count`</li><li>`write_count`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
 | `kata_firecracker_vcpu`: <br> Metrics specific to VCPUs' mode of functioning. | `GAUGE` |  | <ul><li>`item`<ul><li>`exit_io_in`</li><li>`exit_io_out`</li><li>`exit_mmio_read`</li><li>`exit_mmio_write`</li><li>`failures`</li><li>`filter_cpuid`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
 | `kata_firecracker_vmm`: <br> Metrics specific to the machine manager as a whole. | `GAUGE` |  | <ul><li>`item`<ul><li>`device_events`</li><li>`panic_count`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
-| `kata_firecracker_vsock`: <br> Vsock-related metrics. | `GAUGE` |  | <ul><li>`item`<ul><li>`activate_fails`</li><li>`cfg_fails`</li><li>`conn_event_fails`</li><li>`conns_added`</li><li>`conns_killed`</li><li>`conns_removed`</li><li>`ev_queue_event_fails`</li><li>`killq_resync`</li><li>`muxer_event_fails`</li><li>`rx_bytes_count`</li><li>`rx_packets_count`</li><li>`rx_queue_event_count`</li><li>`rx_queue_event_fails`</li><li>`rx_read_fails`</li><li>`tx_bytes_count`</li><li>`tx_flush_fails`</li><li>`tx_packets_count`</li><li>`tx_queue_event_count`</li><li>`tx_queue_event_fails`</li><li>`tx_write_fails`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
+| `kata_firecracker_vsock`: <br> VSOCK-related metrics. | `GAUGE` |  | <ul><li>`item`<ul><li>`activate_fails`</li><li>`cfg_fails`</li><li>`conn_event_fails`</li><li>`conns_added`</li><li>`conns_killed`</li><li>`conns_removed`</li><li>`ev_queue_event_fails`</li><li>`killq_resync`</li><li>`muxer_event_fails`</li><li>`rx_bytes_count`</li><li>`rx_packets_count`</li><li>`rx_queue_event_count`</li><li>`rx_queue_event_fails`</li><li>`rx_read_fails`</li><li>`tx_bytes_count`</li><li>`tx_flush_fails`</li><li>`tx_packets_count`</li><li>`tx_queue_event_count`</li><li>`tx_queue_event_fails`</li><li>`tx_write_fails`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
 
 ### Kata guest OS metrics
 

docs/design/kata-api-design.md

@@ -1,4 +1,5 @@
 # Kata API Design
+
 To fulfill the [Kata design requirements](kata-design-requirements.md), and based on the discussion on [Virtcontainers API extensions](https://docs.google.com/presentation/d/1dbGrD1h9cpuqAPooiEgtiwWDGCYhVPdatq7owsKHDEQ), the Kata runtime library features the following APIs:
 -  Sandbox based top API
 -  Storage and network hotplug API

docs/design/kata-design-requirements.md

@@ -30,7 +30,7 @@ The Kata Containers runtime **MUST** implement the following command line option
 The Kata Containers project **MUST** provide two interfaces for CRI shims to manage hardware
 virtualization based Kubernetes pods and containers:
 -  An OCI and `runc` compatible command line interface, as described in the previous section.
-This interface is used by implementations such as [`CRI-O`](http://cri-o.io) and [`cri-containerd`](https://github.com/containerd/cri-containerd), for example.
+This interface is used by implementations such as [`CRI-O`](http://cri-o.io) and [`containerd`](https://github.com/containerd/containerd), for example.
 - A hardware virtualization runtime library API for CRI shims to consume and provide a more
 CRI native implementation. The [`frakti`](https://github.com/kubernetes/frakti) CRI shim is an example of such a consumer.
 

docs/design/proposals/README.md

@@ -0,0 +1,5 @@
+# Design proposals
+
+Kata Containers design proposal documents:
+
+- [Kata Containers tracing](tracing-proposals.md)

docs/design/proposals/tracing-proposals.md

@@ -0,0 +1,213 @@
+# Kata Tracing proposals
+
+## Overview
+
+This document summarises a set of proposals triggered by the
+[tracing documentation PR][tracing-doc-pr].
+
+## Required context
+
+This section explains some terminology required to understand the proposals.
+Further details can be found in the
+[tracing documentation PR][tracing-doc-pr].
+
+### Agent trace mode terminology
+
+| Trace mode | Description | Use-case |
+|-|-|-|
+| Static |  Trace agent from startup to shutdown | Entire lifespan |
+| Dynamic | Toggle tracing on/off as desired | On-demand "snapshot" |
+
+### Agent trace type terminology
+
+| Trace type | Description | Use-case |
+|-|-|-|
+| isolated | traces all relate to single component | Observing lifespan |
+| collated | traces "grouped" (runtime+agent) | Understanding component interaction |
+
+### Container lifespan
+
+| Lifespan | trace mode | trace type |
+|-|-|-|
+| short-lived | static | collated if possible, else isolated? |
+| long-running | dynamic | collated? (to see interactions) |
+
+## Original plan for agent
+
+- Implement all trace types and trace modes for agent.
+
+- Why?
+  - Maximum flexibility.
+
+    > **Counterargument:**
+    >
+    > Due to the intrusive nature of adding tracing, we have
+    > learnt that landing small incremental changes is simpler and quicker!
+
+  - Compatibility with [Kata 1.x tracing][kata-1x-tracing].
+
+    > **Counterargument:**
+    >
+    > Agent tracing in Kata 1.x was extremely awkward to setup (to the extent
+    > that it's unclear how many users actually used it!)
+    >
+    > This point, coupled with the new architecture for Kata 2.x, suggests
+    > that we may not need to supply the same set of tracing features (in fact
+    > they may not make sense)).
+
+## Agent tracing proposals
+
+### Agent tracing proposal 1: Don't implement dynamic trace mode
+
+- All tracing will be static.
+
+- Why?
+  - Because dynamic tracing will always be "partial"
+
+    > In fact, not only would it be only a "snapshot" of activity, it may not
+    > even be possible to create a complete "trace transaction". If this is
+    > true, the trace output would be partial and would appear "unstructured".
+
+### Agent tracing proposal 2: Simplify handling of trace type
+
+- Agent tracing will be "isolated" by default.
+- Agent tracing will be "collated" if runtime tracing is also enabled.
+
+- Why?
+  - Offers a graceful fallback for agent tracing if runtime tracing disabled.
+  - Simpler code!
+
+## Questions to ask yourself (part 1)
+
+- Are your containers long-running or short-lived?
+
+- Would you ever need to turn on tracing "briefly"?
+  - If "yes", is a "partial trace" useful or useless?
+
+    > Likely to be considered useless as it is a partial snapshot.
+    > Alternative tracing methods may be more appropriate to dynamic
+    > OpenTelemetry tracing.
+
+## Questions to ask yourself (part 2)
+
+- Are you happy to stop a container to enable tracing?
+  If "no", dynamic tracing may be required.
+
+- Would you ever want to trace the agent and the runtime "in isolation" at the
+  same time?
+  - If "yes", we need to fully implement `trace_mode=isolated`
+
+    > This seems unlikely though.
+
+## Trace collection
+
+The second set of proposals affect the way traces are collected.
+
+### Motivation
+
+Currently:
+
+- The runtime sends trace spans to Jaeger directly.
+- The agent will send trace spans to the [`trace-forwarder`][trace-forwarder] component.
+- The trace forwarder will send trace spans to Jaeger.
+
+Kata agent tracing overview:
+
+```
++-------------------------------------------+
+| Host                                      |
+|                                           |
+| +-----------+                             |
+| | Trace     |                             |
+| | Collector |                             |
+| +-----+-----+                             |
+|       ^                  +--------------+ |
+|       | spans            | Kata VM      | |
+| +-----+-----+            |              | |
+| | Kata      |    spans   |     +-----+  | |
+| | Trace     |<-----------------|Kata |  | |
+| | Forwarder |    VSOCK   |     |Agent|  | |
+| +-----------+    Channel |     +-----+  | |
+|                          +--------------+ |
++-------------------------------------------+
+```
+
+Currently:
+
+- If agent tracing is enabled but the trace forwarder is not running,
+  the agent will error.
+
+- If the trace forwarder is started but Jaeger is not running,
+  the trace forwarder will error.
+
+### Goals
+
+- The runtime and agent should:
+  - Use the same trace collection implementation.
+  - Use the most the common configuration items.
+
+- Kata should should support more trace collection software or `SaaS`
+  (for example `Zipkin`, `datadog`).
+
+- Trace collection should not block normal runtime/agent operations
+  (for example if `vsock-exporter`/Jaeger is not running, Kata Containers should work normally).
+
+### Trace collection proposals
+
+#### Trace collection proposal 1: Send all spans to the trace forwarder as a span proxy
+
+Kata runtime/agent all send spans to trace forwarder, and the trace forwarder,
+acting as a tracing proxy, sends all spans to a tracing back-end, such as Jaeger or `datadog`.
+
+**Pros:**
+
+- Runtime/agent will be simple.
+- Could update trace collection target while Kata Containers are running.
+
+**Cons:**
+
+- Requires the trace forwarder component to be running (that is a pressure to operation).
+
+#### Trace collection proposal 2: Send spans to collector directly from runtime/agent
+
+Send spans to collector directly from runtime/agent, this proposal need
+network accessible to the collector.
+
+**Pros:**
+
+- No additional trace forwarder component needed.
+
+**Cons:**
+
+- Need more code/configuration to support all trace collectors.
+
+## Future work
+
+- We could add dynamic and fully isolated tracing at a later stage,
+  if required.
+
+## Further details
+
+- See the new [GitHub project](https://github.com/orgs/kata-containers/projects/28).
+- [kata-containers-tracing-status](https://gist.github.com/jodh-intel/0ee54d41d2a803ba761e166136b42277) gist.
+- [tracing documentation PR][tracing-doc-pr].
+
+## Summary
+
+### Time line
+
+- 2021-07-01: A summary of the discussion was
+  [posted to the mail list](http://lists.katacontainers.io/pipermail/kata-dev/2021-July/001996.html).
+- 2021-06-22: These proposals were
+  [discussed in the Kata Architecture Committee meeting](https://etherpad.opendev.org/p/Kata_Containers_2021_Architecture_Committee_Mtgs).
+- 2021-06-18: These proposals where
+  [announced on the mailing list](http://lists.katacontainers.io/pipermail/kata-dev/2021-June/001980.html).
+
+### Outcome
+
+- Nobody opposed the agent proposals, so they are being implemented.
+- The trace collection proposals are still being considered.
+
+[kata-1x-tracing]: https://github.com/kata-containers/agent/blob/master/TRACING.md
+[trace-forwarder]: /src/trace-forwarder
+[tracing-doc-pr]: https://github.com/kata-containers/kata-containers/pull/1937

docs/design/vcpu-handling.md

@@ -1,11 +1,3 @@
-- [Virtual machine vCPU sizing in Kata Containers](#virtual-machine-vcpu-sizing-in-kata-containers)
-  * [Default number of virtual CPUs](#default-number-of-virtual-cpus)
-  * [Virtual CPUs and Kubernetes pods](#virtual-cpus-and-kubernetes-pods)
-  * [Container lifecycle](#container-lifecycle)
-  * [Container without CPU constraint](#container-without-cpu-constraint)
-  * [Container with CPU constraint](#container-with-cpu-constraint)
-  * [Do not waste resources](#do-not-waste-resources)
-
 # Virtual machine vCPU sizing in Kata Containers
 
 ## Default number of virtual CPUs

docs/design/virtualization.md

@@ -1,16 +1,5 @@
 # Virtualization in Kata Containers
 
-- [Virtualization in Kata Containers](#virtualization-in-kata-containers)
-  - [Mapping container concepts to virtual machine technologies](#mapping-container-concepts-to-virtual-machine-technologies)
-  - [Kata Containers Hypervisor and VMM support](#kata-containers-hypervisor-and-vmm-support)
-    - [QEMU/KVM](#qemukvm)
-      - [Machine accelerators](#machine-accelerators)
-      - [Hotplug devices](#hotplug-devices)
-    - [Firecracker/KVM](#firecrackerkvm)
-    - [Cloud Hypervisor/KVM](#cloud-hypervisorkvm)
-    - [Summary](#summary)
-
-
 Kata Containers, a second layer of isolation is created on top of those provided by traditional namespace-containers. The
 hardware virtualization interface is the basis of this additional layer. Kata will launch a lightweight virtual machine,
 and use the guest’s Linux kernel to create a container workload, or workloads in the case of multi-container pods. In Kubernetes

docs/how-to/README.md

@@ -1,15 +1,11 @@
 # Howto Guides
 
-* [Howto Guides](#howto-guides)
-  * [Kubernetes Integration](#kubernetes-integration)
-  * [Hypervisors Integration](#hypervisors-integration)
-  * [Advanced Topics](#advanced-topics)
-
 ## Kubernetes Integration
+
 - [Run Kata containers with `crictl`](run-kata-with-crictl.md)
 - [Run Kata Containers with Kubernetes](run-kata-with-k8s.md)
 - [How to use Kata Containers and Containerd](containerd-kata.md)
-- [How to use Kata Containers and CRI (containerd plugin) with Kubernetes](how-to-use-k8s-with-cri-containerd-and-kata.md)
+- [How to use Kata Containers and CRI (containerd) with Kubernetes](how-to-use-k8s-with-cri-containerd-and-kata.md)
 - [Kata Containers and service mesh for Kubernetes](service-mesh.md)
 - [How to import Kata Containers logs into Fluentd](how-to-import-kata-logs-with-fluentd.md)
 
@@ -21,13 +17,13 @@
 - `firecracker`
 - `ACRN`
 
-  While `qemu` and `cloud-hypervisor` work out of the box with installation of Kata,
-  some additional configuration is needed in case of `firecracker` and `ACRN`.
+  While `qemu` , `cloud-hypervisor` and `firecracker` work out of the box with installation of Kata,
+  some additional configuration is needed in case of `ACRN`.
   Refer to the following guides for additional configuration steps:
-- [Kata Containers with Firecracker](https://github.com/kata-containers/documentation/wiki/Initial-release-of-Kata-Containers-with-Firecracker-support)
 - [Kata Containers with ACRN Hypervisor](how-to-use-kata-containers-with-acrn.md)
 
 ## Advanced Topics
+
 - [How to use Kata Containers with virtio-fs](how-to-use-virtio-fs-with-kata.md)
 - [Setting Sysctls with Kata](how-to-use-sysctls-with-kata.md)
 - [What Is VMCache and How To Enable It](what-is-vm-cache-and-how-do-I-use-it.md)
@@ -38,3 +34,5 @@
 - [How to set sandbox Kata Containers configurations with pod annotations](how-to-set-sandbox-config-kata.md)
 - [How to monitor Kata Containers in K8s](how-to-set-prometheus-in-k8s.md)
 - [How to use hotplug memory on arm64 in Kata Containers](how-to-hotplug-memory-arm64.md)
+- [How to setup swap devices in guest kernel](how-to-setup-swap-devices-in-guest-kernel.md)
+- [How to run rootless vmm](how-to-run-rootless-vmm.md)

docs/how-to/containerd-kata.md

@@ -1,23 +1,5 @@
 # How to use Kata Containers and Containerd
 
-- [Concepts](#concepts)
-    - [Kubernetes `RuntimeClass`](#kubernetes-runtimeclass)
-    - [Containerd Runtime V2 API: Shim V2 API](#containerd-runtime-v2-api-shim-v2-api)
-- [Install](#install)
-    - [Install Kata Containers](#install-kata-containers)
-    - [Install containerd with CRI plugin](#install-containerd-with-cri-plugin)
-    - [Install CNI plugins](#install-cni-plugins)
-    - [Install `cri-tools`](#install-cri-tools)
-- [Configuration](#configuration)
-    - [Configure containerd to use Kata Containers](#configure-containerd-to-use-kata-containers)
-        - [Kata Containers as a `RuntimeClass`](#kata-containers-as-a-runtimeclass)
-        - [Kata Containers as the runtime for untrusted workload](#kata-containers-as-the-runtime-for-untrusted-workload)
-    - [Kata Containers as the default runtime](#kata-containers-as-the-default-runtime)
-    - [Configuration for `cri-tools`](#configuration-for-cri-tools)
-- [Run](#run)
-    - [Launch containers with `ctr` command line](#launch-containers-with-ctr-command-line)
-    - [Launch Pods with `crictl` command line](#launch-pods-with-crictl-command-line)
-
 This document covers the installation and configuration of [containerd](https://containerd.io/) 
 and [Kata Containers](https://katacontainers.io). The containerd provides not only the `ctr`
 command line tool, but also the [CRI](https://kubernetes.io/blog/2016/12/container-runtime-interface-cri-in-kubernetes/) 
@@ -57,7 +39,7 @@ use `RuntimeClass` instead of the deprecated annotations.
 
 ### Containerd Runtime V2 API: Shim V2 API
 
-The [`containerd-shim-kata-v2` (short as `shimv2` in this documentation)](../../src/runtime/containerd-shim-v2) 
+The [`containerd-shim-kata-v2` (short as `shimv2` in this documentation)](../../src/runtime/cmd/containerd-shim-kata-v2/)
 implements the [Containerd Runtime V2 (Shim API)](https://github.com/containerd/containerd/tree/master/runtime/v2) for Kata.
 With `shimv2`, Kubernetes can launch Pod and OCI-compatible containers with one shim per Pod. Prior to `shimv2`, `2N+1` 
 shims (i.e. a `containerd-shim` and a `kata-shim` for each container and the Pod sandbox itself) and no standalone `kata-proxy` 

docs/how-to/data/kata-monitor-daemonset.yml

@@ -26,7 +26,7 @@ spec:
       hostNetwork: true
       containers:
       - name: kata-monitor
-        image: docker.io/katadocker/kata-monitor:2.0.0
+        image: quay.io/kata-containers/kata-monitor:2.0.0
         args: 
           - -log-level=debug
         ports:

docs/how-to/how-to-hotplug-memory-arm64.md

@@ -1,9 +1,5 @@
 # How to use memory hotplug feature in Kata Containers on arm64
 
-- [Introduction](#introduction)
-- [Install UEFI ROM](#install-uefi-rom)
-- [Run for test](#run-for-test)
-
 ## Introduction
 
 Memory hotplug is a key feature for containers to allocate memory dynamically in deployment.

docs/how-to/how-to-import-kata-logs-with-fluentd.md

@@ -1,20 +1,5 @@
 # Importing Kata Containers logs with Fluentd
 
-* [Introduction](#introduction)
-* [Overview](#overview)
-    * [Test stack](#test-stack)
-    * [Importing the logs](#importing-the-logs)
-    * [Direct import `logfmt` from `systemd`](#direct-import-logfmt-from-systemd)
-        * [Configuring `minikube`](#configuring-minikube)
-        * [Pull from `systemd`](#pull-from-systemd)
-        * [Systemd Summary](#systemd-summary)
-    * [Directly importing JSON](#directly-importing-json)
-        * [JSON in files](#json-in-files)
-            * [Prefixing all keys](#prefixing-all-keys)
-* [Kata `shimv2`](#kata-shimv2)
-* [Caveats](#caveats)
-* [Summary](#summary)
-
 # Introduction
 
 This document describes how to import Kata Containers logs into [Fluentd](https://www.fluentd.org/),
@@ -143,7 +128,7 @@ YAML can be found
   tag kata-containers
   path /run/log/journal
   pos_file /run/log/journal/kata-journald.pos
-  filters [{"SYSLOG_IDENTIFIER": "kata-runtime"}, {"SYSLOG_IDENTIFIER": "kata-proxy"}, {"SYSLOG_IDENTIFIER": "kata-shim"}]
+  filters [{"SYSLOG_IDENTIFIER": "kata-runtime"}, {"SYSLOG_IDENTIFIER": "kata-shim"}]
   read_from_head true
 </source>
 ```
@@ -161,7 +146,7 @@ generate some Kata specific log entries:
 
 ```bash
 $ minikube addons open efk
-$ cd $GOPATH/src/github.com/kata-containers/packaging/kata-deploy
+$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/packaging/kata-deploy
 $ kubectl apply -f examples/nginx-deployment-qemu.yaml
 ```
 
@@ -178,7 +163,7 @@ sub-filter on, for instance, the `SYSLOG_IDENTIFIER` to differentiate the Kata c
 on the `PRIORITY` to filter out critical issues etc.
 
 Kata generates a significant amount of Kata specific information, which can be seen as
-[`logfmt`](https://github.com/kata-containers/tests/tree/master/cmd/log-parser#logfile-requirements).
+[`logfmt`](https://github.com/kata-containers/tests/tree/main/cmd/log-parser#logfile-requirements).
 data contained in the `MESSAGE` field. Imported as-is, there is no easy way to filter on that data
 in Kibana:
 
@@ -272,9 +257,8 @@ go directly to a full Kata specific JSON format logfile test.
 
 Kata runtime has the ability to generate JSON logs directly, rather than its default `logfmt` format. Passing
 the `--log-format=json` argument to the Kata runtime enables this. The easiest way to pass in this extra
-parameter from a [Kata deploy](https://github.com/kata-containers/packaging/tree/master/kata-deploy) installation
-is to edit the `/opt/kata/bin/kata-qemu` shell script (generated by the
-[Kata packaging release scripts](https://github.com/kata-containers/packaging/blob/master/release/kata-deploy-binaries.sh)).
+parameter from a [Kata deploy](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy) installation
+is to edit the `/opt/kata/bin/kata-qemu` shell script.
 
 At the same time, we will add the `--log=/var/log/kata-runtime.log` argument to store the Kata logs in their
 own file (rather than into the system journal).

docs/how-to/how-to-run-rootless-vmm.md

@@ -0,0 +1,33 @@
+## Introduction
+To improve security, Kata Container supports running the VMM process (currently only QEMU) as a non-`root` user.
+This document describes how to enable the rootless VMM mode and its limitations.
+
+## Pre-requisites
+The permission and ownership of the `kvm` device node (`/dev/kvm`) need to be configured to:
+```
+$ crw-rw---- 1 root kvm
+```
+use the following commands:
+```
+$ sudo groupadd kvm -r
+$ sudo chown root:kvm /dev/kvm
+$ sudo chmod 660 /dev/kvm
+```
+
+## Configure rootless VMM
+By default, the VMM process still runs as the root user. There are two ways to enable rootless VMM:
+1. Set the `rootless` flag to `true` in the hypervisor section of `configuration.toml`.
+2. Set the Kubernetes annotation `io.katacontainers.hypervisor.rootless` to `true`.
+
+## Implementation details
+When `rootless` flag is enabled, upon a request to create a Pod, Kata Containers runtime creates a random user and group (e.g. `kata-123`), and uses them to start the hypervisor process. 
+The `kvm` group is also given to the hypervisor process as a supplemental group to give the hypervisor process access to the `/dev/kvm` device. 
+Another necessary change is to move the hypervisor runtime files (e.g. `vhost-fs.sock`, `qmp.sock`) to a directory (under `/run/user/[uid]/`) where only the non-root hypervisor has access to.
+
+## Limitations
+
+1. Only the VMM process is running as a non-root user. Other processes such as Kata Container shimv2 and `virtiofsd` still run as the root user.
+2. Currently, this feature is only supported in QEMU. Still need to bring it to Firecracker and Cloud Hypervisor (see https://github.com/kata-containers/kata-containers/issues/2567).
+3. Certain features will not work when rootless VMM is enabled, including:
+   1. Passing devices to the guest (`virtio-blk`, `virtio-scsi`) will not work if the non-privileged user does not have permission to access it (leading to a permission denied error). A more permissive permission (e.g. 666) may overcome this issue. However, you need to be aware of the potential security implications of reducing the security on such devices.
+   2. `vfio` device will also not work because of permission denied error.

docs/how-to/how-to-set-prometheus-in-k8s.md

@@ -2,14 +2,6 @@
 
 This document describes how to run `kata-monitor` in a Kubernetes cluster using Prometheus's service discovery to scrape metrics from `kata-agent`.
 
-- [Introduction](#introduction)
-- [Pre-requisites](#pre-requisites)
-- [Configure Prometheus](#configure-prometheus)
-- [Configure `kata-monitor`](#configure-kata-monitor)
-- [Setup Grafana](#setup-grafana)
-  * [Create `datasource`](#create-datasource)
-  * [Import dashboard](#import-dashboard)
-
 > **Warning**: This how-to is only for evaluation purpose, you **SHOULD NOT** running it in production using this configurations.
 
 ## Introduction

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -34,8 +34,6 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.agent.enable_tracing` | `boolean` | enable tracing for the agent |
 | `io.katacontainers.config.agent.container_pipe_size` | uint32 | specify the size of the std(in/out) pipes created for containers |
 | `io.katacontainers.config.agent.kernel_modules` | string | the list of kernel modules and their parameters that will be loaded in the guest kernel. Semicolon separated list of kernel modules and their parameters. These modules will be loaded in the guest kernel using `modprobe`(8). E.g., `e1000e InterruptThrottleRate=3000,3000,3000 EEE=1; i915 enable_ppgtt=0` |
-| `io.katacontainers.config.agent.trace_mode` | string | the trace mode for the agent |
-| `io.katacontainers.config.agent.trace_type` | string | the trace type for the agent |
 
 ## Hypervisor Options
 | Key | Value Type | Comments |
@@ -91,6 +89,13 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.virtio_fs_cache` | string | the cache mode for virtio-fs, valid values are `always`, `auto` and `none` |
 | `io.katacontainers.config.hypervisor.virtio_fs_daemon` | string | virtio-fs `vhost-user` daemon path |
 | `io.katacontainers.config.hypervisor.virtio_fs_extra_args` | string | extra options passed to `virtiofs` daemon |
+| `io.katacontainers.config.hypervisor.enable_guest_swap` | `boolean` | enable swap in the guest |
+
+## Container Options
+| Key | Value Type | Comments |
+|-------| ----- | ----- |
+| `io.katacontainers.container.resource.swappiness"` | `uint64` | specify the `Resources.Memory.Swappiness` |
+| `io.katacontainers.container.resource.swap_in_bytes"` | `uint64` | specify the `Resources.Memory.Swap` |
 
 # CRI-O Configuration
 
@@ -100,11 +105,12 @@ In case of CRI-O, all annotations specified in the pod spec are passed down to K
 
 For containerd, annotations specified in the pod spec are passed down to Kata
 starting with version `1.3.0` of containerd. Additionally, extra configuration is
-needed for containerd, by providing a `pod_annotations` field in the containerd config
-file.  The `pod_annotations` field is a list of annotations that can be passed down to
-Kata as OCI annotations. It supports golang match patterns. Since annotations supported
-by Kata follow the pattern `io.katacontainers.*`, the following configuration would work
-for passing annotations to Kata from containerd:
+needed for containerd, by providing `pod_annotations` field and
+`container_annotations` field in the containerd config
+file.  The `pod_annotations` field and `container_annotations` field are two lists of
+annotations that can be passed down to Kata as OCI annotations. They support golang match
+patterns. Since annotations supported by Kata follow the pattern `io.katacontainers.*`,
+the following configuration would work for passing annotations to Kata from containerd:
 
 ```
 $ cat /etc/containerd/config
@@ -113,6 +119,7 @@ $ cat /etc/containerd/config
          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
            runtime_type = "io.containerd.kata.v2"
            pod_annotations = ["io.katacontainers.*"]
+           container_annotations = ["io.katacontainers.*"]
 ....
 
 ```

docs/how-to/how-to-setup-swap-devices-in-guest-kernel.md

@@ -0,0 +1,59 @@
+# Setup swap device in guest kernel
+
+## Introduction
+
+Setup swap device in guest kernel can help to increase memory capacity, handle some memory issues and increase file access speed sometimes.
+Kata Containers can insert a raw file to the guest as the swap device.
+
+## Requisites
+
+The swap config of the containers should be set by [annotations](how-to-set-sandbox-config-kata.md#container-options).  So [extra configuration is needed for containerd](how-to-set-sandbox-config-kata.md#containerd-configuration).
+
+Kata Containers just supports setup swap device in guest kernel with QEMU.
+Install and setup Kata Containers as shown [here](../install/README.md).
+
+Enable setup swap device in guest kernel as follows:
+```
+$ sudo sed -i -e 's/^#enable_guest_swap.*$/enable_guest_swap = true/g' /etc/kata-containers/configuration.toml
+```
+
+## Run a Kata Container utilizing swap device
+
+Use following command to start a Kata Container with swappiness 60 and 1GB swap device (swap_in_bytes - memory_limit_in_bytes).
+```
+$ pod_yaml=pod.yaml
+$ container_yaml=container.yaml
+$ image="quay.io/prometheus/busybox:latest"
+$ cat << EOF > "${pod_yaml}"
+metadata:
+  name: busybox-sandbox1
+EOF
+$ cat << EOF > "${container_yaml}"
+metadata:
+  name: busybox-test-swap
+annotations:
+  io.katacontainers.container.resource.swappiness: "60"
+  io.katacontainers.container.resource.swap_in_bytes: "2147483648"
+linux:
+  resources:
+    memory_limit_in_bytes: 1073741824
+image:
+  image: "$image"
+command:
+- top
+EOF
+$ sudo crictl pull $image
+$ podid=$(sudo crictl runp $pod_yaml)
+$ cid=$(sudo crictl create $podid $container_yaml $pod_yaml)
+$ sudo crictl start $cid
+```
+
+Kata Container setups swap device for this container only when `io.katacontainers.container.resource.swappiness` is set.
+
+The following table shows the swap size how to decide if `io.katacontainers.container.resource.swappiness` is set.
+|`io.katacontainers.container.resource.swap_in_bytes`|`memory_limit_in_bytes`|swap size|
+|---|---|---|
+|set|set| `io.katacontainers.container.resource.swap_in_bytes` - `memory_limit_in_bytes`|
+|not set|set| `memory_limit_in_bytes`|
+|not set|not set| `io.katacontainers.config.hypervisor.default_memory`|
+|set|not set|cgroup doesn't support this usage|

docs/how-to/how-to-use-k8s-with-cri-containerd-and-kata.md

@@ -1,22 +1,9 @@
 # How to use Kata Containers and CRI (containerd plugin) with Kubernetes
 
-* [Requirements](#requirements)
-* [Install and configure containerd](#install-and-configure-containerd)
-* [Install and configure Kubernetes](#install-and-configure-kubernetes)
-    * [Install Kubernetes](#install-kubernetes)
-    * [Configure Kubelet to use containerd](#configure-kubelet-to-use-containerd)
-    * [Configure HTTP proxy - OPTIONAL](#configure-http-proxy---optional)
-* [Start Kubernetes](#start-kubernetes)
-* [Configure Pod Network](#configure-pod-network)
-* [Allow pods to run in the master node](#allow-pods-to-run-in-the-master-node)
-* [Create runtime class for Kata Containers](#create-runtime-class-for-kata-containers)
-* [Run pod in Kata Containers](#run-pod-in-kata-containers)
-* [Delete created pod](#delete-created-pod)
-
 This document describes how to set up a single-machine Kubernetes (k8s) cluster.
 
 The Kubernetes cluster will use the
-[CRI containerd plugin](https://github.com/containerd/cri) and
+[CRI containerd](https://github.com/containerd/containerd/) and
 [Kata Containers](https://katacontainers.io) to launch untrusted workloads.
 
 ## Requirements
@@ -84,12 +71,12 @@ $ for service in ${services}; do
     service_dir="/etc/systemd/system/${service}.service.d/"
     sudo mkdir -p ${service_dir}
 
-    cat << EOT | sudo tee "${service_dir}/proxy.conf"
+    cat << EOF | sudo tee "${service_dir}/proxy.conf"
 [Service]
 Environment="HTTP_PROXY=${http_proxy}"
 Environment="HTTPS_PROXY=${https_proxy}"
 Environment="NO_PROXY=${no_proxy}"
-EOT
+EOF
 done
 
 $ sudo systemctl daemon-reload
@@ -185,7 +172,7 @@ If a pod has the `runtimeClassName` set to `kata`, the CRI plugin runs the pod w
 - Create an pod configuration that using Kata Containers runtime
 
   ```bash
-  $ cat << EOT | tee nginx-kata.yaml
+  $ cat << EOF | tee nginx-kata.yaml
   apiVersion: v1
   kind: Pod
   metadata:
@@ -196,7 +183,7 @@ If a pod has the `runtimeClassName` set to `kata`, the CRI plugin runs the pod w
     - name: nginx
       image: nginx
       
-  EOT
+  EOF
   ```
 
 - Create the pod

docs/how-to/how-to-use-kata-containers-with-acrn.md

@@ -2,11 +2,6 @@
 
 This document provides an overview on how to run Kata containers with ACRN hypervisor and device model.
 
-- [Introduction](#introduction)
-- [Pre-requisites](#pre-requisites)
-- [Configure Docker](#configure-docker)
-- [Configure Kata Containers with ACRN](#configure-kata-containers-with-acrn)
-
 ## Introduction
 
 ACRN is a flexible, lightweight Type-1 reference hypervisor built with real-time and safety-criticality in mind. ACRN uses an open source platform making it optimized to streamline embedded development.
@@ -27,7 +22,7 @@ This document requires the presence of the ACRN hypervisor and Kata Containers o
 
 - ACRN supported [Hardware](https://projectacrn.github.io/latest/hardware.html#supported-hardware).
   > **Note:** Please make sure to have a minimum of 4 logical processors (HT) or cores.
-- ACRN [software](https://projectacrn.github.io/latest/tutorials/kbl-nuc-sdc.html#use-the-script-to-set-up-acrn-automatically) setup.
+- ACRN [software](https://projectacrn.github.io/latest/tutorials/run_kata_containers.html) setup.
 - For networking, ACRN supports either MACVTAP or TAP. If MACVTAP is not enabled in the Service OS, please follow the below steps to update the kernel:
 
   ```sh

docs/how-to/how-to-use-sysctls-with-kata.md

@@ -1,6 +1,7 @@
 # Setting Sysctls with Kata
 
 ## Sysctls
+
 In Linux, the sysctl interface allows an administrator to modify kernel 
 parameters at runtime. Parameters are available via the `/proc/sys/` virtual 
 process file system. 

docs/how-to/how-to-use-virtio-fs-with-kata.md

@@ -1,8 +1,5 @@
 # Kata Containers with virtio-fs
 
-- [Kata Containers with virtio-fs](#kata-containers-with-virtio-fs)
-  - [Introduction](#introduction)
-
 ## Introduction
 
 Container deployments utilize explicit or implicit file sharing between host filesystem and containers. From a trust perspective, avoiding a shared file-system between the trusted host and untrusted container is recommended. This is not always feasible. In Kata Containers, block-based volumes are preferred as they allow usage of either device pass through or `virtio-blk` for access within the virtual machine.

docs/how-to/how-to-use-virtio-mem-with-kata.md

@@ -1,9 +1,5 @@
 # Kata Containers with `virtio-mem`
 
-- [Introduction](#introduction)
-- [Requisites](#requisites)
-- [Run a Kata Container utilizing `virtio-mem`](#run-a-kata-container-utilizing-virtio-mem)
-
 ## Introduction
 
 The basic idea of `virtio-mem` is to provide a flexible, cross-architecture memory hot plug and hot unplug solution that avoids many limitations imposed by existing technologies, architectures, and interfaces.
@@ -41,7 +37,7 @@ $ echo 1 | sudo tee /proc/sys/vm/overcommit_memory
 Use following command to start a Kata Container.
 ```
 $ pod_yaml=pod.yaml
-$ container_yaml=${REPORT_DIR}/container.yaml
+$ container_yaml=container.yaml
 $ image="quay.io/prometheus/busybox:latest"
 $ cat << EOF > "${pod_yaml}"
 metadata:

docs/how-to/privileged.md

@@ -3,11 +3,6 @@
 Kata Containers supports creation of containers that are "privileged" (i.e. have additional capabilities and access
 that is not normally granted).
 
-* [Warnings](#warnings)
-    * [Host Devices](#host-devices)
-        * [Containerd and CRI](#containerd-and-cri)
-        * [CRI-O](#cri-o)
-
 ## Warnings
 
 **Warning:** Whilst this functionality is supported, it can decrease the security of Kata Containers if not configured 
@@ -21,9 +16,9 @@ from the host, a potentially undesirable side-effect that decreases the security
 
 The following sections document how to configure this behavior in different container runtimes.
 
-#### Containerd and CRI
+#### Containerd
 
-The Containerd CRI allows configuring the privileged host devices behavior for each runtime in the CRI config. This is
+The Containerd allows configuring the privileged host devices behavior for each runtime in the containerd config. This is
 done with the `privileged_without_host_devices` option. Setting this to `true` will disable hot plugging of the host 
 devices into the guest, even when privileged is enabled.
 
@@ -46,7 +41,7 @@ See below example config:
 ```
 
  - [Kata Containers with Containerd and CRI documentation](how-to-use-k8s-with-cri-containerd-and-kata.md)
- - [Containerd CRI config documentation](https://github.com/containerd/cri/blob/master/docs/config.md)
+ - [Containerd CRI config documentation](https://github.com/containerd/containerd/blob/main/docs/cri/config.md)
 
 #### CRI-O
 

docs/how-to/run-kata-with-crictl.md

@@ -1,16 +1,5 @@
 # Working with `crictl`
 
-* [What's `cri-tools`](#whats-cri-tools)
-* [Use `crictl` run Pods in Kata containers](#use-crictl-run-pods-in-kata-containers)
-  * [Run `busybox` Pod](#run-busybox-pod)
-    * [Run pod sandbox with config file](#run-pod-sandbox-with-config-file)
-    * [Create container in the pod sandbox with config file](#create-container-in-the-pod-sandbox-with-config-file)
-    * [Start container](#start-container)
-  * [Run `redis` Pod](#run-redis-pod)
-    * [Create `redis-server` Pod](#create-redis-server-pod)
-    * [Create `redis-client` Pod](#create-redis-client-pod)
-    * [Check `redis` server is working](#check-redis-server-is-working)
-
 ## What's `cri-tools`
 
 [`cri-tools`](https://github.com/kubernetes-sigs/cri-tools) provides debugging and validation tools for Kubelet Container Runtime Interface (CRI).

docs/how-to/run-kata-with-k8s.md

@@ -1,18 +1,5 @@
 # Run Kata Containers with Kubernetes
 
-* [Run Kata Containers with Kubernetes](#run-kata-containers-with-kubernetes)
-  * [Prerequisites](#prerequisites)
-  * [Install a CRI implementation](#install-a-cri-implementation)
-     * [CRI-O](#cri-o)
-        * [Kubernetes Runtime Class (CRI-O v1.12 )](#kubernetes-runtime-class-cri-o-v112)
-        * [Untrusted annotation (until CRI-O v1.12)](#untrusted-annotation-until-cri-o-v112)
-        * [Network namespace management](#network-namespace-management)
-     * [containerd with CRI plugin](#containerd-with-cri-plugin)
-  * [Install Kubernetes](#install-kubernetes)
-     * [Configure for CRI-O](#configure-for-cri-o)
-     * [Configure for containerd](#configure-for-containerd)
-  * [Run a Kubernetes pod with Kata Containers](#run-a-kubernetes-pod-with-kata-containers)
-
 ## Prerequisites
 This guide requires Kata Containers available on your system, install-able by following [this guide](../install/README.md).
 
@@ -22,7 +9,7 @@ Kubernetes CRI (Container Runtime Interface) implementations allow using any
 OCI-compatible runtime with Kubernetes, such as the Kata Containers runtime.
 
 Kata Containers support both the [CRI-O](https://github.com/kubernetes-incubator/cri-o) and
-[CRI-containerd](https://github.com/containerd/cri) CRI implementations.
+[containerd](https://github.com/containerd/containerd) CRI implementations.
 
 After choosing one CRI implementation, you must make the appropriate configuration
 to ensure it integrates with Kata Containers.
@@ -33,7 +20,7 @@ required to spawn pods and containers, and this is the preferred way to run Kata
 An equivalent shim implementation for CRI-O is planned.
 
 ### CRI-O
-For CRI-O installation instructions, refer to the [CRI-O Tutorial](https://github.com/kubernetes-incubator/cri-o/blob/master/tutorial.md) page.
+For CRI-O installation instructions, refer to the [CRI-O Tutorial](https://github.com/cri-o/cri-o/blob/main/tutorial.md) page.
 
 The following sections show how to set up the CRI-O configuration file (default path: `/etc/crio/crio.conf`) for Kata.
 
@@ -43,7 +30,7 @@ Unless otherwise stated, all the following settings are specific to the `crio.ru
 # runtime used and options for how to set up and manage the OCI runtime.
 [crio.runtime]
 ```
-A comprehensive documentation of the configuration file can be found [here](https://github.com/cri-o/cri-o/blob/master/docs/crio.conf.5.md).
+A comprehensive documentation of the configuration file can be found [here](https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md).
 
 > **Note**: After any change to this file, the CRI-O daemon have to be restarted with:
 >````
@@ -124,11 +111,7 @@ manage_ns_lifecycle = true
 ```
 
 
-### containerd with CRI plugin
-
-If you select containerd with `cri` plugin, follow the "Getting Started for Developers"
-instructions [here](https://github.com/containerd/cri#getting-started-for-developers)
-to properly install it.
+### containerd
 
 To customize containerd to select Kata Containers runtime, follow our
 "Configure containerd to use Kata Containers" internal documentation
@@ -173,7 +156,7 @@ $ sudo systemctl restart kubelet
 # If using CRI-O
 $ sudo kubeadm init --ignore-preflight-errors=all --cri-socket /var/run/crio/crio.sock --pod-network-cidr=10.244.0.0/16
 
-# If using CRI-containerd
+# If using containerd
 $ sudo kubeadm init --ignore-preflight-errors=all --cri-socket /run/containerd/containerd.sock --pod-network-cidr=10.244.0.0/16
 
 $ export KUBECONFIG=/etc/kubernetes/admin.conf

docs/how-to/service-mesh.md

@@ -1,21 +1,5 @@
 # Kata Containers and service mesh for Kubernetes
 
-* [Assumptions](#assumptions)
-* [How they work](#how-they-work)
-* [Prerequisites](#prerequisites)
-    * [Kata and Kubernetes](#kata-and-kubernetes)
-    * [Restrictions](#restrictions)
-* [Install and deploy your service mesh](#install-and-deploy-your-service-mesh)
-    * [Service Mesh Istio](#service-mesh-istio)
-    * [Service Mesh Linkerd](#service-mesh-linkerd)
-* [Inject your services with sidecars](#inject-your-services-with-sidecars)
-    * [Sidecar Istio](#sidecar-istio)
-    * [Sidecar Linkerd](#sidecar-linkerd)
-* [Run your services with Kata](#run-your-services-with-kata)
-    * [Lower privileges](#lower-privileges)
-    * [Add annotations](#add-annotations)
-    * [Deploy](#deploy)
-
 A service mesh is a way to monitor and control the traffic between
 micro-services running in your Kubernetes cluster. It is a powerful
 tool that you might want to use in combination with the security
@@ -50,7 +34,7 @@ as the proxy starts.
 
 Follow the [instructions](../install/README.md)
 to get Kata Containers properly installed and configured with Kubernetes.
-You can choose between CRI-O and CRI-containerd, both are supported
+You can choose between CRI-O and containerd, both are supported
 through this document.
 
 For both cases, select the workloads as _trusted_ by default. This way,
@@ -76,15 +60,16 @@ is not able to perform a proper setup of the rules.
 
 ### Service Mesh Istio
 
-As a reference, you can follow Istio [instructions](https://istio.io/docs/setup/kubernetes/quick-start/#download-and-prepare-for-the-installation).
-
 The following is a summary of what you need to install Istio on your system:
+
 ```
 $ curl -L https://git.io/getLatestIstio | sh -
 $ cd istio-*
 $ export PATH=$PWD/bin:$PATH
 ```
 
+See the [Istio documentation](https://istio.io/docs) for further details.
+
 Now deploy Istio in the control plane of your cluster with the following:
 ```
 $ kubectl apply -f install/kubernetes/istio-demo.yaml
@@ -174,7 +159,7 @@ containers with `privileged: true` to `privileged: false`.
 There is no difference between Istio and Linkerd in this section. It is
 about which CRI implementation you use.
 
-For both CRI-O and CRI-containerd, you have to add an annotation indicating
+For both CRI-O and containerd, you have to add an annotation indicating
 the workload for this deployment is not _trusted_, which will trigger
 `kata-runtime` to be called instead of `runc`.
 
@@ -208,9 +193,9 @@ spec:
 ...
 ```
 
-__CRI-containerd:__
+__containerd:__
 
-Add the following annotation for CRI-containerd
+Add the following annotation for containerd
 ```yaml
 io.kubernetes.cri.untrusted-workload: "true"
 ```

docs/how-to/what-is-vm-cache-and-how-do-I-use-it.md

@@ -1,10 +1,5 @@
 # What Is VMCache and How To Enable It
 
-* [What is VMCache](#what-is-vmcache)
-* [How is this different to VM templating](#how-is-this-different-to-vm-templating)
-* [How to enable VMCache](#how-to-enable-vmcache)
-* [Limitations](#limitations)
-
 ### What is VMCache
 
 VMCache is a new function that creates VMs as caches before using it.

docs/how-to/what-is-vm-templating-and-how-do-I-use-it.md

@@ -1,6 +1,7 @@
 # What Is VM Templating and How To Enable It
 
 ### What is VM templating
+
 VM templating is a Kata Containers feature that enables new VM
 creation using a cloning technique. When enabled, new VMs are created
 by cloning from a pre-created template VM, and they will share the
@@ -8,11 +9,13 @@ same initramfs, kernel and agent memory in readonly mode. It is very
 much like a process fork done by the kernel but here we *fork* VMs.
 
 ### How is this different from VMCache
+
 Both [VMCache](../how-to/what-is-vm-cache-and-how-do-I-use-it.md) and VM templating help speed up new container creation.  
 When VMCache enabled, new VMs are created by the VMCache server.  So it is not vulnerable to share memory CVE because each VM doesn't share the memory.  
 VM templating saves a lot of memory if there are many Kata Containers running on the same host.
 
 ### What are the Pros
+
 VM templating helps speed up new container creation and saves a lot
 of memory if there are many Kata Containers running on the same host.
 If you are running a density workload, or care a lot about container
@@ -29,6 +32,7 @@ showed that VM templating speeds up Kata Containers creation by as much as
 38.68%. See [full results here](https://gist.github.com/bergwolf/06974a3c5981494a40e2c408681c085d).
 
 ### What are the Cons
+
 One drawback of VM templating is that it cannot avoid cross-VM side-channel
 attack such as [CVE-2015-2877](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2015-2877)
 that originally targeted at the Linux KSM feature.
@@ -39,10 +43,11 @@ and can be classified as potentially misunderstood behaviors rather than vulnera
 **Warning**: If you care about such attack vector, do not use VM templating or KSM.
 
 ### How to enable VM templating
+
 VM templating can be enabled by changing your Kata Containers config file (`/usr/share/defaults/kata-containers/configuration.toml`,
 overridden by `/etc/kata-containers/configuration.toml` if provided) such that:
 
-  - `qemu-lite` is specified in `hypervisor.qemu`->`path` section
+  - `qemu` version `v4.1.0` or above is specified in `hypervisor.qemu`->`path` section
   - `enable_template = true`
   - `initrd =` is set
   - `image =` option is commented out or removed

docs/hypervisors.md

@@ -1,11 +1,5 @@
 # Hypervisors
 
-* [Hypervisors](#hypervisors)
-    * [Introduction](#introduction)
-    * [Types](#types)
-    * [Determine currently configured hypervisor](#determine-currently-configured-hypervisor)
-    * [Choose a Hypervisor](#choose-a-hypervisor)
-
 ## Introduction
 
 Kata Containers supports multiple hypervisors. This document provides a very

docs/install/README.md

@@ -1,39 +1,19 @@
-# Kata Containers installation user guides
+# Kata Containers installation guides
 
-* [Kata Containers installation user guides](#kata-containers-installation-user-guides)
-    * [Prerequisites](#prerequisites)
-    * [Legacy installation](#legacy-installation)
-    * [Packaged installation methods](#packaged-installation-methods)
-        * [Official packages](#official-packages)
-        * [Snap Installation](#snap-installation)
-        * [Automatic Installation](#automatic-installation)
-        * [Manual Installation](#manual-installation)
-    * [Build from source installation](#build-from-source-installation)
-    * [Installing on a Cloud Service Platform](#installing-on-a-cloud-service-platform)
-    * [Further information](#further-information)
-
-The following is an overview of the different installation methods available. All of these methods equally result
-in a system configured to run Kata Containers.
+The following is an overview of the different installation methods available. 
 
 ## Prerequisites
 
-Kata Containers requires nested virtualization or bare metal.
-See the
-[hardware requirements](/src/runtime/README.md#hardware-requirements)
-to see if your system is capable of running Kata Containers.
-
-## Legacy installation
-
-If you wish to install a legacy 1.x version of Kata Containers, see
-[the Kata Containers 1.x installation documentation](https://github.com/kata-containers/documentation/tree/master/install/).
+Kata Containers requires nested virtualization or bare metal. Check 
+[hardware requirements](/src/runtime/README.md#hardware-requirements) to see if your system is capable of running Kata 
+Containers.
 
 ## Packaged installation methods
 
-> **Notes:**
->
-> - Packaged installation methods uses your distribution's native package format (such as RPM or DEB).
-> - You are strongly encouraged to choose an installation method that provides
->   automatic updates, to ensure you benefit from security updates and bug fixes.
+Packaged installation methods uses your distribution's native package format (such as RPM or DEB).
+
+*Note:* We encourage installation methods that provides automatic updates, it ensures security updates and bug fixes are
+easily applied.
 
 | Installation method                                  | Description                                                         | Automatic updates | Use case                                                 |
 |------------------------------------------------------|---------------------------------------------------------------------|-------------------|----------------------------------------------------------|
@@ -52,16 +32,9 @@ Kata packages are provided by official distribution repositories for:
 | [CentOS](centos-installation-guide.md)                   | 8                                                                              |
 | [Fedora](fedora-installation-guide.md)                   | 34                                                                             |
 
-> **Note::**
->
-> All users are encouraged to uses the official distribution versions of Kata
-> Containers unless they understand the implications of alternative methods.
-
 ### Snap Installation
 
-> **Note:** The snap installation is available for all distributions which support `snapd`.
-
-[![Get it from the Snap Store](https://snapcraft.io/static/images/badges/en/snap-store-black.svg)](https://snapcraft.io/kata-containers)
+The snap installation is available for all distributions which support `snapd`.
 
 [Use snap](snap-installation-guide.md) to install Kata Containers from https://snapcraft.io.
 
@@ -75,11 +48,9 @@ Follow the [containerd installation guide](container-manager/containerd/containe
 
 ## Build from source installation
 
-> **Notes:**
->
-> - Power users who decide to build from sources should be aware of the
->   implications of using an unpackaged system which will not be automatically
->   updated as new [releases](../Stable-Branch-Strategy.md) are made available.
+*Note:* Power users who decide to build from sources should be aware of the
+implications of using an unpackaged system which will not be automatically
+updated as new [releases](../Stable-Branch-Strategy.md) are made available.
 
 [Building from sources](../Developer-Guide.md#initial-setup)  allows power users
 who are comfortable building software from source to use the latest component
@@ -95,6 +66,6 @@ versions. This is not recommended for normal users.
 
 ## Further information
 
-* The [upgrading document](../Upgrading.md).
-* The [developer guide](../Developer-Guide.md).
-* The [runtime documentation](../../src/runtime/README.md).
+* [upgrading document](../Upgrading.md)
+* [developer guide](../Developer-Guide.md)
+* [runtime documentation](../../src/runtime/README.md)

docs/install/aws-installation-guide.md

@@ -1,10 +1,5 @@
 # Install Kata Containers on Amazon Web Services
 
-* [Install and Configure AWS CLI](#install-and-configure-aws-cli)
-* [Create or Import an EC2 SSH key pair](#create-or-import-an-ec2-ssh-key-pair)
-* [Launch i3.metal instance](#launch-i3metal-instance)
-* [Install Kata](#install-kata)
-
 Kata Containers on Amazon Web Services (AWS) makes use of [i3.metal](https://aws.amazon.com/ec2/instance-types/i3/) instances. Most of the installation procedure is identical to that for Kata on your preferred distribution, except that you have to run it on bare metal instances since AWS doesn't support nested virtualization yet. This guide walks you through creating an i3.metal instance.
 
 ## Install and Configure AWS CLI

docs/install/container-manager/containerd/containerd-install.md

@@ -98,12 +98,12 @@
 
     ```toml
     [plugins]
-        [plugins.cri]
-            [plugins.cri.containerd]
-            default_runtime_name = "kata"
-
-            [plugins.cri.containerd.runtimes.kata]
-            runtime_type = "io.containerd.kata.v2"
+      [plugins."io.containerd.grpc.v1.cri"]
+        [plugins."io.containerd.grpc.v1.cri".containerd]
+          default_runtime_name = "kata"
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
+              runtime_type = "io.containerd.kata.v2"
     ```
 
     > **Note:**

docs/install/gce-installation-guide.md

@@ -1,11 +1,5 @@
 # Install Kata Containers on Google Compute Engine
 
-* [Create an Image with Nested Virtualization Enabled](#create-an-image-with-nested-virtualization-enabled)
-    * [Create the Image](#create-the-image)
-    * [Verify VMX is Available](#verify-vmx-is-available)
-* [Install Kata](#install-kata)
-* [Create a Kata-enabled Image](#create-a-kata-enabled-image)
-
 Kata Containers on Google Compute Engine (GCE) makes use of [nested virtualization](https://cloud.google.com/compute/docs/instances/enable-nested-virtualization-vm-instances). Most of the installation procedure is identical to that for Kata on your preferred distribution, but enabling nested virtualization currently requires extra steps on GCE. This guide walks you through creating an image and instance with nested virtualization enabled. Note that `kata-runtime check` checks for nested virtualization, but does not fail if support is not found.
 
 As a pre-requisite this guide assumes an installed and configured instance of the [Google Cloud SDK](https://cloud.google.com/sdk/downloads). For a zero-configuration option, all of the commands below were been tested under [Google Cloud Shell](https://cloud.google.com/shell/) (as of Jun 2018). Verify your `gcloud` installation and configuration:

docs/install/minikube-installation-guide.md

@@ -1,17 +1,5 @@
 # Installing Kata Containers in Minikube
 
-* [Installing Kata Containers in Minikube](#installing-kata-containers-in-minikube)
-    * [Introduction](#introduction)
-    * [Prerequisites](#prerequisites)
-    * [Setting up Minikube](#setting-up-minikube)
-    * [Checking for nested virtualization](#checking-for-nested-virtualization)
-    * [Check Minikube is running](#check-minikube-is-running)
-    * [Installing Kata Containers](#installing-kata-containers)
-    * [Enabling Kata Containers](#enabling-kata-containers)
-        * [Register the runtime](#register-the-runtime)
-    * [Testing Kata Containers](#testing-kata-containers)
-    * [Wrapping up](#wrapping-up)
-
 ## Introduction
 
 [Minikube](https://kubernetes.io/docs/setup/minikube/) is an easy way to try out a Kubernetes (k8s)
@@ -166,8 +154,8 @@ $ kubectl apply -f https://raw.githubusercontent.com/kubernetes/node-api/master/
 Now register the `kata qemu` runtime with that class. This should result in no errors:
 
 ```sh
-$ cd kata-containers/tools/packaging/kata-deploy/k8s-1.14
-$ kubectl apply -f kata-qemu-runtimeClass.yaml
+$ cd kata-containers/tools/packaging/kata-deploy/runtimeclasses
+$ kubectl apply -f kata-runtimeClasses.yaml
 ```
 
 The Kata Containers installation process should be complete and enabled in the Minikube cluster.

docs/install/snap-installation-guide.md

@@ -1,11 +1,5 @@
 # Kata Containers snap package
 
-* [Install Kata Containers](#install-kata-containers)
-* [Configure Kata Containers](#configure-kata-containers)
-* [Integration with shim v2 Container Engines](#integration-with-shim-v2-container-engines)
-* [Remove Kata Containers snap package](#remove-kata-containers-snap-package)
-
-
 ## Install Kata Containers
 
 Kata Containers can be installed in any Linux distribution that supports

docs/threat-model/threat-model.md

@@ -0,0 +1,137 @@
+# Kata Containers threat model
+
+This document discusses threat models associated with the Kata Containers project.
+Kata was designed to provide additional isolation of container workloads, protecting
+the host infrastructure from potentially malicious container users or workloads. Since
+Kata Containers adds a level of isolation on top of traditional containers, the focus
+is on the additional layer provided, not on traditional container security.
+
+This document provides a brief background on containers and layered security, describes
+the interface to Kata from CRI runtimes, a review of utilized virtual machine interfaces, and then
+a review of threats.
+
+## Kata security objective
+
+Kata seeks to prevent an untrusted container workload or user of that container workload to gain
+control of, obtain information from, or tamper with the host infrastructure.
+
+In our scenario, an asset is anything on the host system, or elsewhere in the cluster
+infrastructure. The attacker is assumed to be either a malicious user or the workload itself
+running within the container. The goal of Kata is to prevent attacks which would allow
+any access to the defined assets.
+
+## Background on containers, layered security
+
+Traditional containers leverage several key Linux kernel features to provide isolation and
+a view that the container workload is the only entity running on the host. Key features include
+`Namespaces`, `cgroups`, `capablities`, `SELinux` and `seccomp`. The canonical runtime for creating such
+a container is `runc`. In the remainder of the document, the term `traditional-container` will be used
+to describe a container workload created by runc.
+
+Kata Containers provides a second layer of isolation on top of those provided by traditional-containers.
+The hardware virtualization interface is the basis of this additional layer. Kata launches a lightweight
+virtual machine, and uses the guest’s Linux kernel to create a container workload, or workloads in the case
+of multi-container pods. In Kubernetes and in the Kata implementation, the sandbox is carried out at the
+pod level. In Kata, this sandbox is created using a virtual machine.
+
+## Interface to Kata Containers: CRI, v2-shim, OCI
+
+A typical Kata Containers deployment uses Kubernetes with a CRI implementation.
+On every node, Kubelet will interact with a CRI implementor, which will in turn interface with
+an OCI based runtime, such as Kata Containers. Typical CRI implementors are `cri-o` and `containerd`.
+
+The CRI API, as defined at the Kubernetes [CRI-API repo](https://github.com/kubernetes/cri-api/),
+results in a few constructs being supported by the CRI implementation, and ultimately in the OCI
+runtime creating the workloads.
+
+In order to run a container inside of the Kata sandbox, several virtual machine devices and interfaces
+are required. Kata translates sandbox and container definitions to underlying virtualization technologies provided
+by a set of virtual machine monitors (VMMs) and hypervisors. These devices and their underlying
+implementations are discussed in detail in the following section.
+
+## Interface to the Kata sandbox/virtual machine
+
+In case of Kata, today the devices which we need in the guest are:
+ - Storage: In the current design of Kata Containers, we are reliant on the CRI implementor to
+ assist in image handling and volume management on the host. As a result, we need to support a way of passing to the sandbox the container rootfs, volumes requested
+ by the workload, and any other volumes created to facilitate sharing of secrets and `configmaps` with the containers. Depending on how these are managed, a block based device or file-system
+ sharing is required. Kata Containers does this by way of `virtio-blk` and/or `virtio-fs`.
+ - Networking: A method for enabling network connectivity with the workload is required. Typically this will be done providing a `TAP` device
+ to the VMM, and this will be exposed to the guest as a `virtio-net` device. It is feasible to pass in a NIC device directly, in which case `VFIO` is leveraged
+ and the device itself will be exposed to the guest.
+ - Control: In order to interact with the guest agent and retrieve `STDIO` from containers, a medium of communication is required.
+ This is available via `virtio-vsock`.
+ - Devices: `VFIO` is utilized when devices are passed directly to the virtual machine and exposed to the container.
+- Dynamic Resource Management: `ACPI` is utilized to allow for dynamic VM resource management (for example: CPU, memory, device hotplug). This is required when containers are resized,
+ or more generally when containers are added to a pod. 
+ 
+How these devices are utilized varies depending on the VMM utilized. We clarify the default settings provided when integrating Kata
+with the QEMU, Firecracker and Cloud Hypervisor VMMs in the following sections.
+
+### Devices
+
+Each virtio device is implemented by a backend, which may execute within userspace on the host (vhost-user), the VMM itself, or within the host kernel (vhost). While it may provide enhanced performance,
+vhost devices are often seen as higher risk since an exploit would be already running within the kernel space. While VMM and vhost-user are both in userspace on the host, `vhost-user` generally allows for the back-end process to require less system calls and capabilities compared to a full VMM.
+
+#### `virtio-blk` and `virtio-scsi`
+
+The backend for `virtio-blk` and `virtio-scsi` are based in the VMM itself (ring3 in the context of x86) by default for Cloud Hypervisor, Firecracker and QEMU.
+While `vhost` based back-ends are available for QEMU, it is not recommended. `vhost-user` back-ends are being added for Cloud Hypervisor, they are not utilized in Kata today.
+
+#### `virtio-fs`
+
+`virtio-fs` is supported in Cloud Hypervisor and QEMU. `virtio-fs`'s interaction with the host filesystem is done through a vhost-user daemon, `virtiofsd`.
+The `virtio-fs` client, running in the guest, will generate requests to access files. `virtiofsd` will receive requests, open the file, and request the VMM
+to `mmap` it into the guest. When DAX is utilized, the guest will access the host's page cache, avoiding the need for copy and duplication. DAX is still an experimental feature,
+and is not enabled by default.
+
+From the `virtiofsd` [documentation](https://qemu-project.gitlab.io/qemu/tools/virtiofsd.html): 
+```This program must be run as the root user. Upon startup the program will switch into a new file system namespace with the shared directory tree as its root. This prevents “file system escapes” due to symlinks and other file system objects that might lead to files outside the shared directory. The program also sandboxes itself using seccomp(2) to prevent ptrace(2) and other vectors that could allow an attacker to compromise the system after gaining control of the virtiofsd process.```
+
+DAX-less support for `virtio-fs` is available as of the 5.4 Linux kernel. QEMU VMM supports virtio-fs as of v4.2. Cloud Hypervisor
+supports `virtio-fs`.
+
+#### `virtio-net`
+
+`virtio-net` has many options, depending on the VMM and Kata configurations.
+
+##### QEMU networking
+
+While QEMU has options for `vhost`, `virtio-net` and `vhost-user`, the `virtio-net` backend
+for Kata defaults to `vhost-net` for performance reasons. The default configuration is being
+reevaluated.
+
+##### Firecracker networking
+
+For Firecracker, the `virtio-net` backend is within Firecracker's VMM.
+
+##### Cloud Hypervisor networking
+
+For Cloud Hypervisor, the current backend default is within the VMM. `vhost-user-net` support
+is being added (written in rust, Cloud Hypervisor specific).
+
+#### virtio-vsock
+
+##### QEMU vsock
+
+In QEMU, vsock is backed by `vhost_vsock`, which runs within the kernel itself.
+
+##### Firecracker and Cloud Hypervisor
+
+In Firecracker and Cloud Hypervisor, vsock is backed by a unix-domain-socket in the hosts userspace.
+
+#### VFIO
+
+Utilizing VFIO, devices can be passed through to the virtual machine. We will assess this separately. Exposure to
+host is limited to gaps in device pass-through handling. This is supported in QEMU and Cloud Hypervisor, but not
+Firecracker.
+
+#### ACPI
+
+ACPI is necessary for hotplug of CPU, memory and devices. ACPI is available in QEMU and Cloud Hypervisor. Device, CPU and memory hotplug
+are not available in Firecracker.
+
+## Devices and threat model
+
+![Threat model](threat-model-boundaries.svg "threat-model")
+

docs/tracing.md

@@ -0,0 +1,213 @@
+# Overview
+
+This document explains how to trace Kata Containers components.
+
+# Introduction
+
+The Kata Containers runtime and agent are able to generate
+[OpenTelemetry][opentelemetry] trace spans, which allow the administrator to
+observe what those components are doing and how much time they are spending on
+each operation.
+
+# OpenTelemetry summary
+
+An OpenTelemetry-enabled application creates a number of trace "spans". A span
+contains the following attributes:
+
+- A name
+- A pair of timestamps (recording the start time and end time of some operation)
+- A reference to the span's parent span
+
+All spans need to be *finished*, or *completed*, to allow the OpenTelemetry
+framework to generate the final trace information (by effectively closing the
+transaction encompassing the initial (root) span and all its children).
+
+For Kata, the root span represents the total amount of time taken to run a
+particular component from startup to its shutdown (the "run time").
+
+# Architecture
+
+## Runtime tracing architecture
+
+The runtime, which runs in the host environment, has been modified to
+optionally generate trace spans which are sent to a trace collector on the
+host.
+
+## Agent tracing architecture
+
+An OpenTelemetry system (such as [Jaeger][jaeger-tracing]) uses a collector to
+gather up trace spans from the application for viewing and processing. For an
+application to use the collector, it must run in the same context as
+the collector.
+
+This poses a problem for tracing the Kata Containers agent since it does not
+run in the same context as the collector: it runs inside a virtual machine (VM).
+
+To allow spans from the agent to be sent to the trace collector, Kata provides
+a [trace forwarder][trace-forwarder] component. This runs in the same context
+as the collector (generally on the host system) and listens on a
+[`VSOCK`][vsock] channel for traces generated by the agent, forwarding them on
+to the trace collector.
+
+> **Note:**
+>
+> This design supports agent tracing without having to make changes to the
+> image, but also means that [custom images][osbuilder] can also benefit from
+> agent tracing.
+
+The following diagram summarises the architecture used to trace the Kata
+Containers agent:
+
+```
++--------------------------------------------+
+| Host                                       |
+|                                            |
+| +---------------+                          |
+| | OpenTelemetry |                          |
+| | Trace         |                          |
+| | Collector     |                          |
+| +---------------+                          |
+|       ^                  +---------------+ |
+|       | spans            | Kata VM       | |
+| +-----+-----+            |               | |
+| | Kata      |    spans   o     +-------+ | |
+| | Trace     |<-----------------| Kata  | | |
+| | Forwarder |    VSOCK   o     | Agent | | |
+| +-----------+    Channel |     +-------+ | |
+|                          +---------------+ |
++--------------------------------------------+
+```
+
+# Agent tracing prerequisites
+
+- You must have a trace collector running.
+
+  Although the collector normally runs on the host, it can also be run from
+  inside a Docker image configured to expose the appropriate host ports to the
+  collector.
+
+  The [Jaeger "all-in-one" Docker image][jaeger-all-in-one] method
+  is the quickest and simplest way to run the collector for testing.
+
+- If you wish to trace the agent, you must start the
+  [trace forwarder][trace-forwarder].
+
+> **Notes:**
+>
+> - If agent tracing is enabled but the forwarder is not running,
+>   the agent will log an error (signalling that it cannot generate trace
+>   spans), but continue to work as normal.
+>
+> - The trace forwarder requires a trace collector (such as Jaeger) to be
+>   running before it is started. If a collector is not running, the trace
+>   forwarder will exit with an error.
+
+# Enable tracing
+
+By default, tracing is disabled for all components. To enable _any_ form of
+tracing an `enable_tracing` option must be enabled for at least one component.
+
+> **Note:** 
+>
+> Enabling this option will only allow tracing for subsequently
+> started containers.
+
+## Enable runtime tracing
+
+To enable runtime tracing, set the tracing option as shown:
+
+```toml
+[runtime]
+enable_tracing = true
+```
+
+## Enable agent tracing
+
+To enable agent tracing, set the tracing option as shown:
+
+```toml
+[agent.kata]
+enable_tracing = true
+```
+
+> **Note:**
+>
+> If both agent tracing and runtime tracing are enabled, the resulting trace
+> spans will be "collated": expanding individual runtime spans in the Jaeger
+> web UI will show the agent trace spans resulting from the runtime
+> operation.
+
+# Appendices
+
+## Agent tracing requirements
+
+### Host environment
+
+- The host kernel must support the VSOCK socket type.
+
+  This will be available if the kernel is built with the
+  `CONFIG_VHOST_VSOCK` configuration option.
+
+- The VSOCK kernel module must be loaded:
+
+   ```
+   $ sudo modprobe vhost_vsock
+   ```
+
+### Guest environment
+
+- The guest kernel must support the VSOCK socket type:
+
+  This will be available if the kernel is built with the
+  `CONFIG_VIRTIO_VSOCKETS` configuration option.
+
+  > **Note:** The default Kata Containers guest kernel provides this feature.
+
+## Agent tracing limitations
+
+- Agent tracing is only "completed" when the workload and the Kata agent
+  process have exited.
+
+  Although trace information *can* be inspected before the workload and agent
+  have exited, it is incomplete. This is shown as `<trace-without-root-span>`
+  in the Jaeger web UI.
+
+  If the workload is still running, the trace transaction -- which spans the entire
+  runtime of the Kata agent -- will not have been completed. To view the complete
+  trace details, wait for the workload to end, or stop the container.
+
+## Performance impact
+
+[OpenTelemetry][opentelemetry] is designed for high performance. It combines
+the best of two previous generation projects (OpenTracing and OpenCensus) and
+uses a very efficient mechanism to capture trace spans. Further, the trace
+points inserted into the agent are generated dynamically at compile time. This
+is advantageous since new versions of the agent will automatically benefit
+from improvements in the tracing infrastructure. Overall, the impact of
+enabling runtime and agent tracing should be extremely low.
+
+## Agent shutdown behaviour
+ 
+In normal operation, the Kata runtime manages the VM shutdown and performs
+certain optimisations to speed up this process. However, if agent tracing is
+enabled, the agent itself is responsible for shutting down the VM. This it to
+ensure all agent trace transactions are completed. This means there will be a
+small performance impact for container shutdown when agent tracing is enabled
+as the runtime must wait for the VM to shutdown fully.
+
+## Set up a tracing development environment
+
+If you want to debug, further develop, or test tracing,
+[enabling full debug][enable-full-debug]
+is highly recommended. For working with the agent, you may also wish to
+[enable a debug console][setup-debug-console]
+to allow you to access the VM environment.
+
+[enable-full-debug]: https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md#enable-full-debug
+[jaeger-all-in-one]: https://www.jaegertracing.io/docs/getting-started/
+[jaeger-tracing]: https://www.jaegertracing.io
+[opentelemetry]: https://opentelemetry.io
+[osbuilder]: https://github.com/kata-containers/kata-containers/blob/main/tools/osbuilder
+[setup-debug-console]: https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md#set-up-a-debug-console
+[trace-forwarder]: /src/trace-forwarder
+[vsock]: https://wiki.qemu.org/Features/VirtioVsock

docs/use-cases/Intel-GPU-passthrough-and-Kata.md

@@ -1,13 +1,5 @@
 # Using Intel GPU device with Kata Containers
 
-- [Using Intel GPU device with Kata Containers](#using-intel-gpu-device-with-kata-containers)   
-   - [Hardware Requirements](#hardware-requirements)   
-   - [Host Kernel Requirements](#host-kernel-requirements)   
-   - [Install and configure Kata Containers](#install-and-configure-kata-containers)   
-   - [Build Kata Containers kernel with GPU support](#build-kata-containers-kernel-with-gpu-support)   
-   - [GVT-d with Kata Containers](#gvt-d-with-kata-containers)   
-   - [GVT-g with Kata Containers](#gvt-g-with-kata-containers)   
-
 An Intel Graphics device can be passed to a Kata Containers container using GPU
 passthrough (Intel GVT-d) as well as GPU mediated passthrough (Intel GVT-g).
 

docs/use-cases/Nvidia-GPU-passthrough-and-Kata.md

@@ -1,17 +1,5 @@
 # Using Nvidia GPU device with Kata Containers
 
-- [Using Nvidia GPU device with Kata Containers](#using-nvidia-gpu-device-with-kata-containers)
-	- [Hardware Requirements](#hardware-requirements)
-	- [Host BIOS Requirements](#host-bios-requirements)
-	- [Host Kernel Requirements](#host-kernel-requirements)
-	- [Install and configure Kata Containers](#install-and-configure-kata-containers)
-	- [Build Kata Containers kernel with GPU support](#build-kata-containers-kernel-with-gpu-support)
-	- [Nvidia GPU pass-through mode with Kata Containers](#nvidia-gpu-pass-through-mode-with-kata-containers)
-	- [Nvidia vGPU mode with Kata Containers](#nvidia-vgpu-mode-with-kata-containers)
-	- [Install Nvidia Driver in Kata Containers](#install-nvidia-driver-in-kata-containers)
-	- [References](#references)
-
-
 An Nvidia GPU device can be passed to a Kata Containers container using GPU passthrough
 (Nvidia GPU pass-through mode) as well as GPU mediated passthrough (Nvidia vGPU mode). 
 
@@ -79,7 +67,7 @@ To use large BARs devices (for example, Nvidia Tesla P100), you need Kata versio
 
 The following configuration in the Kata `configuration.toml` file as shown below can work:
 
-Hotplug for PCI devices by `shpchp` (Linux's SHPC PCI Hotplug driver):
+Hotplug for PCI devices by `acpi_pcihp` (Linux's ACPI PCI Hotplug driver):
 ```
 machine_type = "q35"
 
@@ -103,7 +91,6 @@ The following kernel config options need to be enabled:
 ```
 # Support PCI/PCIe device hotplug (Required for large BARs device)
 CONFIG_HOTPLUG_PCI_PCIE=y
-CONFIG_HOTPLUG_PCI_SHPC=y
 
 # Support for loading modules (Required for load Nvidia drivers)
 CONFIG_MODULES=y
@@ -303,4 +290,4 @@ Tue Mar  3 00:03:49 2020
 
 - [Configuring a VM for GPU Pass-Through by Using the QEMU Command Line](https://docs.nvidia.com/grid/latest/grid-vgpu-user-guide/index.html#using-gpu-pass-through-red-hat-el-qemu-cli)
 - https://gitlab.com/nvidia/container-images/driver/-/tree/master
-- https://github.com/NVIDIA/nvidia-docker/wiki/Driver-containers-(Beta)
+- https://github.com/NVIDIA/nvidia-docker/wiki/Driver-containers

docs/use-cases/using-Intel-QAT-and-kata.md

@@ -1,33 +1,5 @@
 # Table of Contents
 
-- [Table of Contents](#table-of-contents)
-- [Introduction](#introduction)
-  - [Helpful Links before starting](#helpful-links-before-starting)
-  - [Steps to enable Intel® QAT in Kata Containers](#steps-to-enable-intel-qat-in-kata-containers)
-  - [Script variables](#script-variables)
-    - [Set environment variables (Every Reboot)](#set-environment-variables-every-reboot)
-  - [Prepare the Ubuntu Host](#prepare-the-ubuntu-host)
-    - [Identify which PCI Bus the Intel® QAT card is on](#identify-which-pci-bus-the-intel-qat-card-is-on)
-    - [Install necessary packages for Ubuntu](#install-necessary-packages-for-ubuntu)
-    - [Download Intel® QAT drivers](#download-intel-qat-drivers)
-    - [Copy Intel® QAT configuration files and enable virtual functions](#copy-intel-qat-configuration-files-and-enable-virtual-functions)
-    - [Expose and Bind Intel® QAT virtual functions to VFIO-PCI (Every reboot)](#expose-and-bind-intel-qat-virtual-functions-to-vfio-pci-every-reboot)
-    - [Check Intel® QAT virtual functions are enabled](#check-intel-qat-virtual-functions-are-enabled)
-  - [Prepare Kata Containers](#prepare-kata-containers)
-    - [Download Kata kernel Source](#download-kata-kernel-source)
-    - [Build Kata kernel](#build-kata-kernel)
-    - [Copy Kata kernel](#copy-kata-kernel)
-    - [Prepare Kata root filesystem](#prepare-kata-root-filesystem)
-    - [Compile Intel® QAT drivers for Kata Containers kernel and add to Kata Containers rootfs](#compile-intel-qat-drivers-for-kata-containers-kernel-and-add-to-kata-containers-rootfs)
-    - [Copy Kata rootfs](#copy-kata-rootfs)
-  - [Verify Intel® QAT works in a container](#verify-intel-qat-works-in-a-container)
-    - [Build OpenSSL Intel® QAT engine container](#build-openssl-intel-qat-engine-container)
-    - [Test Intel® QAT with the ctr tool](#test-intel-qat-with-the-ctr-tool)
-    - [Test Intel® QAT in Kubernetes](#test-intel-qat-in-kubernetes)
-    - [Troubleshooting](#troubleshooting)
-  - [Optional Scripts](#optional-scripts)
-    - [Verify Intel® QAT card counters are incremented](#verify-intel-qat-card-counters-are-incremented)
-
 # Introduction
 
 Intel® QuickAssist Technology (QAT) provides hardware acceleration 
@@ -604,4 +576,4 @@ $ for i in 0434 0435 37c8 1f18 1f19; do lspci -d 8086:$i; done
 $ sudo watch cat /sys/kernel/debug/qat_c6xx_0000\:b1\:00.0/fw_counters
 $ sudo watch cat /sys/kernel/debug/qat_c6xx_0000\:b3\:00.0/fw_counters
 $ sudo watch cat /sys/kernel/debug/qat_c6xx_0000\:b5\:00.0/fw_counters
-```
+```

docs/use-cases/using-Intel-SGX-and-kata.md

@@ -1,112 +1,113 @@
 # Kata Containers with SGX
 
-- [Check if SGX is enabled](#check-if-sgx-is-enabled)
-- [Install Host kernel with SGX support](#install-host-kernel-with-sgx-support)
-- [Install Guest kernel with SGX support](#install-guest-kernel-with-sgx-support)
-- [Run Kata Containers with SGX enabled](#run-kata-containers-with-sgx-enabled)
-
-Intel® Software Guard Extensions (SGX) is a set of instructions that increases the security
+Intel Software Guard Extensions (SGX) is a set of instructions that increases the security
 of applications code and data, giving them more protections from disclosure or modification.
 
-> **Note:** At the time of writing this document, SGX patches have not landed on the Linux kernel
-> project, so specific versions for guest and host kernels must be installed to enable SGX.
+This document guides you to run containers with SGX enclaves with Kata Containers in Kubernetes.
 
-## Check if SGX is enabled
+## Preconditions
 
-Run the following command to check if your host supports SGX.
+* Intel SGX capable bare metal nodes
+* Host kernel Linux 5.13 or later with SGX and SGX KVM enabled:
 
 ```sh
-$ grep -o sgx /proc/cpuinfo
+$ grep SGX /boot/config-`uname -r`
+CONFIG_X86_SGX=y
+CONFIG_X86_SGX_KVM=y
 ```
 
-Continue to the following section if the output of the above command is empty,
-otherwise continue to section [Install Guest kernel with SGX support](#install-guest-kernel-with-sgx-support)
+* Kubernetes cluster configured with:
+   * [`kata-deploy`](https://github.com/kata-containers/kata-containers/tree/main/tools/packaging/kata-deploy) based Kata Containers installation
+   * [Intel SGX Kubernetes device plugin](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/main/cmd/sgx_plugin#deploying-with-pre-built-images)
 
-## Install Host kernel with SGX support
+> Note: Kata Containers supports creating VM sandboxes with Intel® SGX enabled
+> using [cloud-hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor/) VMM only. QEMU support is waiting to get the
+> Intel SGX enabled QEMU upstream release.
 
-The following commands were tested on Fedora 32, they might work on other distros too.
+## Installation
+
+### Kata Containers Guest Kernel
+
+Follow the instructions to [setup](../../tools/packaging/kernel/README.md#setup-kernel-source-code) and [build](../../tools/packaging/kernel/README.md#build-the-kernel) the experimental guest kernel. Then, install as:
 
 ```sh
-$ git clone --depth=1 https://github.com/intel/kvm-sgx
-$ pushd kvm-sgx
-$ cp /boot/config-$(uname -r) .config
-$ yes "" | make oldconfig
-$ # In the following step, enable: INTEL_SGX and INTEL_SGX_VIRTUALIZATION
-$ make menuconfig
-$ make -j$(($(nproc)-1)) bzImage
-$ make -j$(($(nproc)-1)) modules
-$ sudo make modules_install
-$ sudo make install
-$ popd
-$ sudo reboot
+$ sudo cp kata-linux-experimental-*/vmlinux /opt/kata/share/kata-containers/vmlinux.sgx
+$ sudo sed -i 's|vmlinux.container|vmlinux.sgx|g' \
+  /opt/kata/share/defaults/kata-containers/configuration-clh.toml
 ```
 
-> **Notes:**
-> * Run: `mokutil --sb-state` to check whether secure boot is enabled, if so, you will need to sign the kernel.
-> * You'll lose SGX support when a new distro kernel is installed and the system rebooted.
-
-Once you have restarted your system with the new brand Linux Kernel with SGX support, run
-the following command to make sure it's enabled. If the output is empty, go to the BIOS
-setup and enable SGX manually.
-
-```sh
-$ grep -o sgx /proc/cpuinfo
-```
-
-## Install Guest kernel with SGX support
-
-Install the guest kernel in the Kata Containers directory, this way it can be used to run
-Kata Containers.
-
-```sh
-$ curl -LOk https://github.com/devimc/kvm-sgx/releases/download/v0.0.1/kata-virtiofs-sgx.tar.gz
-$ sudo tar -xf kata-virtiofs-sgx.tar.gz -C /usr/share/kata-containers/
-$ sudo sed -i 's|kernel =|kernel = "/usr/share/kata-containers/vmlinux-virtiofs-sgx.container"|g' \
-  /usr/share/defaults/kata-containers/configuration.toml
-```
-
-## Run Kata Containers with SGX enabled
+### Kata Containers Configuration
 
 Before running a Kata Container make sure that your version of `crio` or `containerd`
 supports annotations.
+
 For `containerd` check in `/etc/containerd/config.toml` that the list of `pod_annotations` passed
 to the `sandbox` are: `["io.katacontainers.*", "sgx.intel.com/epc"]`.
 
-> `sgx.yaml`
+## Usage
+
+With the following sample job deployed using `kubectl apply -f`:
+
 ```yaml
-apiVersion: v1
-kind: Pod
+apiVersion: batch/v1
+kind: Job
 metadata:
-  name: sgx
-  annotations:
-    sgx.intel.com/epc: "32Mi"
+  name: oesgx-demo-job
+  labels:
+    jobgroup: oesgx-demo
 spec:
-  terminationGracePeriodSeconds: 0
-  runtimeClassName: kata
-  containers:
-  - name: c1
-    image: busybox
-    command:
-        - sh
-    stdin: true
-    tty: true
-    volumeMounts:
-    - mountPath: /dev/sgx/
-      name: test-volume
-  volumes:
-  - name: test-volume
-    hostPath:
-      path: /dev/sgx/
-      type: Directory
+  template:
+    metadata:
+      labels:
+        jobgroup: oesgx-demo
+    spec:
+      runtimeClassName: kata-clh
+      initContainers:
+        - name: init-sgx
+          image: busybox
+          command: ['sh', '-c', 'mkdir /dev/sgx; ln -s /dev/sgx_enclave /dev/sgx/enclave; ln -s /dev/sgx_provision /dev/sgx/provision']
+          volumeMounts:
+          - mountPath: /dev
+            name: dev-mount
+      restartPolicy: Never
+      containers:
+        -
+          name: eosgx-demo-job-1
+          image: oeciteam/oe-helloworld:latest
+          imagePullPolicy: IfNotPresent
+          securityContext:
+            readOnlyRootFilesystem: true
+            capabilities:
+              add: ["IPC_LOCK"]
+          resources:
+            limits:
+              sgx.intel.com/epc: "512Ki"
+      volumes:
+        - name: dev-mount
+          hostPath:
+            path: /dev
 ```
 
+You'll see the enclave output:
+
 ```sh
-$ kubectl apply -f sgx.yaml
-$ kubectl exec -ti sgx ls /dev/sgx/
-enclave    provision
+$ kubectl logs oesgx-demo-job-wh42g
+Hello world from the enclave
+Enclave called into host to print: Hello World!
 ```
 
-The output of the latest command shouldn't be empty, otherwise check
-your system environment to make sure SGX is fully supported.
+### Notes
 
-[1]: github.com/cloud-hypervisor/cloud-hypervisor/
+* The Kata VM's SGX Encrypted Page Cache (EPC) memory size is based on the sum of `sgx.intel.com/epc`
+resource requests within the pod.
+* `init-sgx` can be removed from the YAML configuration file if the Kata rootfs is modified with the
+necessary udev rules.
+   See the [note on SGX backwards compatibility](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/main/cmd/sgx_plugin#backwards-compatibility-note).
+* Intel SGX DCAP attestation is known to work from Kata sandboxes but it comes with one limitation: If
+the Intel SGX `aesm` daemon runs on the bare metal node and DCAP `out-of-proc` attestation is used,
+containers within the Kata sandbox cannot get the access to the host's `/var/run/aesmd/aesm.sock`
+because socket passthrough is not supported. An alternative is to deploy the `aesm` daemon as a side-car
+container.
+* Projects like [Gramine Shielded Containers (GSC)](https://gramine-gsc.readthedocs.io/en/latest/) are
+also known to work. For GSC specifically, the Kata guest kernel needs to have the `CONFIG_NUMA=y`
+enabled and at least one CPU online when running the GSC container.

docs/use-cases/using-SPDK-vhostuser-and-kata.md

@@ -1,13 +1,6 @@
 # Setup to run SPDK vhost-user devices with Kata Containers and Docker*
 
-- [SPDK vhost-user target overview](#spdk-vhost-user-target-overview)
-- [Install and setup SPDK vhost-user target](#install-and-setup-spdk-vhost-user-target)
-  - [Get source code and build SPDK](#get-source-code-and-build-spdk)
-  - [Run SPDK vhost-user target](#run-spdk-vhost-user-target)
-- [Host setup for vhost-user devices](#host-setup-for-vhost-user-devices)
-- [Launch a Kata container with SPDK vhost-user block device](#launch-a-kata-container-with-spdk-vhost-user-block-device)
-
-> **NOTE:** This guide only applies to QEMU, since the vhost-user storage
+> **Note:** This guide only applies to QEMU, since the vhost-user storage
 > device is only available for QEMU now. The enablement work on other
 > hypervisors is still ongoing.
 

docs/use-cases/using-SRIOV-and-kata.md

@@ -1,13 +1,5 @@
 # Setup to use SR-IOV with Kata Containers and Docker*
 
-- [Install the SR-IOV Docker\* plugin](#install-the-sr-iov-docker-plugin)
-- [Host setup for SR-IOV](#host-setup-for-sr-iov)
-	- [Checking your NIC for SR-IOV](#checking-your-nic-for-sr-iov)
-	- [IOMMU Groups and PCIe Access Control Services](#iommu-groups-and-pcie-access-control-services)
-	- [Update the host kernel](#update-the-host-kernel)
-- [Set up the SR-IOV Device](#set-up-the-sr-iov-device)
-- [Example: Launch a Kata Containers container using SR-IOV](#example-launch-a-kata-containers-container-using-sr-iov)
-
 Single Root I/O Virtualization (SR-IOV) enables splitting a physical device into
 virtual functions (VFs). Virtual functions enable direct passthrough to virtual
 machines or containers. For Kata Containers, we enabled a Container Network

docs/use-cases/using-vpp-and-kata.md

@@ -12,7 +12,7 @@ For more information about VPP visit their [wiki](https://wiki.fd.io/view/VPP).
 
 ## Install and configure Kata Containers
 
-Follow  the [Kata Containers setup instructions](https://github.com/kata-containers/documentation/wiki/Developer-Guide).
+Follow  the [Kata Containers setup instructions](../Developer-Guide.md).
 
 In order to make use of VHOST-USER based interfaces, the container needs to be backed
 by huge pages. `HugePages` support is required for the large memory pool allocation used for

docs/use-cases/zun_kata.md

@@ -1,4 +1,5 @@
 # OpenStack Zun DevStack working with Kata Containers
+
 ## Introduction
 
 This guide describes how to get Kata Containers to work with OpenStack Zun

pkg/logging/Cargo.toml

@@ -7,15 +7,15 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-serde_json = "1.0.39"
+serde_json = "1.0.73"
 # slog:
 # - Dynamic keys required to allow HashMap keys to be slog::Serialized.
 # - The 'max_*' features allow changing the log level at runtime
 #   (by stopping the compiler from removing log calls).
-slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_info"] }
-slog-json = "2.3.0"
-slog-async = "2.3.0"
-slog-scope = "4.1.2"
+slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug"] }
+slog-json = "2.4.0"
+slog-async = "2.7.0"
+slog-scope = "4.4.0"
 
 [dev-dependencies]
-tempfile = "3.1.0"
+tempfile = "3.2.0"

snap/README.md

@@ -1,13 +1,5 @@
 # Kata Containers snap image
 
-* [Initial setup](#initial-setup)
-* [Install snap](#install-snap)
-* [Build and install snap image](#build-and-install-snap-image)
-* [Configure Kata Containers](#configure-kata-containers)
-* [Integration with docker and Kubernetes](#integration-with-docker-and-kubernetes)
-* [Remove snap](#remove-snap)
-* [Limitations](#limitations)
-
 This directory contains the resources needed to build the Kata Containers
 [snap][1] image.
 

snap/snapcraft.yaml

@@ -59,7 +59,7 @@ parts:
 
       yq_version=3.4.1
       yq_url="https://${yq_pkg}/releases/download/${yq_version}/yq_${goos}_${goarch}"
-      curl -o "${yq_path}" -LSsf "${yq_url}"
+      curl -o "${yq_path}" -L "${yq_url}"
       chmod +x "${yq_path}"
 
       kata_dir=gopath/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
@@ -115,7 +115,6 @@ parts:
       cd ${kata_dir}/tools/osbuilder
 
       # build image
-      export AGENT_VERSION=$(cat ${kata_dir}/VERSION)
       export AGENT_INIT=yes
       export USE_DOCKER=1
       export DEBUG=1
@@ -140,7 +139,7 @@ parts:
       cp kata-containers*.img ${kata_image_dir}
 
   runtime:
-    after: [godeps, image]
+    after: [godeps, image, cloud-hypervisor]
     plugin: nil
     build-attributes: [no-patchelf]
     override-build: |
@@ -170,12 +169,9 @@ parts:
         SKIP_GO_VERSION_CHECK=1 \
         QEMUCMD=qemu-system-$arch
 
-      if [ -e ${SNAPCRAFT_PART_INSTALL}/../../image/install/usr/share/kata-containers/kata-containers.img ]; then
-        # Use rootfs image by default
-        sed -i -e '/^initrd =/d' ${SNAPCRAFT_PART_INSTALL}/usr/share/defaults/${SNAPCRAFT_PROJECT_NAME}/configuration.toml
-      else
-        # Use initrd by default
-        sed -i -e '/^image =/d' ${SNAPCRAFT_PART_INSTALL}/usr/share/defaults/${SNAPCRAFT_PROJECT_NAME}/configuration.toml
+      if [ ! -f ${SNAPCRAFT_PART_INSTALL}/../../image/install/usr/share/kata-containers/kata-containers.img ]; then
+        sed -i -e "s|^image =.*|initrd = \"/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr/share/kata-containers/kata-containers-initrd.img\"|" \
+          ${SNAPCRAFT_PART_INSTALL}/usr/share/defaults/${SNAPCRAFT_PROJECT_NAME}/configuration.toml
       fi
 
   kernel:
@@ -188,6 +184,15 @@ parts:
       - bison
       - flex
     override-build: |
+      yq=${SNAPCRAFT_STAGE}/yq
+      export PATH="${PATH}:${SNAPCRAFT_STAGE}"
+      export GOPATH=${SNAPCRAFT_STAGE}/gopath
+      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
+      versions_file="${kata_dir}/versions.yaml"
+      kernel_version="$(${yq} r $versions_file assets.kernel.version)"
+      #Remove extra 'v'
+      kernel_version=${kernel_version#v}
+
       [ "$(uname -m)" = "s390x" ] && sudo apt-get --no-install-recommends install -y libssl-dev
 
       export GOPATH=${SNAPCRAFT_STAGE}/gopath
@@ -195,15 +200,21 @@ parts:
       kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
 
       cd ${kata_dir}/tools/packaging/kernel
+      kernel_dir_prefix="kata-linux-"
 
       # Setup and build kernel
-      ./build-kernel.sh -d setup
-      kernel_dir_prefix="kata-linux-"
+      if [ "$(uname -m)" = "x86_64" ]; then
+        kernel_version="$(${yq} r $versions_file assets.kernel-experimental.tag)"
+        kernel_version=${kernel_version#v}
+        kernel_dir_prefix="kata-linux-experimental-"
+        ./build-kernel.sh -e -v ${kernel_version} -d setup
+      else
+        ./build-kernel.sh -v ${kernel_version} -d setup
+      fi
       cd ${kernel_dir_prefix}*
-      version=$(basename ${PWD} | sed 's|'"${kernel_dir_prefix}"'||' | cut -d- -f1)
       make -j $(($(nproc)-1)) EXTRAVERSION=".container"
 
-      kernel_suffix=${version}.container
+      kernel_suffix=${kernel_version}.container
       kata_kernel_dir=${SNAPCRAFT_PART_INSTALL}/usr/share/kata-containers
       mkdir -p ${kata_kernel_dir}
 
@@ -253,10 +264,11 @@ parts:
       # arch-specific definition
       case "$(uname -m)" in
         "aarch64")
-          branch="$(${yq} r ${versions_file} assets.hypervisor.qemu.architecture.aarch64.branch)"
+          branch="$(${yq} r ${versions_file} assets.hypervisor.qemu.architecture.aarch64.version)"
           url="$(${yq} r ${versions_file} assets.hypervisor.qemu.url)"
           commit="$(${yq} r ${versions_file} assets.hypervisor.qemu.architecture.aarch64.commit)"
-          patches_dir="${kata_dir}/tools/packaging/obs-packaging/qemu-aarch64/patches/"
+          patches_dir="${kata_dir}/tools/packaging/qemu/patches/$(echo ${branch} | sed -e 's/.[[:digit:]]*$//' -e 's/^v//').x"
+          patches_version_dir="${kata_dir}/tools/packaging/qemu/patches/tag_patches/${branch}"
         ;;
 
         *)
@@ -270,6 +282,7 @@ parts:
 
       # download source
       qemu_dir=${SNAPCRAFT_STAGE}/qemu
+      rm -rf "${qemu_dir}"
       git clone --branch ${branch} --single-branch ${url} "${qemu_dir}"
       cd ${qemu_dir}
       [ -z "${commit}" ] || git checkout ${commit}
@@ -278,6 +291,7 @@ parts:
       [ -n "$(ls -A capstone)" ] || git clone https://github.com/qemu/capstone capstone
 
       # Apply branch patches
+      [ -d "${patches_version_dir}" ] || mkdir "${patches_version_dir}"
       ${kata_dir}/tools/packaging/scripts/apply_patches.sh "${patches_dir}"
       ${kata_dir}/tools/packaging/scripts/apply_patches.sh "${patches_version_dir}"
 
@@ -293,7 +307,15 @@ parts:
         | xargs ./configure
 
       # Copy QEMU configurations (Kconfigs)
-      cp -a ${kata_dir}/tools/packaging/qemu/default-configs/* default-configs/devices/
+      case "${branch}" in
+      "v5.1.0")
+        cp -a ${kata_dir}/tools/packaging/qemu/default-configs/* default-configs
+        ;;
+
+      *)
+        cp -a ${kata_dir}/tools/packaging/qemu/default-configs/* configs/devices/
+        ;;
+      esac
 
       # build and install
       make -j $(($(nproc)-1))
@@ -313,6 +335,22 @@ parts:
       # Hack: move qemu to /
       "snap/kata-containers/current/": "./"
 
+  cloud-hypervisor:
+    plugin: nil
+    after: [godeps]
+    override-build: |
+      export GOPATH=${SNAPCRAFT_STAGE}/gopath
+      yq=${SNAPCRAFT_STAGE}/yq
+      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
+      versions_file="${kata_dir}/versions.yaml"
+      version="$(${yq} r ${versions_file} assets.hypervisor.cloud_hypervisor.version)"
+      url="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/${version}"
+      curl -L ${url}/cloud-hypervisor-static -o cloud-hypervisor
+      curl -LO ${url}/clh-remote
+
+      install -D cloud-hypervisor ${SNAPCRAFT_PART_INSTALL}/usr/bin/cloud-hypervisor
+      install -D clh-remote ${SNAPCRAFT_PART_INSTALL}/usr/bin/clh-remote
+
 apps:
   runtime:
     command: usr/bin/kata-runtime

src/agent/Cargo.toml

@@ -6,7 +6,6 @@ edition = "2018"
 
 [dependencies]
 oci = { path = "oci" }
-logging = { path = "../../pkg/logging" }
 rustjail = { path = "rustjail" }
 protocols = { path = "protocols" }
 lazy_static = "1.3.0"
@@ -18,15 +17,17 @@ capctl = "0.2.0"
 serde_json = "1.0.39"
 scan_fmt = "0.2.3"
 scopeguard = "1.0.0"
+thiserror = "1.0.26"
 regex = "1"
+serial_test = "0.5.1"
 
 # Async helpers
 async-trait = "0.1.42"
 async-recursion = "0.3.2"
-futures = "0.3.12"
+futures = "0.3.17"
 
 # Async runtime
-tokio = { version = "1.2.0", features = ["full"] }
+tokio = { version = "1", features = ["full"] }
 tokio-vsock = "0.3.1"
 
 netlink-sys = { version = "0.7.0", features = ["tokio_socket",]}
@@ -34,29 +35,32 @@ rtnetlink = "0.8.0"
 netlink-packet-utils = "0.4.1"
 ipnetwork = "0.17.0"
 
-# slog:
-# - Dynamic keys required to allow HashMap keys to be slog::Serialized.
-# - The 'max_*' features allow changing the log level at runtime
-#   (by stopping the compiler from removing log calls).
-slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_info"] }
+# Note: this crate sets the slog 'max_*' features which allows the log level
+# to be modified at runtime.
+logging = { path = "../../pkg/logging" }
+slog = "2.5.2"
 slog-scope = "4.1.2"
 
 # Redirect ttrpc log calls
 slog-stdlog = "4.0.0"
 log = "0.4.11"
 
-prometheus = { version = "0.9.0", features = ["process"] }
-procfs = "0.7.9"
+prometheus = { version = "0.13.0", features = ["process"] }
+procfs = "0.12.0"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.2.5" }
+cgroups = { package = "cgroups-rs", version = "0.2.8" }
 
 # Tracing
 tracing = "0.1.26"
 tracing-subscriber = "0.2.18"
 tracing-opentelemetry = "0.13.0"
-opentelemetry = "0.14.0"
+opentelemetry = { version = "0.14.0", features = ["rt-tokio-current-thread"]}
 vsock-exporter = { path = "vsock-exporter" }
 
+# Configuration
+serde = { version = "1.0.129", features = ["derive"] }
+toml = "0.5.8"
+
 [dev-dependencies]
 tempfile = "3.1.0"
 
@@ -69,3 +73,6 @@ members = [
 
 [profile.release]
 lto = true
+
+[features]
+seccomp = ["rustjail/seccomp"]

src/agent/LICENSE

@@ -1,202 +0,0 @@
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-

src/agent/Makefile

@@ -27,6 +27,20 @@ COMMIT_MSG = $(if $(COMMIT),$(COMMIT),unknown)
 # Exported to allow cargo to see it
 export VERSION_COMMIT := $(if $(COMMIT),$(VERSION)-$(COMMIT),$(VERSION))
 
+EXTRA_RUSTFEATURES :=
+
+##VAR SECCOMP=yes|no define if agent enables seccomp feature
+SECCOMP := yes
+
+# Enable seccomp feature of rust build
+ifeq ($(SECCOMP),yes)
+    override EXTRA_RUSTFEATURES += seccomp
+endif
+
+ifneq ($(EXTRA_RUSTFEATURES),)
+    override EXTRA_RUSTFEATURES := --features $(EXTRA_RUSTFEATURES)
+endif
+
 include ../../utils.mk
 
 TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET)
@@ -90,15 +104,14 @@ default: $(TARGET) show-header
 $(TARGET): $(GENERATED_CODE) $(TARGET_PATH)
 
 $(TARGET_PATH): $(SOURCES) | show-summary
-	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE)
+	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
 
 $(GENERATED_FILES): %: %.in
 	@sed $(foreach r,$(GENERATED_REPLACEMENTS),-e 's|@$r@|$($r)|g') "$<" > "$@"
 
 ##TARGET optimize: optimized  build
 optimize: $(SOURCES) | show-summary show-header
-	@RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny-warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE)
-
+	@RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny-warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
 
 ##TARGET clippy: run clippy linter
 clippy: $(GENERATED_CODE)
@@ -127,7 +140,7 @@ vendor:
 
 #TARGET test: run cargo tests
 test:
-	@cargo test --all --target $(TRIPLE)
+	@cargo test --all --target $(TRIPLE) $(EXTRA_RUSTFEATURES) -- --nocapture
 
 ##TARGET check: run test
 check: clippy format

src/agent/README.md

@@ -19,6 +19,7 @@ After that, we drafted the initial code here, and any contributions are welcome.
 | I/O stream              | :white_check_mark: |
 | Cgroups                 | :white_check_mark: |
 | Capabilities, `rlimit`, readonly path, masked path, users | :white_check_mark: |
+| Seccomp                 | :white_check_mark: |
 | container stats (`stats_container`)                     | :white_check_mark: |
 | Hooks                   | :white_check_mark: |
 | **Agent Features & APIs** |

src/agent/oci/Cargo.toml

@@ -5,7 +5,7 @@ authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
 edition = "2018"
 
 [dependencies]
-serde = "1.0.91"
-serde_derive = "1.0.91"
-serde_json = "1.0.39"
-libc = "0.2.58"
+serde = "1.0.131"
+serde_derive = "1.0.131"
+serde_json = "1.0.73"
+libc = "0.2.112"

src/agent/protocols/protos/agent.proto

@@ -66,6 +66,7 @@ service AgentService {
 	rpc SetGuestDateTime(SetGuestDateTimeRequest) returns (google.protobuf.Empty);
 	rpc CopyFile(CopyFileRequest) returns (google.protobuf.Empty);
 	rpc GetOOMEvent(GetOOMEventRequest) returns (OOMEvent);
+	rpc AddSwap(AddSwapRequest) returns (google.protobuf.Empty);
 }
 
 message CreateContainerRequest {
@@ -503,6 +504,10 @@ message OOMEvent {
 	string container_id = 1;
 }
 
+message AddSwapRequest {
+	repeated uint32 PCIPath = 1;
+}
+
 message GetMetricsRequest {}
 
 message Metrics {

src/agent/protocols/protos/types.proto

@@ -46,6 +46,7 @@ message Route {
 	string device = 3;
 	string source = 4;
 	uint32 scope = 5;
+	IPFamily family = 6;
 }
 
 message ARPNeighbor {

src/agent/rustjail/Cargo.toml

@@ -23,14 +23,18 @@ scan_fmt = "0.2"
 regex = "1.1"
 path-absolutize = "1.2.0"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.2.5" }
+cgroups = { package = "cgroups-rs", version = "0.2.8" }
 rlimit = "0.5.3"
 
 tokio = { version = "1.2.0", features = ["sync", "io-util", "process", "time", "macros"] }
 futures = "0.3"
 async-trait = "0.1.31"
 inotify = "0.9.2"
+libseccomp = { version = "0.1.3", optional = true }
 
 [dev-dependencies]
 serial_test = "0.5.0"
 tempfile = "3.1.0"
+
+[features]
+seccomp = ["libseccomp"]

src/agent/rustjail/src/cgroups/fs/mod.rs

@@ -232,19 +232,19 @@ fn set_devices_resources(
     let mut devices = vec![];
 
     for d in device_resources.iter() {
-        if let Some(dev) = linux_device_group_to_cgroup_device(&d) {
+        if let Some(dev) = linux_device_group_to_cgroup_device(d) {
             devices.push(dev);
         }
     }
 
     for d in DEFAULT_DEVICES.iter() {
-        if let Some(dev) = linux_device_to_cgroup_device(&d) {
+        if let Some(dev) = linux_device_to_cgroup_device(d) {
             devices.push(dev);
         }
     }
 
     for d in DEFAULT_ALLOWED_DEVICES.iter() {
-        if let Some(dev) = linux_device_group_to_cgroup_device(&d) {
+        if let Some(dev) = linux_device_group_to_cgroup_device(d) {
             devices.push(dev);
         }
     }
@@ -828,7 +828,7 @@ fn get_blkio_stats_v2(cg: &cgroups::Cgroup) -> SingularPtrField<BlkioStats> {
 
 fn get_blkio_stats(cg: &cgroups::Cgroup) -> SingularPtrField<BlkioStats> {
     if cg.v2() {
-        return get_blkio_stats_v2(&cg);
+        return get_blkio_stats_v2(cg);
     }
 
     let blkio_controller: &BlkIoController = get_controller_or_return_singular_none!(cg);
@@ -1022,7 +1022,7 @@ impl Manager {
                 .unwrap()
                 .trim_start_matches(root_path.to_str().unwrap());
             info!(sl!(), "updating cpuset for parent path {:?}", &r_path);
-            let cg = new_cgroup(cgroups::hierarchies::auto(), &r_path);
+            let cg = new_cgroup(cgroups::hierarchies::auto(), r_path);
             let cpuset_controller: &CpuSetController = cg.controller_of().unwrap();
             cpuset_controller.set_cpus(guest_cpuset)?;
         }

src/agent/rustjail/src/container.rs

@@ -25,6 +25,8 @@ use crate::cgroups::mock::Manager as FsManager;
 use crate::cgroups::Manager;
 use crate::log_child;
 use crate::process::Process;
+#[cfg(feature = "seccomp")]
+use crate::seccomp;
 use crate::specconv::CreateOpts;
 use crate::{mount, validator};
 
@@ -151,7 +153,7 @@ lazy_static! {
             },
             LinuxDevice {
                 path: "/dev/full".to_string(),
-                r#type: String::from("c"),
+                r#type: "c".to_string(),
                 major: 1,
                 minor: 7,
                 file_mode: Some(0o666),
@@ -390,7 +392,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     let linux = spec.linux.as_ref().unwrap();
 
     // get namespace vector to join/new
-    let nses = get_namespaces(&linux);
+    let nses = get_namespaces(linux);
 
     let mut userns = false;
     let mut to_new = CloneFlags::empty();
@@ -561,7 +563,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     }
 
     if to_new.contains(CloneFlags::CLONE_NEWNS) {
-        mount::finish_rootfs(cfd_log, &spec)?;
+        mount::finish_rootfs(cfd_log, &spec, &oci_process)?;
     }
 
     if !oci_process.cwd.is_empty() {
@@ -593,11 +595,22 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         })?;
     }
 
-    // NoNewPeiviledges, Drop capabilities
+    // NoNewPrivileges
     if oci_process.no_new_privileges {
         capctl::prctl::set_no_new_privs().map_err(|_| anyhow!("cannot set no new privileges"))?;
     }
 
+    // Without NoNewPrivileges, we need to set seccomp
+    // before dropping capabilities because the calling thread
+    // must have the CAP_SYS_ADMIN.
+    #[cfg(feature = "seccomp")]
+    if !oci_process.no_new_privileges {
+        if let Some(ref scmp) = linux.seccomp {
+            seccomp::init_seccomp(scmp)?;
+        }
+    }
+
+    // Drop capabilities
     if oci_process.capabilities.is_some() {
         let c = oci_process.capabilities.as_ref().unwrap();
         capabilities::drop_privileges(cfd_log, c)?;
@@ -641,7 +654,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     let exec_file = Path::new(&args[0]);
     log_child!(cfd_log, "process command: {:?}", &args);
     if !exec_file.exists() {
-        find_file(exec_file).ok_or_else(|| anyhow!("the file {} is not exist", &args[0]))?;
+        find_file(exec_file).ok_or_else(|| anyhow!("the file {} was not found", &args[0]))?;
     }
 
     // notify parent that the child's ready to start
@@ -669,6 +682,16 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         unistd::read(fd, &mut buf)?;
     }
 
+    // With NoNewPrivileges, we should set seccomp as close to
+    // do_exec as possible in order to reduce the amount of
+    // system calls in the seccomp profiles.
+    #[cfg(feature = "seccomp")]
+    if oci_process.no_new_privileges {
+        if let Some(ref scmp) = linux.seccomp {
+            seccomp::init_seccomp(scmp)?;
+        }
+    }
+
     do_exec(&args);
 }
 
@@ -833,6 +856,20 @@ impl BaseContainer for LinuxContainer {
         }
         let linux = spec.linux.as_ref().unwrap();
 
+        if p.oci.capabilities.is_none() {
+            // No capabilities, inherit from container process
+            let process = spec
+                .process
+                .as_ref()
+                .ok_or_else(|| anyhow!("no process config"))?;
+            p.oci.capabilities = Some(
+                process
+                    .capabilities
+                    .clone()
+                    .ok_or_else(|| anyhow!("missing process capabilities"))?,
+            );
+        }
+
         let (pfd_log, cfd_log) = unistd::pipe().context("failed to create pipe")?;
 
         let _ = fcntl::fcntl(pfd_log, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC))
@@ -939,7 +976,7 @@ impl BaseContainer for LinuxContainer {
 
         join_namespaces(
             &logger,
-            &spec,
+            spec,
             &p,
             self.cgroup_manager.as_ref().unwrap(),
             &st,
@@ -1031,7 +1068,7 @@ impl BaseContainer for LinuxContainer {
         let fifo = format!("{}/{}", &self.root, EXEC_FIFO_FILENAME);
         let fd = fcntl::open(fifo.as_str(), OFlag::O_WRONLY, Mode::from_bits_truncate(0))?;
         let data: &[u8] = &[0];
-        unistd::write(fd, &data)?;
+        unistd::write(fd, data)?;
         info!(self.logger, "container started");
         self.init_process_start_time = SystemTime::now()
             .duration_since(SystemTime::UNIX_EPOCH)

src/agent/rustjail/src/lib.rs

@@ -34,6 +34,8 @@ pub mod container;
 pub mod mount;
 pub mod pipestream;
 pub mod process;
+#[cfg(feature = "seccomp")]
+pub mod seccomp;
 pub mod specconv;
 pub mod sync;
 pub mod sync_with_async;

src/agent/rustjail/src/mount.rs

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use anyhow::{anyhow, bail, Context, Result};
+use anyhow::{anyhow, Context, Result};
 use libc::uid_t;
 use nix::errno::Errno;
 use nix::fcntl::{self, OFlag};
@@ -13,13 +13,13 @@ use nix::mount::{MntFlags, MsFlags};
 use nix::sys::stat::{self, Mode, SFlag};
 use nix::unistd::{self, Gid, Uid};
 use nix::NixPath;
-use oci::{LinuxDevice, Mount, Spec};
+use oci::{LinuxDevice, Mount, Process, Spec};
 use std::collections::{HashMap, HashSet};
 use std::fs::{self, OpenOptions};
 use std::mem::MaybeUninit;
 use std::os::unix;
 use std::os::unix::io::RawFd;
-use std::path::{Path, PathBuf};
+use std::path::{Component, Path, PathBuf};
 
 use path_absolutize::*;
 use std::fs::File;
@@ -189,7 +189,7 @@ pub fn init_rootfs(
 
     let mut bind_mount_dev = false;
     for m in &spec.mounts {
-        let (mut flags, pgflags, data) = parse_mount(&m);
+        let (mut flags, pgflags, data) = parse_mount(m);
         if !m.destination.starts_with('/') || m.destination.contains("..") {
             return Err(anyhow!(
                 "the mount destination {} is invalid",
@@ -198,7 +198,7 @@ pub fn init_rootfs(
         }
 
         if m.r#type == "cgroup" {
-            mount_cgroups(cfd_log, &m, rootfs, flags, &data, cpath, mounts)?;
+            mount_cgroups(cfd_log, m, rootfs, flags, &data, cpath, mounts)?;
         } else {
             if m.destination == "/dev" {
                 if m.r#type == "bind" {
@@ -226,7 +226,7 @@ pub fn init_rootfs(
                 }
             }
 
-            mount_from(cfd_log, &m, &rootfs, flags, &data, "")?;
+            mount_from(cfd_log, m, rootfs, flags, &data, "")?;
             // bind mount won't change mount options, we need remount to make mount options
             // effective.
             // first check that we have non-default options required before attempting a
@@ -356,7 +356,7 @@ fn mount_cgroups(
     mounts: &HashMap<String, String>,
 ) -> Result<()> {
     if cgroups::hierarchies::is_cgroup2_unified_mode() {
-        return mount_cgroups_v2(cfd_log, &m, rootfs, flags);
+        return mount_cgroups_v2(cfd_log, m, rootfs, flags);
     }
     // mount tmpfs
     let ctm = Mount {
@@ -745,7 +745,7 @@ fn mount_from(
         let _ = fs::create_dir_all(&dir).map_err(|e| {
             log_child!(
                 cfd_log,
-                "creat dir {}: {}",
+                "create dir {}: {}",
                 dir.to_str().unwrap(),
                 e.to_string()
             )
@@ -828,18 +828,35 @@ fn default_symlinks() -> Result<()> {
     }
     Ok(())
 }
+
+fn dev_rel_path(path: &str) -> Option<&Path> {
+    let path = Path::new(path);
+
+    if !path.starts_with("/dev")
+        || path == Path::new("/dev")
+        || path.components().any(|c| c == Component::ParentDir)
+    {
+        return None;
+    }
+    path.strip_prefix("/").ok()
+}
+
 fn create_devices(devices: &[LinuxDevice], bind: bool) -> Result<()> {
-    let op: fn(&LinuxDevice) -> Result<()> = if bind { bind_dev } else { mknod_dev };
+    let op: fn(&LinuxDevice, &Path) -> Result<()> = if bind { bind_dev } else { mknod_dev };
     let old = stat::umask(Mode::from_bits_truncate(0o000));
     for dev in DEFAULT_DEVICES.iter() {
-        op(dev)?;
+        let path = Path::new(&dev.path[1..]);
+        op(dev, path).context(format!("Creating container device {:?}", dev))?;
     }
     for dev in devices {
-        if !dev.path.starts_with("/dev") || dev.path.contains("..") {
+        let path = dev_rel_path(&dev.path).ok_or_else(|| {
             let msg = format!("{} is not a valid device path", dev.path);
-            bail!(anyhow!(msg));
+            anyhow!(msg)
+        })?;
+        if let Some(dir) = path.parent() {
+            fs::create_dir_all(dir).context(format!("Creating container device {:?}", dev))?;
         }
-        op(dev)?;
+        op(dev, path).context(format!("Creating container device {:?}", dev))?;
     }
     stat::umask(old);
     Ok(())
@@ -861,21 +878,21 @@ lazy_static! {
     };
 }
 
-fn mknod_dev(dev: &LinuxDevice) -> Result<()> {
+fn mknod_dev(dev: &LinuxDevice, relpath: &Path) -> Result<()> {
     let f = match LINUXDEVICETYPE.get(dev.r#type.as_str()) {
         Some(v) => v,
         None => return Err(anyhow!("invalid spec".to_string())),
     };
 
     stat::mknod(
-        &dev.path[1..],
+        relpath,
         *f,
         Mode::from_bits_truncate(dev.file_mode.unwrap_or(0)),
         nix::sys::stat::makedev(dev.major as u64, dev.minor as u64),
     )?;
 
     unistd::chown(
-        &dev.path[1..],
+        relpath,
         Some(Uid::from_raw(dev.uid.unwrap_or(0) as uid_t)),
         Some(Gid::from_raw(dev.gid.unwrap_or(0) as uid_t)),
     )?;
@@ -883,9 +900,9 @@ fn mknod_dev(dev: &LinuxDevice) -> Result<()> {
     Ok(())
 }
 
-fn bind_dev(dev: &LinuxDevice) -> Result<()> {
+fn bind_dev(dev: &LinuxDevice, relpath: &Path) -> Result<()> {
     let fd = fcntl::open(
-        &dev.path[1..],
+        relpath,
         OFlag::O_RDWR | OFlag::O_CREAT,
         Mode::from_bits_truncate(0o644),
     )?;
@@ -894,7 +911,7 @@ fn bind_dev(dev: &LinuxDevice) -> Result<()> {
 
     mount(
         Some(&*dev.path),
-        &dev.path[1..],
+        relpath,
         None::<&str>,
         MsFlags::MS_BIND,
         None::<&str>,
@@ -902,10 +919,21 @@ fn bind_dev(dev: &LinuxDevice) -> Result<()> {
     Ok(())
 }
 
-pub fn finish_rootfs(cfd_log: RawFd, spec: &Spec) -> Result<()> {
+pub fn finish_rootfs(cfd_log: RawFd, spec: &Spec, process: &Process) -> Result<()> {
     let olddir = unistd::getcwd()?;
     log_child!(cfd_log, "old cwd: {}", olddir.to_str().unwrap());
     unistd::chdir("/")?;
+
+    if !process.cwd.is_empty() {
+        // Although the process.cwd string can be unclean/malicious (../../dev, etc),
+        // we are running on our own mount namespace and we just chrooted into the
+        // container's root. It's safe to create CWD from there.
+        log_child!(cfd_log, "Creating CWD {}", process.cwd.as_str());
+        // Unconditionally try to create CWD, create_dir_all will not fail if
+        // it already exists.
+        fs::create_dir_all(process.cwd.as_str())?;
+    }
+
     if spec.linux.is_some() {
         let linux = spec.linux.as_ref().unwrap();
 
@@ -1211,7 +1239,7 @@ mod tests {
             options: vec!["ro".to_string(), "shared".to_string()],
         }];
 
-        let ret = finish_rootfs(stdout_fd, &spec);
+        let ret = finish_rootfs(stdout_fd, &spec, &oci::Process::default());
         assert!(ret.is_ok(), "Should pass. Got: {:?}", ret);
     }
 
@@ -1247,11 +1275,12 @@ mod tests {
             uid: Some(unistd::getuid().as_raw()),
             gid: Some(unistd::getgid().as_raw()),
         };
+        let path = Path::new("fifo");
 
-        let ret = mknod_dev(&dev);
+        let ret = mknod_dev(&dev, path);
         assert!(ret.is_ok(), "Should pass. Got: {:?}", ret);
 
-        let ret = stat::stat("fifo");
+        let ret = stat::stat(path);
         assert!(ret.is_ok(), "Should pass. Got: {:?}", ret);
     }
     #[test]
@@ -1368,4 +1397,26 @@ mod tests {
             assert!(result == t.result, "{}", msg);
         }
     }
+
+    #[test]
+    fn test_dev_rel_path() {
+        // Valid device paths
+        assert_eq!(dev_rel_path("/dev/sda").unwrap(), Path::new("dev/sda"));
+        assert_eq!(dev_rel_path("//dev/sda").unwrap(), Path::new("dev/sda"));
+        assert_eq!(
+            dev_rel_path("/dev/vfio/99").unwrap(),
+            Path::new("dev/vfio/99")
+        );
+        assert_eq!(dev_rel_path("/dev/...").unwrap(), Path::new("dev/..."));
+        assert_eq!(dev_rel_path("/dev/a..b").unwrap(), Path::new("dev/a..b"));
+        assert_eq!(dev_rel_path("/dev//foo").unwrap(), Path::new("dev/foo"));
+
+        // Bad device paths
+        assert!(dev_rel_path("/devfoo").is_none());
+        assert!(dev_rel_path("/etc/passwd").is_none());
+        assert!(dev_rel_path("/dev/../etc/passwd").is_none());
+        assert!(dev_rel_path("dev/foo").is_none());
+        assert!(dev_rel_path("").is_none());
+        assert!(dev_rel_path("/dev").is_none());
+    }
 }

src/agent/rustjail/src/process.rs

@@ -24,6 +24,16 @@ use tokio::io::{split, ReadHalf, WriteHalf};
 use tokio::sync::Mutex;
 use tokio::sync::Notify;
 
+macro_rules! close_process_stream {
+    ($self: ident, $stream:ident, $stream_type: ident) => {
+        if $self.$stream.is_some() {
+            $self.close_stream(StreamType::$stream_type);
+            let _ = unistd::close($self.$stream.unwrap());
+            $self.$stream = None;
+        }
+    };
+}
+
 #[derive(Debug, PartialEq, Eq, Hash, Clone)]
 pub enum StreamType {
     Stdin,
@@ -147,6 +157,22 @@ impl Process {
         notify.notify_one();
     }
 
+    pub fn close_stdin(&mut self) {
+        close_process_stream!(self, term_master, TermMaster);
+        close_process_stream!(self, parent_stdin, ParentStdin);
+
+        self.notify_term_close();
+    }
+
+    pub fn cleanup_process_stream(&mut self) {
+        close_process_stream!(self, parent_stdin, ParentStdin);
+        close_process_stream!(self, parent_stdout, ParentStdout);
+        close_process_stream!(self, parent_stderr, ParentStderr);
+        close_process_stream!(self, term_master, TermMaster);
+
+        self.notify_term_close();
+    }
+
     fn get_fd(&self, stream_type: &StreamType) -> Option<RawFd> {
         match stream_type {
             StreamType::Stdin => self.stdin,

src/agent/rustjail/src/seccomp.rs

@@ -0,0 +1,237 @@
+// Copyright 2021 Sony Group Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use anyhow::{anyhow, Result};
+use libseccomp::*;
+use oci::{LinuxSeccomp, LinuxSeccompArg};
+use std::str::FromStr;
+
+fn get_filter_attr_from_flag(flag: &str) -> Result<ScmpFilterAttr> {
+    match flag {
+        "SECCOMP_FILTER_FLAG_TSYNC" => Ok(ScmpFilterAttr::CtlTsync),
+        "SECCOMP_FILTER_FLAG_LOG" => Ok(ScmpFilterAttr::CtlLog),
+        "SECCOMP_FILTER_FLAG_SPEC_ALLOW" => Ok(ScmpFilterAttr::CtlSsb),
+        _ => Err(anyhow!("Invalid seccomp flag")),
+    }
+}
+
+// get_rule_conditions gets rule conditions for a system call from the args.
+fn get_rule_conditions(args: &[LinuxSeccompArg]) -> Result<Vec<ScmpArgCompare>> {
+    let mut conditions: Vec<ScmpArgCompare> = Vec::new();
+
+    for arg in args {
+        if arg.op.is_empty() {
+            return Err(anyhow!("seccomp opreator is required"));
+        }
+
+        let cond = ScmpArgCompare::new(
+            arg.index,
+            ScmpCompareOp::from_str(&arg.op)?,
+            arg.value,
+            Some(arg.value_two),
+        );
+
+        conditions.push(cond);
+    }
+
+    Ok(conditions)
+}
+
+// init_seccomp creates a seccomp filter and loads it for the current process
+// including all the child processes.
+pub fn init_seccomp(scmp: &LinuxSeccomp) -> Result<()> {
+    let def_action = ScmpAction::from_str(scmp.default_action.as_str(), Some(libc::EPERM as u32))?;
+
+    // Create a new filter context
+    let mut filter = ScmpFilterContext::new_filter(def_action)?;
+
+    // Add extra architectures
+    for arch in &scmp.architectures {
+        let scmp_arch = ScmpArch::from_str(arch)?;
+        filter.add_arch(scmp_arch)?;
+    }
+
+    // Unset no new privileges bit
+    filter.set_no_new_privs_bit(false)?;
+
+    // Add a rule for each system call
+    for syscall in &scmp.syscalls {
+        if syscall.names.is_empty() {
+            return Err(anyhow!("syscall name is required"));
+        }
+
+        let action = ScmpAction::from_str(&syscall.action, Some(syscall.errno_ret))?;
+        if action == def_action {
+            continue;
+        }
+
+        for name in &syscall.names {
+            let syscall_num = get_syscall_from_name(name, None)?;
+
+            if syscall.args.is_empty() {
+                filter.add_rule(action, syscall_num, None)?;
+            } else {
+                let conditions = get_rule_conditions(&syscall.args)?;
+                filter.add_rule(action, syscall_num, Some(&conditions))?;
+            }
+        }
+    }
+
+    // Set filter attributes for each seccomp flag
+    for flag in &scmp.flags {
+        let scmp_attr = get_filter_attr_from_flag(flag)?;
+        filter.set_filter_attr(scmp_attr, 1)?;
+    }
+
+    // Load the filter
+    filter.load()?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::skip_if_not_root;
+    use libc::{dup3, process_vm_readv, EPERM, O_CLOEXEC};
+    use std::io::Error;
+    use std::ptr::null;
+
+    macro_rules! syscall_assert {
+        ($e1: expr, $e2: expr) => {
+            let mut errno: i32 = 0;
+            if $e1 < 0 {
+                errno = -Error::last_os_error().raw_os_error().unwrap();
+            }
+            assert_eq!(errno, $e2);
+        };
+    }
+
+    #[test]
+    fn test_get_filter_attr_from_flag() {
+        skip_if_not_root!();
+
+        assert_eq!(
+            get_filter_attr_from_flag("SECCOMP_FILTER_FLAG_TSYNC").unwrap(),
+            ScmpFilterAttr::CtlTsync
+        );
+
+        assert_eq!(get_filter_attr_from_flag("ERROR").is_err(), true);
+    }
+
+    #[test]
+    fn test_init_seccomp() {
+        skip_if_not_root!();
+
+        let data = r#"{
+            "defaultAction": "SCMP_ACT_ALLOW",
+            "architectures": [
+            ],
+            "flags": [
+                "SECCOMP_FILTER_FLAG_LOG"
+            ],
+            "syscalls": [
+                {
+                   "names": [
+                        "dup3"
+                    ],
+                    "action": "SCMP_ACT_ERRNO"
+                },
+                {
+                   "names": [
+                        "process_vm_readv"
+                    ],
+                    "action": "SCMP_ACT_ERRNO",
+                    "errnoRet": 111,
+                    "args": [
+                        {
+                            "index": 0,
+                            "value": 10,
+                            "op": "SCMP_CMP_EQ"
+                        }
+                    ]
+                },
+                {
+                   "names": [
+                        "process_vm_readv"
+                    ],
+                    "action": "SCMP_ACT_ERRNO",
+                    "errnoRet": 111,
+                    "args": [
+                        {
+                            "index": 0,
+                            "value": 20,
+                            "op": "SCMP_CMP_EQ"
+                        }
+                    ]
+                },
+                {
+                   "names": [
+                        "process_vm_readv"
+                    ],
+                    "action": "SCMP_ACT_ERRNO",
+                    "errnoRet": 222,
+                    "args": [
+                        {
+                            "index": 0,
+                            "value": 30,
+                            "op": "SCMP_CMP_EQ"
+                        },
+                        {
+                            "index": 2,
+                            "value": 40,
+                            "op": "SCMP_CMP_EQ"
+                        }
+                    ]
+                }
+            ]
+        }"#;
+
+        let mut scmp: oci::LinuxSeccomp = serde_json::from_str(data).unwrap();
+        let mut arch: Vec<oci::Arch>;
+
+        if cfg!(target_endian = "little") {
+            // For little-endian architectures
+            arch = vec![
+                "SCMP_ARCH_X86".to_string(),
+                "SCMP_ARCH_X32".to_string(),
+                "SCMP_ARCH_X86_64".to_string(),
+                "SCMP_ARCH_AARCH64".to_string(),
+                "SCMP_ARCH_ARM".to_string(),
+                "SCMP_ARCH_PPC64LE".to_string(),
+            ];
+        } else {
+            // For big-endian architectures
+            arch = vec!["SCMP_ARCH_S390X".to_string()];
+        }
+
+        scmp.architectures.append(&mut arch);
+
+        init_seccomp(&scmp).unwrap();
+
+        // Basic syscall with simple rule
+        syscall_assert!(unsafe { dup3(0, 1, O_CLOEXEC) }, -EPERM);
+
+        // Syscall with permitted arguments
+        syscall_assert!(unsafe { process_vm_readv(1, null(), 0, null(), 0, 0) }, 0);
+
+        // Multiple arguments with OR rules with ERRNO
+        syscall_assert!(
+            unsafe { process_vm_readv(10, null(), 0, null(), 0, 0) },
+            -111
+        );
+        syscall_assert!(
+            unsafe { process_vm_readv(20, null(), 0, null(), 0, 0) },
+            -111
+        );
+
+        // Multiple arguments with AND rules with ERRNO
+        syscall_assert!(unsafe { process_vm_readv(30, null(), 0, null(), 0, 0) }, 0);
+        syscall_assert!(
+            unsafe { process_vm_readv(30, null(), 40, null(), 0, 0) },
+            -222
+        );
+    }
+}

src/agent/rustjail/src/validator.rs

@@ -266,7 +266,7 @@ pub fn validate(conf: &Config) -> Result<()> {
     security(oci).context("security")?;
     usernamespace(oci).context("usernamespace")?;
     cgroupnamespace(oci).context("cgroupnamespace")?;
-    sysctl(&oci).context("sysctl")?;
+    sysctl(oci).context("sysctl")?;
 
     if conf.rootless_euid {
         rootless_euid(oci).context("rootless euid")?;

src/agent/samples/configuration-all-endpoints.toml

@@ -0,0 +1,41 @@
+# This is an agent configuration file example.
+dev_mode = true
+server_addr = 'vsock://8:2048'
+
+[endpoints]
+# All endpoints are allowed
+allowed = [
+        "AddARPNeighborsRequest",
+        "AddSwapRequest",
+        "CloseStdinRequest",
+        "CopyFileRequest",
+        "CreateContainerRequest",
+        "CreateSandboxRequest",
+        "DestroySandboxRequest",
+        "ExecProcessRequest",
+        "GetMetricsRequest",
+        "GetOOMEventRequest",
+        "GuestDetailsRequest",
+        "ListInterfacesRequest",
+        "ListRoutesRequest",
+        "MemHotplugByProbeRequest",
+        "OnlineCPUMemRequest",
+        "PauseContainerRequest",
+        "PullImageRequest",
+        "ReadStreamRequest",
+        "RemoveContainerRequest",
+        "ReseedRandomDevRequest",
+        "ResumeContainerRequest",
+        "SetGuestDateTimeRequest",
+        "SignalProcessRequest",
+        "StartContainerRequest",
+        "StartTracingRequest",
+        "StatsContainerRequest",
+        "StopTracingRequest",
+        "TtyWinResizeRequest",
+        "UpdateContainerRequest",
+        "UpdateInterfaceRequest",
+        "UpdateRoutesRequest",
+        "WaitProcessRequest",
+        "WriteStreamRequest"
+]

src/agent/src/config.rs

@@ -2,10 +2,13 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 //
-use crate::tracer;
+use crate::rpc;
 use anyhow::{bail, ensure, Context, Result};
+use serde::Deserialize;
+use std::collections::HashSet;
 use std::env;
 use std::fs;
+use std::str::FromStr;
 use std::time;
 use tracing::instrument;
 
@@ -19,6 +22,7 @@ const DEBUG_CONSOLE_VPORT_OPTION: &str = "agent.debug_console_vport";
 const LOG_VPORT_OPTION: &str = "agent.log_vport";
 const CONTAINER_PIPE_SIZE_OPTION: &str = "agent.container_pipe_size";
 const UNIFIED_CGROUP_HIERARCHY_OPTION: &str = "agent.unified_cgroup_hierarchy";
+const CONFIG_FILE: &str = "agent.config_file";
 
 const DEFAULT_LOG_LEVEL: slog::Level = slog::Level::Info;
 const DEFAULT_HOTPLUG_TIMEOUT: time::Duration = time::Duration::from_secs(3);
@@ -29,7 +33,7 @@ const VSOCK_PORT: u16 = 1024;
 // Environment variables used for development and testing
 const SERVER_ADDR_ENV_VAR: &str = "KATA_AGENT_SERVER_ADDR";
 const LOG_LEVEL_ENV_VAR: &str = "KATA_AGENT_LOG_LEVEL";
-const TRACE_TYPE_ENV_VAR: &str = "KATA_AGENT_TRACE_TYPE";
+const TRACING_ENV_VAR: &str = "KATA_AGENT_TRACING";
 
 const ERR_INVALID_LOG_LEVEL: &str = "invalid log level";
 const ERR_INVALID_LOG_LEVEL_PARAM: &str = "invalid log level parameter";
@@ -47,6 +51,17 @@ const ERR_INVALID_CONTAINER_PIPE_SIZE_PARAM: &str = "unable to parse container p
 const ERR_INVALID_CONTAINER_PIPE_SIZE_KEY: &str = "invalid container pipe size key name";
 const ERR_INVALID_CONTAINER_PIPE_NEGATIVE: &str = "container pipe size should not be negative";
 
+#[derive(Debug, Default, Deserialize)]
+pub struct EndpointsConfig {
+    pub allowed: Vec<String>,
+}
+
+#[derive(Debug, Default)]
+pub struct AgentEndpoints {
+    pub allowed: HashSet<String>,
+    pub all_allowed: bool,
+}
+
 #[derive(Debug)]
 pub struct AgentConfig {
     pub debug_console: bool,
@@ -58,7 +73,38 @@ pub struct AgentConfig {
     pub container_pipe_size: i32,
     pub server_addr: String,
     pub unified_cgroup_hierarchy: bool,
-    pub tracing: tracer::TraceType,
+    pub tracing: bool,
+    pub endpoints: AgentEndpoints,
+    pub supports_seccomp: bool,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct AgentConfigBuilder {
+    pub debug_console: Option<bool>,
+    pub dev_mode: Option<bool>,
+    pub log_level: Option<String>,
+    pub hotplug_timeout: Option<time::Duration>,
+    pub debug_console_vport: Option<i32>,
+    pub log_vport: Option<i32>,
+    pub container_pipe_size: Option<i32>,
+    pub server_addr: Option<String>,
+    pub unified_cgroup_hierarchy: Option<bool>,
+    pub tracing: Option<bool>,
+    pub endpoints: Option<EndpointsConfig>,
+}
+
+macro_rules! config_override {
+    ($builder:ident, $config:ident, $field:ident) => {
+        if let Some(v) = $builder.$field {
+            $config.$field = v;
+        }
+    };
+
+    ($builder:ident, $config:ident, $field:ident, $func: ident) => {
+        if let Some(v) = $builder.$field {
+            $config.$field = $func(&v)?;
+        }
+    };
 }
 
 // parse_cmdline_param parse commandline parameters.
@@ -91,8 +137,8 @@ macro_rules! parse_cmdline_param {
     };
 }
 
-impl AgentConfig {
-    pub fn new() -> AgentConfig {
+impl Default for AgentConfig {
+    fn default() -> Self {
         AgentConfig {
             debug_console: false,
             dev_mode: false,
@@ -103,34 +149,84 @@ impl AgentConfig {
             container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
             server_addr: format!("{}:{}", VSOCK_ADDR, VSOCK_PORT),
             unified_cgroup_hierarchy: false,
-            tracing: tracer::TraceType::Disabled,
+            tracing: false,
+            endpoints: Default::default(),
+            supports_seccomp: rpc::have_seccomp(),
         }
     }
+}
 
+impl FromStr for AgentConfig {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let agent_config_builder: AgentConfigBuilder =
+            toml::from_str(s).map_err(anyhow::Error::new)?;
+        let mut agent_config: AgentConfig = Default::default();
+
+        // Overwrite default values with the configuration files ones.
+        config_override!(agent_config_builder, agent_config, debug_console);
+        config_override!(agent_config_builder, agent_config, dev_mode);
+        config_override!(
+            agent_config_builder,
+            agent_config,
+            log_level,
+            logrus_to_slog_level
+        );
+        config_override!(agent_config_builder, agent_config, hotplug_timeout);
+        config_override!(agent_config_builder, agent_config, debug_console_vport);
+        config_override!(agent_config_builder, agent_config, log_vport);
+        config_override!(agent_config_builder, agent_config, container_pipe_size);
+        config_override!(agent_config_builder, agent_config, server_addr);
+        config_override!(agent_config_builder, agent_config, unified_cgroup_hierarchy);
+        config_override!(agent_config_builder, agent_config, tracing);
+
+        // Populate the allowed endpoints hash set, if we got any from the config file.
+        if let Some(endpoints) = agent_config_builder.endpoints {
+            for ep in endpoints.allowed {
+                agent_config.endpoints.allowed.insert(ep);
+            }
+        }
+
+        Ok(agent_config)
+    }
+}
+
+impl AgentConfig {
     #[instrument]
-    pub fn parse_cmdline(&mut self, file: &str) -> Result<()> {
+    pub fn from_cmdline(file: &str) -> Result<AgentConfig> {
+        let mut config: AgentConfig = Default::default();
         let cmdline = fs::read_to_string(file)?;
         let params: Vec<&str> = cmdline.split_ascii_whitespace().collect();
         for param in params.iter() {
+            // If we get a configuration file path from the command line, we
+            // generate our config from it.
+            // The agent will fail to start if the configuration file is not present,
+            // or if it can't be parsed properly.
+            if param.starts_with(format!("{}=", CONFIG_FILE).as_str()) {
+                let config_file = get_string_value(param)?;
+                return AgentConfig::from_config_file(&config_file);
+            }
+
             // parse cmdline flags
-            parse_cmdline_param!(param, DEBUG_CONSOLE_FLAG, self.debug_console);
-            parse_cmdline_param!(param, DEV_MODE_FLAG, self.dev_mode);
+            parse_cmdline_param!(param, DEBUG_CONSOLE_FLAG, config.debug_console);
+            parse_cmdline_param!(param, DEV_MODE_FLAG, config.dev_mode);
 
             // Support "bare" tracing option for backwards compatibility with
             // Kata 1.x.
             if param == &TRACE_MODE_OPTION {
-                self.tracing = tracer::TraceType::Isolated;
+                config.tracing = true;
                 continue;
             }
 
-            parse_cmdline_param!(param, TRACE_MODE_OPTION, self.tracing, get_trace_type);
+            parse_cmdline_param!(param, TRACE_MODE_OPTION, config.tracing, get_bool_value);
 
             // parse cmdline options
-            parse_cmdline_param!(param, LOG_LEVEL_OPTION, self.log_level, get_log_level);
+            parse_cmdline_param!(param, LOG_LEVEL_OPTION, config.log_level, get_log_level);
             parse_cmdline_param!(
                 param,
                 SERVER_ADDR_OPTION,
-                self.server_addr,
+                config.server_addr,
                 get_string_value
             );
 
@@ -138,7 +234,7 @@ impl AgentConfig {
             parse_cmdline_param!(
                 param,
                 HOTPLUG_TIMOUT_OPTION,
-                self.hotplug_timeout,
+                config.hotplug_timeout,
                 get_hotplug_timeout,
                 |hotplug_timeout: time::Duration| hotplug_timeout.as_secs() > 0
             );
@@ -147,14 +243,14 @@ impl AgentConfig {
             parse_cmdline_param!(
                 param,
                 DEBUG_CONSOLE_VPORT_OPTION,
-                self.debug_console_vport,
+                config.debug_console_vport,
                 get_vsock_port,
                 |port| port > 0
             );
             parse_cmdline_param!(
                 param,
                 LOG_VPORT_OPTION,
-                self.log_vport,
+                config.log_vport,
                 get_vsock_port,
                 |port| port > 0
             );
@@ -162,34 +258,47 @@ impl AgentConfig {
             parse_cmdline_param!(
                 param,
                 CONTAINER_PIPE_SIZE_OPTION,
-                self.container_pipe_size,
+                config.container_pipe_size,
                 get_container_pipe_size
             );
             parse_cmdline_param!(
                 param,
                 UNIFIED_CGROUP_HIERARCHY_OPTION,
-                self.unified_cgroup_hierarchy,
+                config.unified_cgroup_hierarchy,
                 get_bool_value
             );
         }
 
         if let Ok(addr) = env::var(SERVER_ADDR_ENV_VAR) {
-            self.server_addr = addr;
+            config.server_addr = addr;
         }
 
         if let Ok(addr) = env::var(LOG_LEVEL_ENV_VAR) {
             if let Ok(level) = logrus_to_slog_level(&addr) {
-                self.log_level = level;
+                config.log_level = level;
             }
         }
 
-        if let Ok(value) = env::var(TRACE_TYPE_ENV_VAR) {
-            if let Ok(result) = value.parse::<tracer::TraceType>() {
-                self.tracing = result;
-            }
+        if let Ok(value) = env::var(TRACING_ENV_VAR) {
+            let name_value = format!("{}={}", TRACING_ENV_VAR, value);
+
+            config.tracing = get_bool_value(&name_value)?;
         }
 
-        Ok(())
+        // We did not get a configuration file: allow all endpoints.
+        config.endpoints.all_allowed = true;
+
+        Ok(config)
+    }
+
+    #[instrument]
+    pub fn from_config_file(file: &str) -> Result<AgentConfig> {
+        let config = fs::read_to_string(file)?;
+        AgentConfig::from_str(&config)
+    }
+
+    pub fn is_allowed_endpoint(&self, ep: &str) -> bool {
+        self.endpoints.all_allowed || self.endpoints.allowed.contains(ep)
     }
 }
 
@@ -236,25 +345,6 @@ fn get_log_level(param: &str) -> Result<slog::Level> {
     logrus_to_slog_level(fields[1])
 }
 
-#[instrument]
-fn get_trace_type(param: &str) -> Result<tracer::TraceType> {
-    ensure!(!param.is_empty(), "invalid trace type parameter");
-
-    let fields: Vec<&str> = param.split('=').collect();
-    ensure!(
-        fields[0] == TRACE_MODE_OPTION,
-        "invalid trace type key name"
-    );
-
-    if fields.len() == 1 {
-        return Ok(tracer::TraceType::Isolated);
-    }
-
-    let result = fields[1].parse::<tracer::TraceType>()?;
-
-    Ok(result)
-}
-
 #[instrument]
 fn get_hotplug_timeout(param: &str) -> Result<time::Duration> {
     let fields: Vec<&str> = param.split('=').collect();
@@ -339,10 +429,6 @@ mod tests {
     use std::time;
     use tempfile::tempdir;
 
-    const ERR_INVALID_TRACE_TYPE_PARAM: &str = "invalid trace type parameter";
-    const ERR_INVALID_TRACE_TYPE: &str = "invalid trace type";
-    const ERR_INVALID_TRACE_TYPE_KEY: &str = "invalid trace type key name";
-
     // Parameters:
     //
     // 1: expected Result
@@ -371,15 +457,15 @@ mod tests {
 
     #[test]
     fn test_new() {
-        let config = AgentConfig::new();
-        assert_eq!(config.debug_console, false);
-        assert_eq!(config.dev_mode, false);
+        let config: AgentConfig = Default::default();
+        assert!(!config.debug_console);
+        assert!(!config.dev_mode);
         assert_eq!(config.log_level, DEFAULT_LOG_LEVEL);
         assert_eq!(config.hotplug_timeout, DEFAULT_HOTPLUG_TIMEOUT);
     }
 
     #[test]
-    fn test_parse_cmdline() {
+    fn test_from_cmdline() {
         const TEST_SERVER_ADDR: &str = "vsock://-1:1024";
 
         #[derive(Debug)]
@@ -393,7 +479,7 @@ mod tests {
             container_pipe_size: i32,
             server_addr: &'a str,
             unified_cgroup_hierarchy: bool,
-            tracing: tracer::TraceType,
+            tracing: bool,
         }
 
         impl Default for TestData<'_> {
@@ -408,7 +494,7 @@ mod tests {
                     container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
                     server_addr: TEST_SERVER_ADDR,
                     unified_cgroup_hierarchy: false,
-                    tracing: tracer::TraceType::Disabled,
+                    tracing: false,
                 }
             }
         }
@@ -667,64 +753,121 @@ mod tests {
             },
             TestData {
                 contents: "trace",
-                tracing: tracer::TraceType::Disabled,
+                tracing: false,
                 ..Default::default()
             },
             TestData {
                 contents: ".trace",
-                tracing: tracer::TraceType::Disabled,
+                tracing: false,
                 ..Default::default()
             },
             TestData {
                 contents: "agent.tracer",
-                tracing: tracer::TraceType::Disabled,
+                tracing: false,
                 ..Default::default()
             },
             TestData {
                 contents: "agent.trac",
-                tracing: tracer::TraceType::Disabled,
+                tracing: false,
                 ..Default::default()
             },
             TestData {
                 contents: "agent.trace",
-                tracing: tracer::TraceType::Isolated,
+                tracing: true,
                 ..Default::default()
             },
             TestData {
-                contents: "agent.trace=isolated",
-                tracing: tracer::TraceType::Isolated,
+                contents: "agent.trace=true",
+                tracing: true,
                 ..Default::default()
             },
             TestData {
-                contents: "agent.trace=disabled",
-                tracing: tracer::TraceType::Disabled,
+                contents: "agent.trace=false",
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.trace=0",
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.trace=1",
+                tracing: true,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.trace=a",
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.trace=foo",
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.trace=.",
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "agent.trace=,",
+                tracing: false,
                 ..Default::default()
             },
             TestData {
                 contents: "",
-                env_vars: vec!["KATA_AGENT_TRACE_TYPE=isolated"],
-                tracing: tracer::TraceType::Isolated,
+                env_vars: vec!["KATA_AGENT_TRACING="],
+                tracing: false,
                 ..Default::default()
             },
             TestData {
                 contents: "",
-                env_vars: vec!["KATA_AGENT_TRACE_TYPE=disabled"],
-                tracing: tracer::TraceType::Disabled,
+                env_vars: vec!["KATA_AGENT_TRACING=''"],
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_TRACING=0"],
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_TRACING=."],
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_TRACING=,"],
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_TRACING=foo"],
+                tracing: false,
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_TRACING=1"],
+                tracing: true,
+                ..Default::default()
+            },
+            TestData {
+                contents: "",
+                env_vars: vec!["KATA_AGENT_TRACING=true"],
+                tracing: true,
                 ..Default::default()
             },
         ];
 
         let dir = tempdir().expect("failed to create tmpdir");
 
-        // First, check a missing file is handled
-        let file_path = dir.path().join("enoent");
-
-        let filename = file_path.to_str().expect("failed to create filename");
-
-        let mut config = AgentConfig::new();
-        let result = config.parse_cmdline(&filename.to_owned());
-        assert!(result.is_err());
-
         // Now, test various combinations of file contents and environment
         // variables.
         for (i, d) in tests.iter().enumerate() {
@@ -753,22 +896,7 @@ mod tests {
                 vars_to_unset.push(name);
             }
 
-            let mut config = AgentConfig::new();
-            assert_eq!(config.debug_console, false, "{}", msg);
-            assert_eq!(config.dev_mode, false, "{}", msg);
-            assert_eq!(config.unified_cgroup_hierarchy, false, "{}", msg);
-            assert_eq!(
-                config.hotplug_timeout,
-                time::Duration::from_secs(3),
-                "{}",
-                msg
-            );
-            assert_eq!(config.container_pipe_size, 0, "{}", msg);
-            assert_eq!(config.server_addr, TEST_SERVER_ADDR, "{}", msg);
-            assert_eq!(config.tracing, tracer::TraceType::Disabled, "{}", msg);
-
-            let result = config.parse_cmdline(filename);
-            assert!(result.is_ok(), "{}", msg);
+            let config = AgentConfig::from_cmdline(filename).expect("Failed to parse command line");
 
             assert_eq!(d.debug_console, config.debug_console, "{}", msg);
             assert_eq!(d.dev_mode, config.dev_mode, "{}", msg);
@@ -1220,60 +1348,33 @@ Caused by:
     }
 
     #[test]
-    fn test_get_trace_type() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            param: &'a str,
-            result: Result<tracer::TraceType>,
-        }
+    fn test_config_builder_from_string() {
+        let config = AgentConfig::from_str(
+            r#"
+               dev_mode = true
+               server_addr = 'vsock://8:2048'
 
-        let tests = &[
-            TestData {
-                param: "",
-                result: Err(anyhow!(ERR_INVALID_TRACE_TYPE_PARAM)),
-            },
-            TestData {
-                param: "agent.tracer",
-                result: Err(anyhow!(ERR_INVALID_TRACE_TYPE_KEY)),
-            },
-            TestData {
-                param: "agent.trac",
-                result: Err(anyhow!(ERR_INVALID_TRACE_TYPE_KEY)),
-            },
-            TestData {
-                param: "agent.trace=",
-                result: Err(anyhow!(ERR_INVALID_TRACE_TYPE)),
-            },
-            TestData {
-                param: "agent.trace==",
-                result: Err(anyhow!(ERR_INVALID_TRACE_TYPE)),
-            },
-            TestData {
-                param: "agent.trace=foo",
-                result: Err(anyhow!(ERR_INVALID_TRACE_TYPE)),
-            },
-            TestData {
-                param: "agent.trace",
-                result: Ok(tracer::TraceType::Isolated),
-            },
-            TestData {
-                param: "agent.trace=isolated",
-                result: Ok(tracer::TraceType::Isolated),
-            },
-            TestData {
-                param: "agent.trace=disabled",
-                result: Ok(tracer::TraceType::Disabled),
-            },
-        ];
+               [endpoints]
+               allowed = ["CreateContainer", "StartContainer"]
+              "#,
+        )
+        .unwrap();
 
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
+        // Verify that the all_allowed flag is false
+        assert!(!config.endpoints.all_allowed);
 
-            let result = get_trace_type(d.param);
+        // Verify that the override worked
+        assert!(config.dev_mode);
+        assert_eq!(config.server_addr, "vsock://8:2048");
+        assert_eq!(
+            config.endpoints.allowed,
+            vec!["CreateContainer".to_string(), "StartContainer".to_string()]
+                .iter()
+                .cloned()
+                .collect()
+        );
 
-            let msg = format!("{}: result: {:?}", msg, result);
-
-            assert_result!(d.result, result, msg);
-        }
+        // Verify that the default values are valid
+        assert_eq!(config.hotplug_timeout, DEFAULT_HOTPLUG_TIMEOUT);
     }
 }

src/agent/src/device.rs

@@ -7,7 +7,10 @@ use libc::{c_uint, major, minor};
 use nix::sys::stat;
 use regex::Regex;
 use std::collections::HashMap;
+use std::ffi::OsStr;
+use std::fmt;
 use std::fs;
+use std::os::unix::ffi::OsStrExt;
 use std::os::unix::fs::MetadataExt;
 use std::path::Path;
 use std::str::FromStr;
@@ -17,10 +20,6 @@ use tokio::sync::Mutex;
 #[cfg(target_arch = "s390x")]
 use crate::ccw;
 use crate::linux_abi::*;
-use crate::mount::{
-    DRIVER_BLK_CCW_TYPE, DRIVER_BLK_TYPE, DRIVER_MMIO_BLK_TYPE, DRIVER_NVDIMM_TYPE,
-    DRIVER_SCSI_TYPE,
-};
 use crate::pci;
 use crate::sandbox::Sandbox;
 use crate::uevent::{wait_for_uevent, Uevent, UeventMatcher};
@@ -38,6 +37,22 @@ macro_rules! sl {
 
 const VM_ROOTFS: &str = "/";
 
+pub const DRIVER_9P_TYPE: &str = "9p";
+pub const DRIVER_VIRTIOFS_TYPE: &str = "virtio-fs";
+pub const DRIVER_BLK_TYPE: &str = "blk";
+pub const DRIVER_BLK_CCW_TYPE: &str = "blk-ccw";
+pub const DRIVER_MMIO_BLK_TYPE: &str = "mmioblk";
+pub const DRIVER_SCSI_TYPE: &str = "scsi";
+pub const DRIVER_NVDIMM_TYPE: &str = "nvdimm";
+pub const DRIVER_EPHEMERAL_TYPE: &str = "ephemeral";
+pub const DRIVER_LOCAL_TYPE: &str = "local";
+pub const DRIVER_WATCHABLE_BIND_TYPE: &str = "watchable-bind";
+// VFIO device to be bound to a guest kernel driver
+pub const DRIVER_VFIO_GK_TYPE: &str = "vfio-gk";
+// VFIO device to be bound to vfio-pci and made available inside the
+// container as a VFIO device node
+pub const DRIVER_VFIO_TYPE: &str = "vfio";
+
 #[derive(Debug)]
 struct DevIndexEntry {
     idx: usize,
@@ -47,27 +62,99 @@ struct DevIndexEntry {
 #[derive(Debug)]
 struct DevIndex(HashMap<String, DevIndexEntry>);
 
-#[instrument]
-pub fn rescan_pci_bus() -> Result<()> {
-    online_device(SYSFS_PCI_BUS_RESCAN_FILE)
-}
-
 #[instrument]
 pub fn online_device(path: &str) -> Result<()> {
     fs::write(path, "1")?;
     Ok(())
 }
 
+// Force a given PCI device to bind to the given driver, does
+// basically the same thing as
+//    driverctl set-override <PCI address> <driver>
+#[instrument]
+pub fn pci_driver_override<T, U>(syspci: T, dev: pci::Address, drv: U) -> Result<()>
+where
+    T: AsRef<OsStr> + std::fmt::Debug,
+    U: AsRef<OsStr> + std::fmt::Debug,
+{
+    let syspci = Path::new(&syspci);
+    let drv = drv.as_ref();
+    info!(sl!(), "rebind_pci_driver: {} => {:?}", dev, drv);
+
+    let devpath = syspci.join("devices").join(dev.to_string());
+    let overridepath = &devpath.join("driver_override");
+
+    fs::write(overridepath, drv.as_bytes())?;
+
+    let drvpath = &devpath.join("driver");
+    let need_unbind = match fs::read_link(drvpath) {
+        Ok(d) if d.file_name() == Some(drv) => return Ok(()), // Nothing to do
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => false, // No current driver
+        Err(e) => return Err(anyhow!("Error checking driver on {}: {}", dev, e)),
+        Ok(_) => true, // Current driver needs unbinding
+    };
+    if need_unbind {
+        let unbindpath = &drvpath.join("unbind");
+        fs::write(unbindpath, dev.to_string())?;
+    }
+    let probepath = syspci.join("drivers_probe");
+    fs::write(probepath, dev.to_string())?;
+    Ok(())
+}
+
+// Represents an IOMMU group
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct IommuGroup(u32);
+
+impl fmt::Display for IommuGroup {
+    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+        write!(f, "{}", self.0)
+    }
+}
+
+// Determine the IOMMU group of a PCI device
+#[instrument]
+fn pci_iommu_group<T>(syspci: T, dev: pci::Address) -> Result<Option<IommuGroup>>
+where
+    T: AsRef<OsStr> + std::fmt::Debug,
+{
+    let syspci = Path::new(&syspci);
+    let grouppath = syspci
+        .join("devices")
+        .join(dev.to_string())
+        .join("iommu_group");
+
+    match fs::read_link(&grouppath) {
+        // Device has no group
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
+        Err(e) => Err(anyhow!("Error reading link {:?}: {}", &grouppath, e)),
+        Ok(group) => {
+            if let Some(group) = group.file_name() {
+                if let Some(group) = group.to_str() {
+                    if let Ok(group) = group.parse::<u32>() {
+                        return Ok(Some(IommuGroup(group)));
+                    }
+                }
+            }
+            Err(anyhow!(
+                "Unexpected IOMMU group link {:?} => {:?}",
+                grouppath,
+                group
+            ))
+        }
+    }
+}
+
 // pcipath_to_sysfs fetches the sysfs path for a PCI path, relative to
 // the sysfs path for the PCI host bridge, based on the PCI path
 // provided.
 #[instrument]
-fn pcipath_to_sysfs(root_bus_sysfs: &str, pcipath: &pci::Path) -> Result<String> {
+pub fn pcipath_to_sysfs(root_bus_sysfs: &str, pcipath: &pci::Path) -> Result<String> {
     let mut bus = "0000:00".to_string();
     let mut relpath = String::new();
 
     for i in 0..pcipath.len() {
-        let bdf = format!("{}:{}.0", bus, pcipath[i]);
+        let bdf = format!("{}:{}", bus, pcipath[i]);
 
         relpath = format!("{}/{}", relpath, bdf);
 
@@ -162,8 +249,6 @@ pub async fn get_virtio_blk_pci_device_name(
     let sysfs_rel_path = pcipath_to_sysfs(&root_bus_sysfs, pcipath)?;
     let matcher = VirtioBlkPciMatcher::new(&sysfs_rel_path);
 
-    rescan_pci_bus()?;
-
     let uev = wait_for_uevent(sandbox, matcher).await?;
     Ok(format!("{}/{}", SYSTEM_DEV_PATH, &uev.devname))
 }
@@ -255,6 +340,72 @@ pub async fn wait_for_pmem_device(sandbox: &Arc<Mutex<Sandbox>>, devpath: &str)
     Ok(())
 }
 
+#[derive(Debug)]
+struct PciMatcher {
+    devpath: String,
+}
+
+impl PciMatcher {
+    fn new(relpath: &str) -> Result<PciMatcher> {
+        let root_bus = create_pci_root_bus_path();
+        Ok(PciMatcher {
+            devpath: format!("{}{}", root_bus, relpath),
+        })
+    }
+}
+
+impl UeventMatcher for PciMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.devpath == self.devpath
+    }
+}
+
+pub async fn wait_for_pci_device(
+    sandbox: &Arc<Mutex<Sandbox>>,
+    pcipath: &pci::Path,
+) -> Result<pci::Address> {
+    let root_bus_sysfs = format!("{}{}", SYSFS_DIR, create_pci_root_bus_path());
+    let sysfs_rel_path = pcipath_to_sysfs(&root_bus_sysfs, pcipath)?;
+    let matcher = PciMatcher::new(&sysfs_rel_path)?;
+
+    let uev = wait_for_uevent(sandbox, matcher).await?;
+
+    let addr = uev
+        .devpath
+        .rsplit('/')
+        .next()
+        .ok_or_else(|| anyhow!("Bad device path {:?} in uevent", &uev.devpath))?;
+    let addr = pci::Address::from_str(addr)?;
+    Ok(addr)
+}
+
+#[derive(Debug)]
+struct VfioMatcher {
+    syspath: String,
+}
+
+impl VfioMatcher {
+    fn new(grp: IommuGroup) -> VfioMatcher {
+        VfioMatcher {
+            syspath: format!("/devices/virtual/vfio/{}", grp),
+        }
+    }
+}
+
+impl UeventMatcher for VfioMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.devpath == self.syspath
+    }
+}
+
+#[instrument]
+async fn get_vfio_device_name(sandbox: &Arc<Mutex<Sandbox>>, grp: IommuGroup) -> Result<String> {
+    let matcher = VfioMatcher::new(grp);
+
+    let uev = wait_for_uevent(sandbox, matcher).await?;
+    Ok(format!("{}/{}", SYSTEM_DEV_PATH, &uev.devname))
+}
+
 /// Scan SCSI bus for the given SCSI address(SCSI-Id and LUN)
 #[instrument]
 fn scan_scsi_bus(scsi_addr: &str) -> Result<()> {
@@ -285,24 +436,27 @@ fn scan_scsi_bus(scsi_addr: &str) -> Result<()> {
     Ok(())
 }
 
-// update_spec_device_list takes a device description provided by the caller,
-// trying to find it on the guest. Once this device has been identified, the
-// "real" information that can be read from inside the VM is used to update
-// the same device in the list of devices provided through the OCI spec.
-// This is needed to update information about minor/major numbers that cannot
-// be predicted from the caller.
+// update_spec_device updates the device list in the OCI spec to make
+// it include details appropriate for the VM, instead of the host.  It
+// is given the host path to the device (to locate the device in the
+// original OCI spec) and the VM path which it uses to determine the
+// VM major/minor numbers, and the final path with which to present
+// the device in the (inner) container
 #[instrument]
-fn update_spec_device_list(device: &Device, spec: &mut Spec, devidx: &DevIndex) -> Result<()> {
+fn update_spec_device(
+    spec: &mut Spec,
+    devidx: &DevIndex,
+    host_path: &str,
+    vm_path: &str,
+    final_path: &str,
+) -> Result<()> {
     let major_id: c_uint;
     let minor_id: c_uint;
 
     // If no container_path is provided, we won't be able to match and
     // update the device in the OCI spec device list. This is an error.
-    if device.container_path.is_empty() {
-        return Err(anyhow!(
-            "container_path cannot empty for device {:?}",
-            device
-        ));
+    if host_path.is_empty() {
+        return Err(anyhow!("Host path cannot empty for device"));
     }
 
     let linux = spec
@@ -310,11 +464,11 @@ fn update_spec_device_list(device: &Device, spec: &mut Spec, devidx: &DevIndex)
         .as_mut()
         .ok_or_else(|| anyhow!("Spec didn't container linux field"))?;
 
-    if !Path::new(&device.vm_path).exists() {
-        return Err(anyhow!("vm_path:{} doesn't exist", device.vm_path));
+    if !Path::new(vm_path).exists() {
+        return Err(anyhow!("vm_path:{} doesn't exist", vm_path));
     }
 
-    let meta = fs::metadata(&device.vm_path)?;
+    let meta = fs::metadata(vm_path)?;
     let dev_id = meta.rdev();
     unsafe {
         major_id = major(dev_id);
@@ -323,24 +477,27 @@ fn update_spec_device_list(device: &Device, spec: &mut Spec, devidx: &DevIndex)
 
     info!(
         sl!(),
-        "got the device: dev_path: {}, major: {}, minor: {}\n", &device.vm_path, major_id, minor_id
+        "update_spec_device(): vm_path={}, major: {}, minor: {}\n", vm_path, major_id, minor_id
     );
 
-    if let Some(idxdata) = devidx.0.get(device.container_path.as_str()) {
+    if let Some(idxdata) = devidx.0.get(host_path) {
         let dev = &mut linux.devices[idxdata.idx];
         let host_major = dev.major;
         let host_minor = dev.minor;
 
         dev.major = major_id as i64;
         dev.minor = minor_id as i64;
+        dev.path = final_path.to_string();
 
         info!(
             sl!(),
-            "change the device from major: {} minor: {} to vm device major: {} minor: {}",
+            "change the device from path: {} major: {} minor: {} to vm device path: {} major: {} minor: {}",
+            host_path,
             host_major,
             host_minor,
-            major_id,
-            minor_id
+            dev.path,
+            dev.major,
+            dev.minor,
         );
 
         // Resources must be updated since they are used to identify
@@ -361,7 +518,7 @@ fn update_spec_device_list(device: &Device, spec: &mut Spec, devidx: &DevIndex)
     } else {
         Err(anyhow!(
             "Should have found a matching device {} in the spec",
-            device.vm_path
+            vm_path
         ))
     }
 }
@@ -379,7 +536,13 @@ async fn virtiommio_blk_device_handler(
         return Err(anyhow!("Invalid path for virtio mmio blk device"));
     }
 
-    update_spec_device_list(device, spec, devidx)
+    update_spec_device(
+        spec,
+        devidx,
+        &device.container_path,
+        &device.vm_path,
+        &device.container_path,
+    )
 }
 
 // device.Id should be a PCI path string
@@ -395,7 +558,13 @@ async fn virtio_blk_device_handler(
 
     dev.vm_path = get_virtio_blk_pci_device_name(sandbox, &pcipath).await?;
 
-    update_spec_device_list(&dev, spec, devidx)
+    update_spec_device(
+        spec,
+        devidx,
+        &dev.container_path,
+        &dev.vm_path,
+        &dev.container_path,
+    )
 }
 
 // device.id should be a CCW path string
@@ -410,7 +579,13 @@ async fn virtio_blk_ccw_device_handler(
     let mut dev = device.clone();
     let ccw_device = ccw::Device::from_str(&device.id)?;
     dev.vm_path = get_virtio_blk_ccw_device_name(sandbox, &ccw_device).await?;
-    update_spec_device_list(&dev, spec, devidx)
+    update_spec_device(
+        spec,
+        devidx,
+        &dev.container_path,
+        &dev.vm_path,
+        &dev.container_path,
+    )
 }
 
 #[cfg(not(target_arch = "s390x"))]
@@ -434,7 +609,13 @@ async fn virtio_scsi_device_handler(
 ) -> Result<()> {
     let mut dev = device.clone();
     dev.vm_path = get_scsi_device_name(sandbox, &device.id).await?;
-    update_spec_device_list(&dev, spec, devidx)
+    update_spec_device(
+        spec,
+        devidx,
+        &dev.container_path,
+        &dev.vm_path,
+        &dev.container_path,
+    )
 }
 
 #[instrument]
@@ -448,7 +629,79 @@ async fn virtio_nvdimm_device_handler(
         return Err(anyhow!("Invalid path for nvdimm device"));
     }
 
-    update_spec_device_list(device, spec, devidx)
+    update_spec_device(
+        spec,
+        devidx,
+        &device.container_path,
+        &device.vm_path,
+        &device.container_path,
+    )
+}
+
+fn split_vfio_option(opt: &str) -> Option<(&str, &str)> {
+    let mut tokens = opt.split('=');
+    let hostbdf = tokens.next()?;
+    let path = tokens.next()?;
+    if tokens.next().is_some() {
+        None
+    } else {
+        Some((hostbdf, path))
+    }
+}
+
+// device.options should have one entry for each PCI device in the VFIO group
+// Each option should have the form "DDDD:BB:DD.F=<pcipath>"
+//     DDDD:BB:DD.F is the device's PCI address in the host
+//     <pcipath> is a PCI path to the device in the guest (see pci.rs)
+async fn vfio_device_handler(
+    device: &Device,
+    spec: &mut Spec,
+    sandbox: &Arc<Mutex<Sandbox>>,
+    devidx: &DevIndex,
+) -> Result<()> {
+    let vfio_in_guest = device.field_type != DRIVER_VFIO_GK_TYPE;
+    let mut group = None;
+
+    for opt in device.options.iter() {
+        let (_, pcipath) =
+            split_vfio_option(opt).ok_or_else(|| anyhow!("Malformed VFIO option {:?}", opt))?;
+        let pcipath = pci::Path::from_str(pcipath)?;
+
+        let guestdev = wait_for_pci_device(sandbox, &pcipath).await?;
+        if vfio_in_guest {
+            pci_driver_override(SYSFS_BUS_PCI_PATH, guestdev, "vfio-pci")?;
+
+            let devgroup = pci_iommu_group(SYSFS_BUS_PCI_PATH, guestdev)?;
+            if devgroup.is_none() {
+                // Devices must have an IOMMU group to be usable via VFIO
+                return Err(anyhow!("{} has no IOMMU group", guestdev));
+            }
+
+            if group.is_some() && group != devgroup {
+                // If PCI devices associated with the same VFIO device
+                // (and therefore group) in the host don't end up in
+                // the same group in the guest, something has gone
+                // horribly wrong
+                return Err(anyhow!(
+                    "{} is not in guest IOMMU group {}",
+                    guestdev,
+                    group.unwrap()
+                ));
+            }
+
+            group = devgroup;
+        }
+    }
+
+    if vfio_in_guest {
+        // If there are any devices at all, logic above ensures that group is not None
+        let group = group.unwrap();
+        let vmpath = get_vfio_device_name(sandbox, group).await?;
+
+        update_spec_device(spec, devidx, &device.container_path, &vmpath, &vmpath)?;
+    }
+
+    Ok(())
 }
 
 impl DevIndex {
@@ -520,6 +773,9 @@ async fn add_device(
         DRIVER_MMIO_BLK_TYPE => virtiommio_blk_device_handler(device, spec, sandbox, devidx).await,
         DRIVER_NVDIMM_TYPE => virtio_nvdimm_device_handler(device, spec, sandbox, devidx).await,
         DRIVER_SCSI_TYPE => virtio_scsi_device_handler(device, spec, sandbox, devidx).await,
+        DRIVER_VFIO_GK_TYPE | DRIVER_VFIO_TYPE => {
+            vfio_device_handler(device, spec, sandbox, devidx).await
+        }
         _ => Err(anyhow!("Unknown device type {}", device.field_type)),
     }
 }
@@ -584,28 +840,28 @@ mod tests {
     }
 
     #[test]
-    fn test_update_spec_device_list() {
+    fn test_update_spec_device() {
         let (major, minor) = (7, 2);
-        let mut device = Device::default();
         let mut spec = Spec::default();
 
         // container_path empty
+        let container_path = "";
+        let vm_path = "";
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_err());
 
-        device.container_path = "/dev/null".to_string();
-
         // linux is empty
+        let container_path = "/dev/null";
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_err());
 
         spec.linux = Some(Linux::default());
 
         // linux.devices is empty
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_err());
 
         spec.linux.as_mut().unwrap().devices = vec![oci::LinuxDevice {
@@ -617,26 +873,32 @@ mod tests {
 
         // vm_path empty
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_err());
 
-        device.vm_path = "/dev/null".to_string();
+        let vm_path = "/dev/null";
 
         // guest and host path are not the same
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
-        assert!(res.is_err(), "device={:?} spec={:?}", device, spec);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
+        assert!(
+            res.is_err(),
+            "container_path={:?} vm_path={:?} spec={:?}",
+            container_path,
+            vm_path,
+            spec
+        );
 
-        spec.linux.as_mut().unwrap().devices[0].path = device.container_path.clone();
+        spec.linux.as_mut().unwrap().devices[0].path = container_path.to_string();
 
         // spec.linux.resources is empty
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_ok());
 
         // update both devices and cgroup lists
         spec.linux.as_mut().unwrap().devices = vec![oci::LinuxDevice {
-            path: device.container_path.clone(),
+            path: container_path.to_string(),
             major,
             minor,
             ..oci::LinuxDevice::default()
@@ -652,12 +914,12 @@ mod tests {
         });
 
         let devidx = DevIndex::new(&spec);
-        let res = update_spec_device_list(&device, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_ok());
     }
 
     #[test]
-    fn test_update_spec_device_list_guest_host_conflict() {
+    fn test_update_spec_device_guest_host_conflict() {
         let null_rdev = fs::metadata("/dev/null").unwrap().rdev();
         let zero_rdev = fs::metadata("/dev/zero").unwrap().rdev();
         let full_rdev = fs::metadata("/dev/full").unwrap().rdev();
@@ -708,20 +970,14 @@ mod tests {
         };
         let devidx = DevIndex::new(&spec);
 
-        let dev_a = Device {
-            container_path: "/dev/a".to_string(),
-            vm_path: "/dev/zero".to_string(),
-            ..Device::default()
-        };
+        let container_path_a = "/dev/a";
+        let vm_path_a = "/dev/zero";
 
         let guest_major_a = stat::major(zero_rdev) as i64;
         let guest_minor_a = stat::minor(zero_rdev) as i64;
 
-        let dev_b = Device {
-            container_path: "/dev/b".to_string(),
-            vm_path: "/dev/full".to_string(),
-            ..Device::default()
-        };
+        let container_path_b = "/dev/b";
+        let vm_path_b = "/dev/full";
 
         let guest_major_b = stat::major(full_rdev) as i64;
         let guest_minor_b = stat::minor(full_rdev) as i64;
@@ -738,7 +994,13 @@ mod tests {
         assert_eq!(Some(host_major_b), specresources.devices[1].major);
         assert_eq!(Some(host_minor_b), specresources.devices[1].minor);
 
-        let res = update_spec_device_list(&dev_a, &mut spec, &devidx);
+        let res = update_spec_device(
+            &mut spec,
+            &devidx,
+            container_path_a,
+            vm_path_a,
+            container_path_a,
+        );
         assert!(res.is_ok());
 
         let specdevices = &spec.linux.as_ref().unwrap().devices;
@@ -753,7 +1015,13 @@ mod tests {
         assert_eq!(Some(host_major_b), specresources.devices[1].major);
         assert_eq!(Some(host_minor_b), specresources.devices[1].minor);
 
-        let res = update_spec_device_list(&dev_b, &mut spec, &devidx);
+        let res = update_spec_device(
+            &mut spec,
+            &devidx,
+            container_path_b,
+            vm_path_b,
+            container_path_b,
+        );
         assert!(res.is_ok());
 
         let specdevices = &spec.linux.as_ref().unwrap().devices;
@@ -770,7 +1038,7 @@ mod tests {
     }
 
     #[test]
-    fn test_update_spec_device_list_char_block_conflict() {
+    fn test_update_spec_device_char_block_conflict() {
         let null_rdev = fs::metadata("/dev/null").unwrap().rdev();
 
         let guest_major = stat::major(null_rdev) as i64;
@@ -819,11 +1087,8 @@ mod tests {
         };
         let devidx = DevIndex::new(&spec);
 
-        let dev = Device {
-            container_path: "/dev/char".to_string(),
-            vm_path: "/dev/null".to_string(),
-            ..Device::default()
-        };
+        let container_path = "/dev/char";
+        let vm_path = "/dev/null";
 
         let specresources = spec.linux.as_ref().unwrap().resources.as_ref().unwrap();
         assert_eq!(Some(host_major), specresources.devices[0].major);
@@ -831,7 +1096,7 @@ mod tests {
         assert_eq!(Some(host_major), specresources.devices[1].major);
         assert_eq!(Some(host_minor), specresources.devices[1].minor);
 
-        let res = update_spec_device_list(&dev, &mut spec, &devidx);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_ok());
 
         // Only the char device, not the block device should be updated
@@ -842,6 +1107,43 @@ mod tests {
         assert_eq!(Some(host_minor), specresources.devices[1].minor);
     }
 
+    #[test]
+    fn test_update_spec_device_final_path() {
+        let null_rdev = fs::metadata("/dev/null").unwrap().rdev();
+        let guest_major = stat::major(null_rdev) as i64;
+        let guest_minor = stat::minor(null_rdev) as i64;
+
+        let host_path = "/dev/host";
+        let host_major: i64 = 99;
+        let host_minor: i64 = 99;
+
+        let mut spec = Spec {
+            linux: Some(Linux {
+                devices: vec![oci::LinuxDevice {
+                    path: host_path.to_string(),
+                    r#type: "c".to_string(),
+                    major: host_major,
+                    minor: host_minor,
+                    ..oci::LinuxDevice::default()
+                }],
+                ..Linux::default()
+            }),
+            ..Spec::default()
+        };
+        let devidx = DevIndex::new(&spec);
+
+        let vm_path = "/dev/null";
+        let final_path = "/dev/final";
+
+        let res = update_spec_device(&mut spec, &devidx, host_path, vm_path, final_path);
+        assert!(res.is_ok());
+
+        let specdevices = &spec.linux.as_ref().unwrap().devices;
+        assert_eq!(guest_major, specdevices[0].major);
+        assert_eq!(guest_minor, specdevices[0].minor);
+        assert_eq!(final_path, specdevices[0].path);
+    }
+
     #[test]
     fn test_pcipath_to_sysfs() {
         let testdir = tempdir().expect("failed to create tmpdir");
@@ -966,12 +1268,12 @@ mod tests {
         uev_a.subsystem = "block".to_string();
         uev_a.devname = devname.to_string();
         uev_a.devpath = format!("{}{}/virtio4/block/{}", root_bus, relpath_a, devname);
-        let matcher_a = VirtioBlkPciMatcher::new(&relpath_a);
+        let matcher_a = VirtioBlkPciMatcher::new(relpath_a);
 
         let mut uev_b = uev_a.clone();
         let relpath_b = "/0000:00:0a.0/0000:00:0b.0";
         uev_b.devpath = format!("{}{}/virtio0/block/{}", root_bus, relpath_b, devname);
-        let matcher_b = VirtioBlkPciMatcher::new(&relpath_b);
+        let matcher_b = VirtioBlkPciMatcher::new(relpath_b);
 
         assert!(matcher_a.is_match(&uev_a));
         assert!(matcher_b.is_match(&uev_b));
@@ -1053,7 +1355,7 @@ mod tests {
             "{}/0000:00:00.0/virtio0/host0/target0:0:0/0:0:{}/block/sda",
             root_bus, addr_a
         );
-        let matcher_a = ScsiBlockMatcher::new(&addr_a);
+        let matcher_a = ScsiBlockMatcher::new(addr_a);
 
         let mut uev_b = uev_a.clone();
         let addr_b = "2:0";
@@ -1061,11 +1363,119 @@ mod tests {
             "{}/0000:00:00.0/virtio0/host0/target0:0:2/0:0:{}/block/sdb",
             root_bus, addr_b
         );
-        let matcher_b = ScsiBlockMatcher::new(&addr_b);
+        let matcher_b = ScsiBlockMatcher::new(addr_b);
 
         assert!(matcher_a.is_match(&uev_a));
         assert!(matcher_b.is_match(&uev_b));
         assert!(!matcher_b.is_match(&uev_a));
         assert!(!matcher_a.is_match(&uev_b));
     }
+
+    #[tokio::test]
+    async fn test_vfio_matcher() {
+        let grpa = IommuGroup(1);
+        let grpb = IommuGroup(22);
+
+        let mut uev_a = crate::uevent::Uevent::default();
+        uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
+        uev_a.devname = format!("vfio/{}", grpa);
+        uev_a.devpath = format!("/devices/virtual/vfio/{}", grpa);
+        let matcher_a = VfioMatcher::new(grpa);
+
+        let mut uev_b = uev_a.clone();
+        uev_b.devpath = format!("/devices/virtual/vfio/{}", grpb);
+        let matcher_b = VfioMatcher::new(grpb);
+
+        assert!(matcher_a.is_match(&uev_a));
+        assert!(matcher_b.is_match(&uev_b));
+        assert!(!matcher_b.is_match(&uev_a));
+        assert!(!matcher_a.is_match(&uev_b));
+    }
+
+    #[test]
+    fn test_split_vfio_option() {
+        assert_eq!(
+            split_vfio_option("0000:01:00.0=02/01"),
+            Some(("0000:01:00.0", "02/01"))
+        );
+        assert_eq!(split_vfio_option("0000:01:00.0=02/01=rubbish"), None);
+        assert_eq!(split_vfio_option("0000:01:00.0"), None);
+    }
+
+    #[test]
+    fn test_pci_driver_override() {
+        let testdir = tempdir().expect("failed to create tmpdir");
+        let syspci = testdir.path(); // Path to mock /sys/bus/pci
+
+        let dev0 = pci::Address::new(0, 0, pci::SlotFn::new(0, 0).unwrap());
+        let dev0path = syspci.join("devices").join(dev0.to_string());
+        let dev0drv = dev0path.join("driver");
+        let dev0override = dev0path.join("driver_override");
+
+        let drvapath = syspci.join("drivers").join("drv_a");
+        let drvaunbind = drvapath.join("unbind");
+
+        let probepath = syspci.join("drivers_probe");
+
+        // Start mocking dev0 as being unbound
+        fs::create_dir_all(&dev0path).unwrap();
+
+        pci_driver_override(syspci, dev0, "drv_a").unwrap();
+        assert_eq!(fs::read_to_string(&dev0override).unwrap(), "drv_a");
+        assert_eq!(fs::read_to_string(&probepath).unwrap(), dev0.to_string());
+
+        // Now mock dev0 already being attached to drv_a
+        fs::create_dir_all(&drvapath).unwrap();
+        std::os::unix::fs::symlink(&drvapath, dev0drv).unwrap();
+        std::fs::remove_file(&probepath).unwrap();
+
+        pci_driver_override(syspci, dev0, "drv_a").unwrap(); // no-op
+        assert_eq!(fs::read_to_string(&dev0override).unwrap(), "drv_a");
+        assert!(!probepath.exists());
+
+        // Now try binding to a different driver
+        pci_driver_override(syspci, dev0, "drv_b").unwrap();
+        assert_eq!(fs::read_to_string(&dev0override).unwrap(), "drv_b");
+        assert_eq!(fs::read_to_string(&probepath).unwrap(), dev0.to_string());
+        assert_eq!(fs::read_to_string(&drvaunbind).unwrap(), dev0.to_string());
+    }
+
+    #[test]
+    fn test_pci_iommu_group() {
+        let testdir = tempdir().expect("failed to create tmpdir"); // mock /sys
+        let syspci = testdir.path().join("bus").join("pci");
+
+        // Mock dev0, which has no group
+        let dev0 = pci::Address::new(0, 0, pci::SlotFn::new(0, 0).unwrap());
+        let dev0path = syspci.join("devices").join(dev0.to_string());
+
+        fs::create_dir_all(&dev0path).unwrap();
+
+        // Test dev0
+        assert!(pci_iommu_group(&syspci, dev0).unwrap().is_none());
+
+        // Mock dev1, which is in group 12
+        let dev1 = pci::Address::new(0, 1, pci::SlotFn::new(0, 0).unwrap());
+        let dev1path = syspci.join("devices").join(dev1.to_string());
+        let dev1group = dev1path.join("iommu_group");
+
+        fs::create_dir_all(&dev1path).unwrap();
+        std::os::unix::fs::symlink("../../../kernel/iommu_groups/12", &dev1group).unwrap();
+
+        // Test dev1
+        assert_eq!(
+            pci_iommu_group(&syspci, dev1).unwrap(),
+            Some(IommuGroup(12))
+        );
+
+        // Mock dev2, which has a bogus group (dir instead of symlink)
+        let dev2 = pci::Address::new(0, 2, pci::SlotFn::new(0, 0).unwrap());
+        let dev2path = syspci.join("devices").join(dev2.to_string());
+        let dev2group = dev2path.join("iommu_group");
+
+        fs::create_dir_all(&dev2group).unwrap();
+
+        // Test dev2
+        assert!(pci_iommu_group(&syspci, dev2).is_err());
+    }
 }

src/agent/src/linux_abi.rs

@@ -9,7 +9,6 @@
 use std::fs;
 
 pub const SYSFS_DIR: &str = "/sys";
-pub const SYSFS_PCI_BUS_RESCAN_FILE: &str = "/sys/bus/pci/rescan";
 #[cfg(any(
     target_arch = "powerpc64",
     target_arch = "s390x",
@@ -84,6 +83,8 @@ pub const SYSFS_MEMORY_ONLINE_PATH: &str = "/sys/devices/system/memory";
 
 pub const SYSFS_SCSI_HOST_PATH: &str = "/sys/class/scsi_host";
 
+pub const SYSFS_BUS_PCI_PATH: &str = "/sys/bus/pci";
+
 pub const SYSFS_CGROUPPATH: &str = "/sys/fs/cgroup";
 pub const SYSFS_ONLINE_FILE: &str = "online";
 
@@ -95,6 +96,7 @@ pub const SYSTEM_DEV_PATH: &str = "/dev";
 // Linux UEvent related consts.
 pub const U_EVENT_ACTION: &str = "ACTION";
 pub const U_EVENT_ACTION_ADD: &str = "add";
+pub const U_EVENT_ACTION_REMOVE: &str = "remove";
 pub const U_EVENT_DEV_PATH: &str = "DEVPATH";
 pub const U_EVENT_SUB_SYSTEM: &str = "SUBSYSTEM";
 pub const U_EVENT_SEQ_NUM: &str = "SEQNUM";